def run(path0, silent=False, verbose=True): ''' Main function to run for a given path containing the raw GNIRS data Parameters ---------- path0 : str Path to raw FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 22 March 2017 Modified by Chun Ly, 23 March 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 8 January 2018 - Import glog and call for stdout and ASCII logging - Pass mylogger to get_files() Modified by Chun Ly, 7 May 2018 - Bug fix: Call check_path() to add '/' in path0 ''' rawdir = check_path(path0) # + on 07/05/2018 # + on 08/01/2018 logfile = path0+'symlink.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin run : '+systime()) # + on 23/03/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) # Mod on 23/03/2017 for path in list_path: files, n_files = get_files(path, mylogger=mylogger, silent=silent, verbose=verbose) for nn in xrange(n_files): c_file = path+'c'+files[nn] if exists(c_file): mylogger.warn('File exists : '+c_file) else: cmd0 = 'ln -fs '+files[nn]+' '+c_file if silent == False: mylogger.info(cmd0) os.system(cmd0) #endfor #endfor if silent == False: mylogger.info('### End run : '+systime())
def delete(path0, silent=False, verbose=True): ''' Remove all symbolic links in given path containing the raw GNIRS data Parameters ---------- path0 : str Path to raw FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 22 March 2017 Modified by Chun Ly, 8 January 2018 - Import glog and call for stdout and ASCII logging - Pass mylogger to get_files() Modified by Chun Ly, 22 January 2018 - Pass rm command strings to mylogger ''' # + on 08/01/2018 logfile = path0+'symlink.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin delete : '+systime()) files, n_files = get_files(path0, mylogger=mylogger, silent=silent, verbose=verbose) for nn in xrange(n_files): c_file = path0+'c'+files[nn] if exists(c_file): if os.path.islink(c_file) == True: cmd0 = 'rm '+c_file mylogger.info(cmd0) os.system(cmd0) else: mylogger.info('File is from cleanir: '+c_file) if silent == False: mylogger.info('### End delete : '+systime())
def residual_wave_cal(path, dataset='', cal='', silent=False, verbose=True): ''' Plot residuals of arc/OH lines against OH/arc dataset Parameters ---------- path : str Full path to where output PDF and FITS file are located. Must end with a '/' dataset : str Either: 'arc' or 'OH' cal : str Either 'arc' or 'OH To use OH calib on OH lines: dataset='OH', cal='OH' To use OH calib on arc lines: dataset='arc', cal='OH' To use arc calib on arc lines: dataset='arc', cal='arc' To use arc calib on OH lines: dataset='OH', cal='arc' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 4 June 2018 Modified by Chun Ly, 5 June 2018 - Define and read in OH/arc line database - Import curve_fit - Import gauss1d from IQ_plot - Call curve_fit to fit Gaussian profiles to line Modified by Chun Ly, 7 June 2018 - Plot residuals of wavelength solution - Set minimum line peak brightness to fit - Limit plotting to non-zero values for central wavelength - Plot rms for each line - Write average array to FITS file - Plot aesthetics - Move mylogger.info() call into else statement - Handle dataset == cal cases - Save average and rms for each arc/OH line - Handle dataset == cal cases for output PDF file - Plot aesthetics: Draw vertical lines for location of lines - Handle dataset == cal cases for output FITS file - If statement for lines with fits - Write ASCII with averages and RMS file Modified by Chun Ly, 19 June 2018 - Determine and plot average, median, and rms for each column - Multi-page PDF: Separate plot illustrating difference from model along longslit - Plot sigma on second page, Exclude no value cases - Add ax.annotation in upper left of second page - Call get_database_model, include fitting model - Overwrite ASCII tables in asc.write Modified by Chun Ly, 20 June 2018 - Use FITS file if it exists, otherwise generate - Use sigma_clipped_stats to remove outliers in statistics - Remove outlier based on offset amount, not using sigma clipping - Tab issue: Move fits.writeto out of for loop - Compute stats for histogram distributions - try/except for OH skyline failure (RunTimeError) Modified by Chun Ly, 21 June 2018 - Use convolved Rousselot2000 table instead of original - Read in OH npz file for line grouping - Define use_lines array for arc case - Handle multi-line fitting (works with dataset='arc') - Handle multi-line fitting for dataset='OH' - Include lower and upper bounds, Move multi-line fitting later in code - Normalize spectrum to peak, Do not set bounds for curve_fit() - Fix typo with np.absolute use - Implement weighted average calculation for centers to expedite calculation - Mark arc/OH lines that are not in database (potentially problematic) - Ignore flag arc/OH lines in stats along X Modified by Chun Ly, 21 June 2018 - Use default Rousselot table if convolved one not available Note: This is OK with previous version of code that use the default table for wavelength calibration (i.e., line_flag won't be all 1's) - Handle old reduction when npz file is unavailable ''' logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin residual_wave_cal : '+systime()) if dataset != cal: infile = path+'tf'+dataset+'_stack_'+cal+'.fits' else: infile = path+'tf'+dataset+'_stack.fits' if not exists(infile): mylogger.warn('File not found : '+infile) mylogger.warn('Exiting!!!') return else: if silent == False: mylogger.info('Reading : '+infile) cal_2D, cal_hdr = fits.getdata(infile, extname='SCI', header=True) NX = cal_2D.shape[1] # spatial direction NY = cal_2D.shape[0] # dispersion direction lam0 = cal_hdr['CRVAL2'] dlam = cal_hdr['CD2_2'] wave0 = lam0 + dlam * np.arange(NY) bins_mid = np.arange(13,NX,10) n_bins = len(bins_mid) if dataset != cal: out_fits = path+'wave_cal_resid_'+dataset+'_'+cal+'.fits' else: out_fits = path+'wave_cal_resid_'+dataset+'.fits' if not exists(out_fits): avg_arr = np.zeros( (NY, n_bins) ) for ii in range(n_bins): start = np.max([0,bins_mid[ii]-1 - 5]) stop = np.min([NX-1,bins_mid[ii]-1 + 5]) avg_arr[:,ii] = np.average(cal_2D[:,start:stop], axis=1) if silent == False: mylogger.info('Writing : '+out_fits) fits.writeto(out_fits, avg_arr, overwrite=True) else: if silent == False: mylogger.info('Reading : '+out_fits) avg_arr = fits.getdata(out_fits) if dataset == 'OH': cal_ref_file = path+'rousselot2000_convl.dat' if not exists(cal_ref_file): cal_ref_file = co_dirname+'/rousselot2000.dat' mylogger.warn('Using default Rousselot2000 table, not convolved one!') if dataset == 'arc': cal_ref_file = co_dirname+'/argon.dat' if silent == False: mylogger.info('Reading : '+cal_ref_file) cal_line_data = asc.read(cal_ref_file, format='no_header') cal_lines = cal_line_data['col1'].data skip = np.zeros(len(cal_lines)) if dataset == 'OH': npz_file = path+'rousselot2000_convl.npz' if exists(npz_file): mylogger.info('Reading : '+npz_file) npz_tab = np.load(npz_file) use_lines = npz_tab['use_lines'] for gg in range(len(use_lines)): matches = [xx for xx in range(len(cal_lines)) if cal_lines[xx] in use_lines[gg]] if len(matches) > 1: skip[np.array(matches[1:])] = 1 in_spec = np.where((cal_lines >= wave0[0]) & (cal_lines <= wave0[-1]))[0] cal_line_data = cal_line_data[in_spec] cal_lines = cal_line_data['col1'].data skip = skip[in_spec] if dataset == 'OH': cal_lines_int = cal_line_data['col2'].data # Remove lines from use_lines if outside coverage | + on 21/06/2018 if exists(npz_file): n_in_spec = np.zeros(len(use_lines)) for gg in range(len(use_lines)): in_spec = np.where((use_lines[gg] >= wave0[0]) & (use_lines[gg] <= wave0[-1]))[0] n_in_spec[gg] = len(in_spec) if len(in_spec) != len(use_lines[gg]) and len(in_spec) > 0: use_lines[gg] = np.array(use_lines[gg])[in_spec].tolist() ignore = np.where(n_in_spec == 0)[0] if len(ignore) > 0: use_lines = np.delete(use_lines, ignore) n_lines = len(cal_lines) # Use database for lines to avoid/use in analysis | + on 21/06/2018 func0, order0, l_val0 = get_database_model(path, dataset, get_lines=True) line_flag = np.repeat(1,n_lines) for ii in range(n_lines): ii_fit = np.where(np.absolute(l_val0 - cal_lines[ii]) <= 0.1)[0] if len(ii_fit) > 0: line_flag[ii] = 0 line_nouse = np.where(line_flag == 1)[0] if dataset == 'arc': use_lines = [] for ll in range(n_lines): use_lines.append([cal_lines[ll]]) use_lines = np.array(use_lines) cen_arr = np.zeros( (n_lines,n_bins) ) if dataset != cal: pdf_file = path+'wave_cal_resid_'+dataset+'_'+cal+'.pdf' else: pdf_file = path+'wave_cal_resid_'+dataset+'.pdf' fig, ax = plt.subplots() diff_avg = np.zeros(n_lines) diff_rms = np.zeros(n_lines) diff_num = np.zeros(n_lines, dtype=np.int) for ll in range(n_lines): #mylogger.info('ll=%i, u_l_ii=%i' % (ll, u_l_ii)) ax.axvline(cal_lines[ll], color='red', linestyle='dashed', linewidth=0.25, zorder=1) z_idx = np.where(np.absolute(wave0-cal_lines[ll]) <= 10)[0] x0 = wave0[z_idx] for ii in range(n_bins): y0 = avg_arr[z_idx,ii] y0 /= max(y0) x_idx = np.where(np.absolute(x0 - cal_lines[ll]) <= 3)[0] wht_val = np.sum(x0[x_idx]*y0[x_idx])/np.sum(y0[x_idx]) if np.absolute(wht_val-cal_lines[ll]) <= 1: cen_arr[ll,ii] = wht_val #endfor good = np.where(cen_arr[ll] != 0)[0] if len(good) > 0: x_test = cen_arr[ll][good] diff = x_test - cal_lines[ll] ax.scatter(x_test, diff, marker='o', s=5, edgecolor='none', facecolor='black', alpha=0.5, zorder=2) diff_rms[ll], diff_avg[ll] = np.std(diff), np.average(diff) diff_num[ll] = len(good) #endif #endfor ax.scatter(cal_lines, diff_avg, marker='o', s=40, facecolor='blue', edgecolor='none', alpha=0.5, zorder=3) ax.errorbar(cal_lines, diff_avg, yerr=diff_rms, ecolor='blue', capsize=2.0, elinewidth=2, fmt=None, alpha=0.5, zorder=4) # Mark with 'X' lines that are not used in wavecal fitting ax.scatter(cal_lines[line_nouse], diff_avg[line_nouse], marker='X', s=100, facecolor='red', zorder=3) ax.minorticks_on() ax.set_xlabel(r'Wavelengths [$\AA$]') ax.set_ylabel(r'Difference from reference values [$\AA$]') func0, order0 = get_database_model(path, cal) an_txt = 'dataset: %s calib: %s\n' % (dataset, cal) an_txt += 'func: %s yorder: %i xorder: %i' % (func0, order0, xorder) ax.annotate(an_txt, [0.05,0.95], xycoords='axes fraction', ha='left', va='top') plt.subplots_adjust(left=0.13, right=0.99, bottom=0.1, top=0.99) fig.savefig(pdf_file) asc_file = pdf_file.replace('.pdf', '.tbl') tab0 = Table([cal_lines, diff_num, diff_avg, diff_rms], names=('Line','N','Avg','RMS')) asc.write(tab0, asc_file, format='fixed_width_two_line', overwrite=True) # Determine average, median, and rms for each column | + on 19/06/2018 avg0 = np.zeros(n_bins) N0 = np.zeros(n_bins) med0 = np.zeros(n_bins) rms0 = np.zeros(n_bins) for nn in range(n_bins): t_diff = cen_arr[:,nn] - cal_lines good = np.where((cen_arr[:,nn] != 0) & (line_flag != 1) & (np.absolute(t_diff) <= 1.5))[0] if len(good) > 0: N0[nn] = len(good) avg0[nn] = np.average(t_diff[good]) med0[nn] = np.median(t_diff[good]) rms0[nn] = np.std(t_diff[good]) pdf_file2 = pdf_file.replace('.pdf', '.stat.pdf') pp = PdfPages(pdf_file2) fig, ax = plt.subplots(ncols=3) good = np.where(rms0 != 0) N1, b1, p1 = ax[0].hist(avg0[good], bins=10, align='mid', color='b', alpha=0.5, edgecolor='none', histtype='stepfilled') N2, b2, p2 = ax[1].hist(med0[good], bins=10, align='mid', color='m', alpha=0.5, edgecolor='none', histtype='stepfilled') N3, b3, p3 = ax[2].hist(rms0[good], bins=10, align='mid', color='k', alpha=0.5, edgecolor='none', histtype='stepfilled') ax[0].set_ylabel('N') ax[0].set_xlabel(r'Average [$\AA$]') ax[1].set_xlabel(r'Median [$\AA$]') ax[2].set_xlabel(r'$\sigma$ [$\AA$]') stat_avg = [np.average(avg0[good]), np.average(med0[good]), np.average(rms0[good])] stat_med = [np.median(avg0[good]), np.median(med0[good]), np.median(rms0[good])] stat_rms = [np.std(avg0[good]), np.std(med0[good]), np.std(rms0[good])] y_max = np.max([np.max(N1),np.max(N2),np.max(N3)])*1.1 for aa in range(3): ax[aa].set_ylim([0,y_max]) ax[aa].axvline(stat_avg[aa], linestyle='solid') ax[aa].axvline(stat_med[aa], linestyle='dashed') txt0 = '' str0 = [r'$<x>$', r'$\tilde{x}$', r'$\sigma$'] val = [stat_avg[aa], stat_med[aa], stat_rms[aa]] for ss,vv in zip(str0,val): txt0 += ss+(' = %.3f' % vv) +'\n' ax[aa].annotate(txt0, [0.95,0.95], xycoords='axes fraction', ha='right', va='top') ax[1].set_yticklabels([]) ax[2].set_yticklabels([]) ax[1].tick_params(axis='y', direction='in') ax[2].tick_params(axis='y', direction='in') plt.subplots_adjust(left=0.08, right=0.99, bottom=0.12, top=0.96, wspace=0.03) fig.set_size_inches(8,4) fig.savefig(pp, format='pdf') asc_file2 = pdf_file2.replace('.pdf', '.tbl') tab2 = Table([bins_mid, N0, avg0, med0, rms0], names=('Column','N','Avg','Med','RMS')) tab2.write(asc_file2, format='ascii.fixed_width_two_line', overwrite=True) fig, ax = plt.subplots() ax.scatter(bins_mid[good], avg0[good], marker='o', color='b', alpha=0.5, edgecolor='none', label='Average') ax.scatter(bins_mid[good], med0[good], marker='o', color='m', alpha=0.5, edgecolor='none', label='Median') ax.scatter(bins_mid[good], rms0[good], marker='o', color='k', alpha=0.5, edgecolor='none', label=r'$\sigma$') ax.annotate(an_txt, [0.05,0.95], xycoords='axes fraction', ha='left', va='top') ax.legend(loc='lower right', fancybox=True) #frameon=False) ax.set_ylabel(r'Average / Median [$\AA$]') ax.set_xlabel(r'X (Along longslit) [pixels]') ax.minorticks_on() fig.subplots_adjust(left=0.11, right=0.99, bottom=0.12, top=0.96) fig.set_size_inches(8,4) fig.savefig(pp, format='pdf') pp.close() if silent == False: mylogger.info('### End residual_wave_cal : '+systime())
def get_database_model(path, source, get_lines=False, silent=False, verbose=True): ''' Determine fitting function and order to pass to nsfitcoords Parameters ---------- path : str Full path to where output PDF and FITS file are located. Must end with a '/' source : str Either 'arc' or 'OH' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Notes ----- Created by Chun Ly, 19 June 2018 Modified by Chun Ly, 21 May 2017 - Add get_lines keyword to get arc/OH reference lines Modified by Chun Ly, 22 May 2017 - Handle INDEF case - Bug fix: get lines, not index ''' logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin get_database_model : '+systime()) ref_file = source+'_stack.fits' if source == 'arc': dbase = 'database' if source == 'OH': dbase = 'database_OH' d_file = '%s%s/idw%s' % (path, dbase, ref_file.replace('.fits','_SCI_1_')) mylogger.info("Reading : "+d_file) f0 = open(d_file, 'r') str0 = f0.readlines() ii_beg = [ii for ii in xrange(len(str0)) if 'begin' in str0[ii]] ii_feat = [ii for ii in xrange(len(str0)) if 'features' in str0[ii]] ii_func = [ii for ii in xrange(len(str0)) if 'function' in str0[ii]] ii_ord = [ii for ii in xrange(len(str0)) if 'order' in str0[ii]] func0 = str0[ii_func[0]].split(' ')[-1].replace('\n','') order0 = np.int(str0[ii_ord[0]].split(' ')[-1].replace('\n','')) if get_lines: # + on 21/06/2018 n_features = np.int(str0[ii_feat[0]].split('\t')[2].replace('\n','')) l_val0 = np.zeros(n_features) for ff in range(n_features): temp1 = str0[ii_feat[0]+1+ff].replace(' ','') temp1 = temp1.split(' ') temp1 = [val for val in temp1 if val != ''] if temp1[3] != 'INDEF': # Mod on 22/06/2018 l_val0[ff] = np.float(temp1[3]) l_val0 = l_val0[l_val0.nonzero()] # + on 22/06/2018 mylogger.info('Function for '+source+' line fitting : '+func0) mylogger.info('Order of '+source+' line fitting : %i' % order0) f0.close() if silent == False: mylogger.info('### End get_database_model : '+systime()) # Mod on 21/06/2018 if not get_lines: return func0, order0 else: return func0, order0, l_val0
def arc_check2(path, arcs=[''], out_pdf='', stack=False, cross_check=False, silent=False, verbose=True): ''' Generate plot illustrating expected location of arc lines to check wavelength calibration Parameters ---------- path : str Full path to where output PDF and FITS file are located. Must end with a '/' arcs : str or list (Optional) List of raw filenames for the arc data (e.g., 'N20170101S0111.fits') stack : boolean Indicate whether to include stacked arc data. Default: False cross_check : boolean Check OH-based wavelength calibration against arc lines. Default: False silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 16 June 2017 - Started as a copy of OH_check Modified by Chun Ly, 17 July 2017 - Fix bug with arcs handling Modified by Chun Ly, 12 November 2017 - Added stack keyword option to include stacked arc products Modified by Chun Ly, 16 November 2017 - Change prefix: tfrnc to tfrbnc Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 31 May 2018 - Add cross_check keyword; Update input/outputs for cross_check == True ''' # + on 09/01/2018 logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin arc_check2 : '+systime()) arc_file = co_dirname+'/argon.dat' if exists(arc_file): if silent == False: mylogger.info('Reading : '+arc_file) arc_data = asc.read(arc_file, format='commented_header') arc_lines = arc_data['Line'].data arc_source = arc_data['Source'].data if arcs[0] == '': arcs_list = path + 'arc.lis' if silent == False: mylogger.info('Reading : '+arcs_list) arcs = np.loadtxt(arcs_list, dtype=type(str)) tfrnc_files = [path+'tfrbnc'+file0 for file0 in arcs] # Mod on 16/11/2017 # + on 12/11/2017 if stack == True: arcs0 = [path + 'arc_stack.fits'] tfrnc_files0 = [path + 'tfarc_stack.fits'] arcs = arcs0 + arcs tfrnc_files = tfrnc_files0 + tfrnc_files # + on 31/05/2018 if cross_check == True: arcs = [path + 'arc_stack_OH.fits'] tfrnc_files = [path + 'tfarc_stack_OH.fits'] chk = [file0 for file0 in tfrnc_files if exists(file0) == True] if len(chk) == 0: mylogger.warn('Files not found!!!') mylogger.warn(', '.join(file0)) mylogger.warn('Exiting!!!') return n_arcs = len(arcs) # Moved lower on 12/11/2017 # Mod on 31/05/2018 if cross_check == False: out_pdf = path+'arc_check2.pdf' if out_pdf == '' else path+out_pdf else: out_pdf = path+'arc_check_OH.pdf' if out_pdf == '' else path+out_pdf pp = PdfPages(out_pdf) for nn in xrange(n_arcs): if exists(tfrnc_files[nn]): # Mod on 20/05/2017 hdu0 = fits.open(tfrnc_files[nn]) im0 = hdu0['SCI'].data gc0 = aplpy.FITSFigure(hdu0, hdu='SCI', figsize=(7.65,10.5)) z1, z2 = zscale.get_limits(im0) gc0.show_grayscale(invert=True, vmin=z1, vmax=z2) gc0.set_tick_color('black') # + on 20/05/2017 hdr0 = hdu0['SCI'].header crval2 = hdr0['CRVAL2'] cdelt2 = hdr0['CDELT2'] npix = hdr0['NAXIS2'] lamb_max = crval2 + cdelt2*npix arc_mark = np.where((arc_lines >= crval2) & (arc_lines <= lamb_max))[0] line_list = [] for ll in arc_mark: line_list.append(np.array([[0,700], [arc_lines[ll],arc_lines[ll]]])) gc0.show_lines(line_list, color='red', alpha=0.5, linewidth=1.0, linestyle='dashed') str0 = tfrnc_files[nn].replace(path,'') gc0.add_label(0.025, 0.975, str0, color='red', relative=True, ha='left', va='top', weight='medium', fontsize=14, bbox=bbox_props) gc0.set_axis_labels(xlabel='X [pixels]', ylabel=r'Wavelength ($\AA$)') gc0.savefig(pp, format='pdf') #endif #endfor if silent == False: mylogger.info('Writing : '+out_pdf) pp.close() if silent == False: mylogger.info('### End arc_check2 : '+systime())
def main(rawdir, silent=False, verbose=True): ''' Main function for get_OH_centers Parameters ---------- silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 12 June 2018 Modified by Chun Ly, 13 June 2018 - Fix rev_lines and rev_int (wrong indexing) - Switch to micron units; Do subplots_adjust for white space - Plot aesthetics: label plots, change page size, legend - Include parameters/assumptions in title - Plot aesthetics: Vertical lines for all possible OH skylines - Plot aesthetics: Label OH skylines - Plot aesthetics: Limit vertical lines for OH skylines - Plot aesthetics: group and label OH skylines to avoid overlap - Group lines (<5 Ang) before annotation - Opaque white background behind lines - Plot aesthetics: Draw OH lines to top of subplots - Plot aesthetics: Remove legend; adjust white space - WARN if more than six lines - Attempt to constraint fit using bounds but that did not work - Fix case if curve_fit solutions are outside of spectral range Modified by Chun Ly, 21 June 2018 - Write npz file containing use lines that are grouped together - mylogger call for writing files - Switching back to uncompressed npz - Got interpret file pickle error (IOError) Modified by Chun Ly, 25 June 2018 - Bug fix: Crash on call to group_OH_lines. Require in_zoom to not be empty array Modified by Chun Ly, 25 June 2018 - Bug fix: Handle odd fit results (check wavelength within 5 Ang of guess) - Bug fix: Add try/except for curve_fit RuntimeError - Bug fix: tab fix - Bug fix: Add try/except for curve_fit RuntimeError for gauss_multi - Bug fix: list to np.array ''' # + on 09/01/2018 logfile = rawdir + 'get_OH_centers.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) if exists(OH_file): if silent == False: mylogger.info('Reading : ' + OH_file) OH_data = np.loadtxt(OH_file) OH_lines = OH_data[:, 0] OH_int = OH_data[:, 1] infile = rawdir + 'hdr_info.QA.tbl' tab0 = asc.read(infile, format='fixed_width_two_line') i_obj = [xx for xx in range(len(tab0)) if tab0['QA'][xx] == 'obj'][0] o_tab0 = tab0[i_obj] gratwave = o_tab0['gratwave'] slitwidth = np.float(o_tab0['slit'].split('arcsec')[0]) if '111/mm' in o_tab0['grating']: if o_tab0['filter2'] == 'X_G0518': R_spec = 6600.0 / (slitwidth / 0.3) x_diff = 0.06 if o_tab0['filter2'] == 'J_G0517': R_spec = 7200.0 / (slitwidth / 0.3) x_diff = 0.07 x_min = (gratwave - x_diff) * 1.E4 x_max = (gratwave + x_diff) * 1.E4 npix = np.round((x_max - x_min) / 0.15) x0 = x_min + 0.15 * np.arange(npix) OH_spec_mod = np.zeros(len(x0)) in_rge = np.where((OH_lines >= x0[0]) & (OH_lines <= x0[-1]))[0] for ii in range(len(in_rge)): idx = in_rge[ii] temp = OH_int[idx] * gaussian_R(x0, OH_lines[idx], R_spec) OH_spec_mod += temp y_max = max(OH_spec_mod) * 1.25 i_lines = np.where(OH_spec_mod >= np.max(OH_spec_mod) * 0.01)[0] lines_set = list(group(i_lines)) OH_spec_mod_resid = OH_spec_mod.copy() rev_lines = [] #np.zeros(len(lines_set)) rev_int = [] #np.zeros(len(lines_set)) nrows = 3 fig, ax = plt.subplots(nrows=nrows) use_lines = [] # + on 21/06/2018 xlim_arr = [] dx = (x0[-1] - x0[0]) / 3.0 for aa in range(nrows): ax[aa].plot(x0 / 1e4, OH_spec_mod, color='black', zorder=3, label="Rousselot (2000)") xlim = np.array([x_min + dx * aa, x_min + dx * (aa + 1)]) xlim_arr.append(xlim) ax[aa].set_xlim(xlim / 1e4) ax[aa].set_ylim([-10, y_max]) # Draw vertical lines for all possible OH skylines for val in OH_lines[in_rge]: ax[aa].axvline(x=val / 1e4, color='black', linewidth=0.25, linestyle=':', zorder=2) for ii in range(len(lines_set)): x_avg = np.int(np.average(lines_set[ii])) tl_min, tl_max = x0[lines_set[ii][0]], x0[lines_set[ii][1]] in_zoom = np.where((OH_lines >= tl_min) & (OH_lines <= tl_max))[0] if len(in_zoom) > 0: group_lines, group_int = group_OH_lines(OH_lines[in_zoom], OH_int[in_zoom]) # print ii, group_lines sig = group_lines / R_spec / (2 * np.sqrt(2 * np.log(2))) peak0 = OH_spec_mod[np.int_( (group_lines - x_min) / (x0[1] - x0[0]))] if len(group_lines) == 1: p0 = [0.0, peak0, group_lines[0], sig[0]] #plt.axvline(group_lines[0]/1e4, color='blue') else: if len(group_lines) > n_multi: log.warn('More than 8 lines found, N=' + str(len(group_lines)) + '!!!') t_peak0 = peak0.tolist() t_lines = group_lines.tolist() t_sig = sig.tolist() t_peak0 += np.zeros(n_multi - len(group_lines)).tolist() t_lines += np.zeros(n_multi - len(group_lines)).tolist() t_sig += np.zeros(n_multi - len(group_lines)).tolist() p0 = t_peak0 p0 += t_lines p0 += t_sig p0 = np.array(p0) zoom = np.arange(lines_set[ii][0], lines_set[ii][1]) # print p0 #bounds = ((-0.001, 0.0, lam_cen-0.5, 0.1), # (0.001, 1.25*p0[1], lam_cen+0.5, 1.5*p0[3])) if len(group_lines) == 1: try: popt, pcov = curve_fit(gauss1d, x0[zoom], OH_spec_mod[zoom], p0=p0) except RuntimeError: mylogger.warn('Did not converge!') mylogger.warn('Using initial guess : %.3f' % group_lines[0]) popt = list(p0) if np.absolute(popt[2] - p0[2]) >= 5: mylogger.warn('Reliable fit not determined!') mylogger.warn('Using initial guess : %.3f' % group_lines[0]) popt = list(p0) t_mod = gauss1d(x0, *popt) use_lines.append([popt[2]]) # + on 21/06/2018 rev_lines.append(popt[2]) rev_int.append(popt[1]) i_ax = [ xx for xx in range(nrows) if (popt[2] >= xlim_arr[xx][0] and popt[2] <= xlim_arr[xx][1]) ][0] ax[i_ax].annotate('%.2f' % popt[2], [popt[2] / 1e4, y_max * 0.99], ha='center', va='top', rotation=90, fontsize=4, bbox=bbox_props) else: #low_bound = tuple([0] * n_multi) + tuple(p0[n_multi:2*n_multi]-0.5) + \ # tuple([0] * n_multi) #up_bound = tuple(p0[0:n_multi]*1.25+0.1) + tuple(p0[n_multi:2*n_multi]+0.5) + \ # tuple(p0[2*n_multi:]+1) try: popt, pcov = curve_fit(gauss_multi, x0[zoom], OH_spec_mod[zoom], p0=p0) except RuntimeError: mylogger.warn('Did not converge!') mylogger.warn('Using initial guesses') popt = np.array(p0) t_mod = gauss_multi(x0, *popt) t_loc = popt[n_multi:2 * n_multi] # Line wavelengths (Ang) t_str = popt[0:n_multi] # Line peak strength nonzero = np.where(t_loc != 0)[0] use_lines.append(t_loc[nonzero].tolist()) # + on 21/06/2018 wave0 = t_loc[nonzero] wave0.sort() # Check that lines are within range in_rge = np.where((wave0 >= x_min) & (wave0 <= x_max))[0] wave0 = wave0[in_rge] t_str = t_str[nonzero[in_rge]] rev_lines += wave0.tolist() rev_int += t_str.tolist() wave0 = np.array(wave0) # Label lines skip = np.zeros(len(wave0)) for ww in range(len(wave0)): if skip[ww] == 0: w_diff = wave0[ww:] - wave0[ww] t_close = np.where(w_diff <= 5)[0] close = np.arange(ww, len(wave0))[t_close] str_comb = "\n".join( ['%.2f' % val for val in wave0[close]]) w_cen = np.average(wave0[close]) #0.5*(wave0[0]+wave0[-1]) #np.average(wave0) i_ax = [ xx for xx in range(nrows) if (w_cen >= xlim_arr[xx][0] and w_cen <= xlim_arr[xx][1]) ][0] ax[i_ax].annotate(str_comb, [w_cen / 1e4, y_max * 0.99], ha='center', va='top', rotation=90, fontsize=4, bbox=bbox_props) skip[close] = 1 #endif #endfor # print '## t_mod : ', np.min(t_mod), np.max(t_mod) OH_spec_mod_resid -= t_mod for aa in range(nrows): ax[aa].plot(x0 / 1e4, OH_spec_mod_resid, linestyle='dashed', color='blue', zorder=3, label='Residual') for t_line in rev_lines: ax[aa].axvline(x=t_line / 1e4, color='red', linestyle='--', linewidth=1.0, zorder=1) l_tab = Table([rev_lines, rev_int]) out_file = rawdir + 'rousselot2000_convl.dat' mylogger.info('Writing : ' + out_file) asc.write(l_tab, out_file, format='no_header') ann_txt = r'%.2f $\mu$m; %s; ' % (gratwave, o_tab0['filter2']) ann_txt += '%s; %.3f" slit; R = %i' % (o_tab0['grating'], slitwidth, R_spec) ax[0].set_title(ann_txt) #leg = ax[0].legend(loc='upper right', fancybox=True) #, frameon=False) #leg.get_frame().set_alpha(0.75) ax[2].set_xlabel(r'Wavelength [$\mu$m]') plt.subplots_adjust(left=0.08, right=0.95, bottom=0.06, top=0.96, hspace=0.12) fig.set_size_inches(6, 8) out_pdf = out_file.replace('.dat', '.pdf') mylogger.info('Writing : ' + out_pdf) fig.savefig(out_pdf) # Write npz file containing final grouping | + on 21/06/2018 savez_file = out_file.replace('.dat', '.npz') mylogger.info('Writing : ' + savez_file) np.savez(savez_file, use_lines=use_lines) if silent == False: mylogger.info('### End main : ' + systime())
def run(path0, clean_file='', out_script='', silent=False, verbose=True, overwrite=False): ''' Create a .sh script to run cleanir.py for a set of files Parameters ---------- silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True overwrite : boolean Overwrite files if they exists. Default: False Returns ------- Notes ----- Created by Chun Ly, 7 March 2017 Modified by Chun Ly, 15 May 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 30 May 2017 - Added overwrite option. Default is to not overwrite .sh files - Fix bug when only one file is found Modified by Chun Ly, 18 June 2017 - Fix to work with no date directory Modified by Chun Ly, 22 January 2018 - Import glog and call for stdout and ASCII logging - Pass mylogger to dir_check.main() Modified by Chun Ly, 20 April 2018 - Change afq to rfq - row filtering generally produces better results ''' logfile = path0+'cleanir_script.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin run : '+systime()) if clean_file == '': clean_file = 'clean.lis' # + on 15/05/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) # Mod on 15/05/2017 for date,path in zip(dir_list,list_path): clean_file0 = path+clean_file if not exists(clean_file0): mylogger.warn('File does not exist!!!') mylogger.warn(clean_file0) else: if silent == False: mylogger.info('Reading : '+clean_file0) files = np.loadtxt(clean_file0, dtype=type(str)).tolist() if type(files) == str: files = [files] # Bug fix. Mod on 30/05/2017 out_script0 = path+'run_cleanir.sh' if out_script == '' \ else out_script if date != '': # Mod on 18/06/2017 out_script0 = out_script0.replace('.sh', '.'+date+'.sh') # Mod on 30/05/2017 if overwrite == False and exists(out_script0): log.warn('## File found!!! : '+out_script0) log.warn('## Will not overwrite!!!') else: if silent == False: stat0 = 'Overwriting' if exists(out_script0) else 'Writing' mylogger.info(stat0+' : '+out_script0) f = open(out_script0, 'w') for ii in xrange(len(files)): cmd1 = cmd0+' -rfqo '+path+'c'+files[ii]+' '+path+files[ii] f.write(cmd1+'\n') f.close() #endelse #endelse #endfor if silent == False: mylogger.info('End run : '+systime())
def main(rawdir, out_pdf='', gz=False, silent=False, verbose=True): ''' Main function to compute and plot statistics Parameters ---------- rawdir : str Path to raw files. gz : boolean Indicate using gz-compressed files (mostly for debugging). Default: False silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 8 May 2018 - Change output PDF filename - Plot averages of statistic measurements - Plot expected Poissonian level - Plot aesthetics (legend, limits, x/ylabel) - Compute and plot rms of stacked image Modified by Chun Ly, 21 May 2018 - Bug fix: Mistake in computing RMS for combined 2-D data - Write RMS to mylogger; Label 'Stack' RMS on right side of plot - Add gz keyword option to use with compressed files Modified by Chun Ly, 28 May 2018 - Add '/' after rawdir if not provided ''' if rawdir[-1] != '/': rawdir = rawdir + '/' logfile = rawdir + 'image_stats.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin run ! ') dir_list, list_path = dir_check.main(rawdir, mylogger=mylogger, silent=silent, verbose=verbose) out_pdf_default = out_pdf for path in list_path: file_lis = path + 'obj.lis' if not exists(file_lis): if silent == False: mylogger.info('File not found : ' + file_lis) else: if silent == False: mylogger.info('Reading : ' + file_lis) t_files = np.loadtxt(file_lis, dtype=type(str)).tolist() files0 = [path + 'tfrbnc' + t_file for t_file in t_files] if gz == True: # + on 21/05/2018 files0 = [t_file + '.gz' for t_file in files0] n_files0 = len(files0) avg_arr = np.zeros(n_files0) med_arr = np.zeros(n_files0) sig_arr = np.zeros(n_files0) for nn in range(n_files0): im0 = fits.getdata(files0[nn], extname='SCI') c_mean, c_med, c_sig = sigma_clipped_stats(im0, sigma=2.0, iters=10) avg_arr[nn] = c_mean med_arr[nn] = c_med sig_arr[nn] = c_sig #endfor #print avg_arr, med_arr, sig_arr if out_pdf == '': out_pdf = path + 'image_stats.pdf' else: out_pdf = path + out_pdf fig, ax_arr = plt.subplots(nrows=2) num0 = 1 + np.arange(n_files0) ax_arr[0].scatter(num0, avg_arr, marker='o', s=50, facecolor='none', edgecolor='k', label='Mean') avg_avg = np.average(avg_arr) ax_arr[0].axhline(y=avg_avg, c='k', linestyle='dashed') ax_arr[0].scatter(num0, med_arr, marker='x', s=25, color='b', label='Median') avg_med = np.average(med_arr) ax_arr[0].axhline(y=avg_med, c='b', linestyle='dotted') ax_arr[0].legend(loc='lower right', fontsize=8) ax_arr[1].scatter(num0, sig_arr, marker='o', s=50, facecolor='none', edgecolor='k') avg_sig = np.average(sig_arr) ax_arr[1].axhline(y=avg_sig, c='k', linestyle='dashed') # Expected Poissionan level sig_poisson = avg_sig / np.sqrt(n_files0) mylogger.info('Expected (Poisson) RMS : %.4f' % sig_poisson) # + on 21/05/2018 ax_arr[1].axhline(y=sig_poisson, c='b', linestyle='dashed') ax_arr[1].text(0, sig_poisson, 'Poisson', color='b', ha='left', va='bottom') ax_arr[0].minorticks_on() ax_arr[1].minorticks_on() ax_arr[0].set_xticklabels([]) ax_arr[0].set_xlim([-0.25, n_files0 + 1]) ax_arr[1].set_xlim([-0.25, n_files0 + 1]) ax_arr[1].set_ylabel(r'$\sigma$') ax_arr[1].set_xlabel('Frame No.') # Plot rms of stacked image comb_file = glob.glob(path + 'obj_comb.fits') if len(comb_file) != 0: mylogger.info('Reading : ' + comb_file[0]) comb_data = fits.getdata(comb_file[0], extname='SCI') # Mod on 21/05/2018 # Mod on 21/05/2018 c_mean, c_med, c_sig = sigma_clipped_stats(comb_data, sigma=2.0, iters=10) mylogger.info('Measured RMS in stack : %.4f' % c_sig) # + on 21/05/2018 ax_arr[1].axhline(y=c_sig, c='g', linestyle='dashed') ax_arr[1].text(n_files0 + 1, c_sig, 'Stack', color='g', ha='right', va='bottom') # Mod on 21/05/2018 #fig, ax_arr = plt.subplots(ncols=2) #ax_arr[0].hist(avg_arr, bins=5, align='mid', color='b', # linestyle='solid', alpha=0.5, edgecolor='b', # histtype='step', label='Average') #ax_arr[0].hist(med_arr, bins=5, align='mid', # color='g', # linestyle='dashed', edgecolor='k', # histtype='step', label='Median') #ax_arr[1].hist(sig_arr, bins=5, align='mid', color='k', # linestyle='solid', alpha=0.5, edgecolor='k', # histtype='step') mylogger.info('Writing : ' + out_pdf) fig.set_size_inches(8, 4) fig.savefig(out_pdf, bbox_inches='tight') plt.close() out_pdf = out_pdf_default #endfor if silent == False: mylogger.info('### End run ! ')
def run(rawdir, silent=False, verbose=False): ''' Main function to combine science data to produce a 2-D FITS image containing OH night skylines for wavelength calibration. Median filtering is used to remove detected objects Parameters ---------- rawdir : str Path to raw files. Must end in a '/' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- 2-D image containing OH skyline called 'OH_stack.fits' Notes ----- Created by Chun Ly, 25 June 2017 Modified by Chun Ly, 20 September 2017 - Call check_path() Modified by Chun Ly, 16 November 2017 - Change prefix: rnc to rbnc Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging ''' # + on 09/01/2018 logfile = rawdir+'OH_stack.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin run : '+systime()) rawdir = check_path(rawdir) # + on 20/09/2017 obj_list = rawdir + 'obj.lis' if silent == False: mylogger.info('Reading : '+obj_list) objs = np.loadtxt(obj_list, dtype=type(str)) rbnc_files = [rawdir+'rbnc'+file0.replace('.fits','.OH.fits') for file0 in objs] # Mod on 16/11/2017 # Mod on 16/11/2017 hdu0 = fits.open(rbnc_files[0]) hdr = fits.getheader(rbnc_files[0], extname='SCI') naxis1 = hdr['NAXIS1'] naxis2 = hdr['NAXIS2'] arr0 = np.zeros((len(rbnc_files), naxis2, naxis1)) # Mod on 16/11/2017 for ii in range(len(rbnc_files)): if verbose == True: mylogger.info('Reading : '+obj_list) t_data = fits.getdata(rbnc_files[ii], extname='SCI') # Mod on 16/11/2017 t_med0 = np.median(t_data, axis=0) # Median along columns # Remove median along columns med_off = np.repeat(t_med0, naxis2).reshape(naxis1,naxis2).transpose() arr0[ii] = t_data - med_off # Compute median med_arr0 = np.median(arr0, axis=0) hdu0['SCI'].data = med_arr0 hdu0['VAR'].data = np.zeros((naxis2,naxis1)) outfile = rawdir+'OH_stack.fits' stat0 = 'Overwriting' if exists(outfile) else 'Writing' if silent == False: mylogger.info(stat0+' : '+outfile) hdu0.writeto(outfile, output_verify='ignore', overwrite=True) if silent == False: mylogger.info('### End run : '+systime())
def plot_spec(rawdir, out_pdf='', silent=False, verbose=False): ''' Plot median spectrum of OH skylines Parameters ---------- rawdir : str Path to raw files. Must end in a '/' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Produces 'OH_spec.pdf' Notes ----- Created by Chun Ly, 4 July 2017 - Overlay Rousselot (2000) model spectrum Modified by Chun Ly, 20 September 2017 - Call check_path() Modified by Chun Ly, 9 January 2018 - Implement glog stdout and ASCII logging ''' # + on 09/01/2018 logfile = rawdir+'OH_stack.log' mylogger = glog.log0(logfile)._get_logger() rawdir = check_path(rawdir) # + on 20/09/2017 out_pdf = rawdir+'OH_spec.pdf' if out_pdf == '' else rawdir+out_pdf im0, hdr = fits.getdata(rawdir+'tfOH_stack.fits', extname='SCI', header=True) OH_med0 = np.median(im0, axis=1) crval2 = hdr['CRVAL2'] cd2_2 = hdr['CD2_2'] crpix = hdr['CRPIX2'] x0 = crval2 + cd2_2*np.arange(len(OH_med0)) x0 /= 1e4 # In microns y0 = OH_med0/max(OH_med0) fig, ax = plt.subplots() if exists(OH_file): if silent == False: mylogger.info('Reading : '+OH_file) OH_data = np.loadtxt(OH_file) OH_lines = OH_data[:,0] / 1E4 OH_int = OH_data[:,1] i_max = np.where(OH_med0 == max(OH_med0))[0] l_max = x0[i_max[0]] p0 = [0.0, max(y0), l_max, 2.0/1e4] popt, pcov = curve_fit(gauss1d, x0, y0, p0=p0) fwhm = popt[3]*2*np.sqrt(2*np.log(2)) R_spec = l_max / fwhm OH_spec_mod = np.zeros(len(x0)) in_rge = np.where((OH_lines >= x0[0]) & (OH_lines <= x0[-1]))[0] for ii in range(len(in_rge)): idx = in_rge[ii] #ax.plot(np.repeat(OH_lines[in_rge[ii]],2), [0,1.1], 'r--') temp = OH_int[idx] * gaussian_R(x0, OH_lines[idx], R_spec) OH_spec_mod += temp ax.plot(x0, y0, 'k-', label='OH_stack') OH_spec_mod /= np.max(OH_spec_mod) ax.plot(x0, OH_spec_mod, 'r--', alpha=0.75, label='OH model (Rousselot 2000)') ax.set_xlabel(r'Wavelength ($\mu$m)', fontsize=16) ax.set_ylabel('Normalized Flux', fontsize=16) ax.set_ylim([0.0, 1.10]) ax.minorticks_on() ax.tick_params(labelsize=14) subplots_adjust(left=0.07, right=0.99, bottom=0.07, top=0.99) ax.legend(loc='upper right', fontsize=14, frameon=False) fig.set_size_inches(11,8) fig.savefig(out_pdf)
def transform(rawdir, silent=False, verbose=False): ''' Transform OH_stack 2-D FITS image for wavelength calibration checks Parameters ---------- rawdir : str Path to raw files. Must end in a '/' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- 2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits' Notes ----- Created by Chun Ly, 4 July 2017 Modified by Chun Ly, 20 September 2017 - Call check_path() Modified by Chun Ly, 22 November 2017 - Add file checking and log.warn calls Modified by Chun Ly, 29 November 2017 - Bug fix: Missing else statement Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 19 June 2018 - Call QA_wave_cal.get_database_model - Pass function and order to nsfitcoords for OH stack Modified by Chun Ly, 20 June 2018 - Set nsfitcoords xorder fitting Modified by Chun Ly, 22 June 2018 - Call residual_wave_cal for wavelength solution check ''' # + on 09/01/2018 logfile = rawdir+'OH_stack.log' mylogger = glog.log0(logfile)._get_logger() cdir = os.getcwd()+'/' # + on 06/05/2017 rawdir = check_path(rawdir) # + on 20/09/2017 iraf.chdir(rawdir) mylogger.info("Running nsfitcoords on OH_stack") # Mod on 09/01/2018 outfile1 = rawdir + 'fOH_stack.fits' if not exists(outfile1): func0, order0 = QA_wave_cal.get_database_model(rawdir, 'OH') iraf.gnirs.nsfitcoords('wOH_stack.fits', outprefix='', outspectra='fOH_stack.fits', lamp='wOH_stack.fits', database='database_OH/', function=func0, lyorder=order0, lxorder=QA_wave_cal.xorder) else: # Mod on 09/01/2018 mylogger.warn('File exists!!! : '+outfile1) mylogger.warn('Will not run nsfitcoords on OH stacked data') outfile2 = rawdir + 'tfOH_stack.fits' if not exists(outfile2): iraf.gnirs.nstransform('fOH_stack.fits', outprefix='', outspectra='tfOH_stack.fits', database='database_OH/') else: # Mod on 09/01/2018 mylogger.warn('File exists!!! : '+outfile2) mylogger.warn('Will not run nstransform on OH stacked data') iraf.chdir(cdir) # + on 22/06/2018 QA_wave_cal.residual_wave_cal(rawdir, dataset='OH', cal='OH')
def wave_cal(rawdir, cdir, silent=False, verbose=False): ''' Run gnirs.nswavelength on OH_stack 2-D FITS image for wavelength calibration Parameters ---------- rawdir : str Path to raw files. Must end in a '/' silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- 2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits' Notes ----- Created by Chun Ly, 13 July 2017 Modified by Chun Ly, 20 September 2017 - Call check_path() Modified by Chun Ly, 16 November 2017 - Call wave_cal_script to get PyRAF code Modified by Chun Ly, 16 November 2017 - Bug fix: indentation typo with else statement Modified by Chun Ly, 20 November 2017 - Bug fix: Pass in cdir Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 14 June 2018 - Import and call get_OH_centers Modified by Chun Ly, 19 June 2018 - Pass gnirs logfile to mylogger ''' # + on 09/01/2018 logfile = rawdir+'OH_stack.log' mylogger = glog.log0(logfile)._get_logger() rawdir = check_path(rawdir) # + on 20/09/2017 iraf.chdir(rawdir) timestamp = systime().replace(':','.') logfile = rawdir+'gnirs_'+timestamp+'.log' iraf.gemini.gnirs.logfile = logfile mylogger.info("GNIRS logfile : "+logfile) get_OH_centers.main(rawdir) # + on 16/11/2017 script_file = 'wave_cal_OH.py' if not exists(script_file): wave_cal_script.main(rawdir, line_source='OH') else: # Mod on 09/01/2018 mylogger.info('File exists!!! : '+script_file) mylogger.info('Will not override!!!') # + on 16/11/2017 do_run = 0 if not exists('wOH_stack.fits'): do_run = 1 if do_run: # Mod on 09/01/2018 mylogger.info("In order to perform interactive calibration, open up") mylogger.info("a PyRAF terminal in an anaconda IRAF environment") mylogger.info("'cd' into "+rawdir) mylogger.info("Execute the following command :") mylogger.info("execfile('"+script_file+"')") t_out = raw_input("## Hit RETURN when OH wavelength calibration is completed") else: # Mod on 09/01/2018 mylogger.warn('Files exist!!!') mylogger.warn('Will not run nswavelength on OH stacked data') iraf.chdir(cdir)
def main(path0, silent=False, verbose=True, overwrite=False): ''' main() function to obtain information from FITS header and write to ASCII file Parameters ---------- path0 : str Path to FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- tab0 : astropy.table.Table Astropy ASCII table written to [path0]+'hdr_info.tbl' Notes ----- Created by Chun Ly, 4 March 2017 - Later re-organized to check for file first Modified by Chun Ly, 5 March 2017 - File exists warning always printed out - Include AIRMASS Modified by Chun Ly, 23 March 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 11 May 2017 - Handle longer filter1 and filter2 FITS values Modified by Chun Ly, 8 December 2017 - Import glog and call for stdout and ASCII logging - Pass mylogger to dir_check.main() ''' # Moved up on 10/12/2017 logfile = path0 + 'hdr_info.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) # Mod on 23/03/2017 for path in list_path: outfile = path + 'hdr_info.tbl' # Mod later on 04/03/2017 to not overwrite file # Mod on 05/03/2017 to always print out this warning if overwrite == False and exists(outfile): # Mod on 08/12/2017 mylogger.warning('File exists : ' + outfile) mylogger.warning('Not over-writing!!! ') else: fits_files = glob.glob(path + 'N*fits') n_files = len(fits_files) # Mod on 05/03/2017 to include airmass names0 = ('filename', 'datelabel', 'UT_date', 'obstype', 'object', 'exptime', 'airmass', 'grating', 'gratwave', 'filter1', 'filter2', 'slit') dtype0 = ('S20', 'S30', 'S25', 'S8', 'S100', 'f8', 'f8', 'S15', 'f8', 'S20', 'S20', 'S20') tab0 = Table(names=names0, dtype=dtype0) for nn in xrange(n_files): basename = os.path.basename(fits_files[nn]) if silent == False: mylogger.info('Reading : ' + basename) h0 = fits.getheader(fits_files[nn]) # Mod on 05/03/2017 to include airmass vec0 = [ basename, h0['DATALAB'], h0['DATE-OBS'] + 'T' + h0['UT'], h0['OBSTYPE'], h0['OBJECT'], h0['EXPTIME'], h0['AIRMASS'], h0['GRATING'], h0['GRATWAVE'], h0['FILTER1'], h0['FILTER2'], h0['SLIT'] ] tab0.add_row(vec0) if silent == False: mylogger.info('Writing : ' + outfile) # Mod on 08/12/2017 asc.write(tab0, outfile, format='fixed_width_two_line') #endelse #endfor if silent == False: mylogger.info('### End main : ' + systime())
def main(file_list, path0='', out_pdf='', silent=False, verbose=True, overwrite=False): ''' main() function to read in each FITS image and display it on a zscale using aplpy Parameters ---------- file_list : str Filename of ASCII file containing images to be plotted. Use 'all.lis' from create_list() since this will include all that will be processed Do NOT include full path path0 : str Full path to where output PDF and FITS file are located. Must end with a '/' out_pdf : str Filename for output PDF. Do NOT include full path silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True overwrite : boolean Overwrite files if they exists. Default: False Returns ------- multi-page PDF plot Notes ----- Created by Chun Ly, 6 March 2017 Modified by Chun Ly, 23 March 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 1 June 2017 - Added overwrite keyword option to overwrite file. Default is not to overwrite .pdf files Modified by Chun Ly, 8 December 2017 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 17 December 2017 - Change glog logging to common path - Add begin and end QA_clean logging to glog logfile - Pass mylogger to dir_check.main() ''' # Moved up on 17/12/2017 logfile = path0 + 'QA_plot.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) # + on 23/03/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) out_pdf_default = out_pdf # Mod on 23/03/2017 for path in list_path: if silent == False: mylogger.info('Reading : ' + path + file_list) # Mod on 08/12/2017 files = np.loadtxt(path + file_list, dtype=type(str)).tolist() n_files = len(files) if out_pdf == '': out_pdf = path + 'QA_plot.pdf' else: out_pdf = path + out_pdf hdr_info_file = path + 'hdr_info.tbl' if exists(hdr_info_file): if silent == False: mylogger.info('Reading : ' + hdr_info_file) # Mod on 08/12/2017 tab0 = asc.read(hdr_info_file, format='fixed_width_two_line') idx1, idx2 = match_nosort_str(files, tab0['filename']) tab0 = tab0[idx2] else: if silent == False: mylogger.warn('File not found : ' + hdr_info_file) # Mod on 08/12/2017 if overwrite == False and exists(out_pdf): mylogger.warn('File exists!! Will not overwrite ' + out_pdf) # Mod on 08/12/2017 else: pp = PdfPages(out_pdf) for nn in xrange(n_files): if silent == False: mylogger.info('Reading : ' + files[nn]) # Mod on 08/12/2017 # h0 = fits.getheader(path+files[nn], 0) # im0 = fits.getdata(path+files[nn], 1) hdu0 = fits.open(path + files[nn]) im0 = hdu0[1].data hdu0[1].header = hdu0[0].header # Copy WCS header over gc = aplpy.FITSFigure(hdu0, hdu=1, figsize=(8, 8)) z1, z2 = zscale.get_limits(im0) gc.show_grayscale(invert=True, vmin=z1, vmax=z2) gc.set_tick_color('black') gc.set_tick_yspacing('auto') gc.ticks.set_yspacing(1 / 60.0) # Every 1 arcmin in Dec gc.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm') txt0 = files[nn] if exists(hdr_info_file): txt0 += '\n' tmp = tab0[nn] txt0 += 'Date Label : ' + tmp['datelabel'] + '\n' txt0 += 'UT Date : ' + tmp['UT_date'] + '\n' txt0 += tmp['object'] + '\n' txt0 += 'EXPTIME=%.1f ' % tmp['exptime'] txt0 += 'AIRMASS=%.3f \n' % tmp['airmass'] txt0 += '%s %.3f %s' % (tmp['grating'], tmp['gratwave'], tmp['filter2']) gc.add_label(0.975, 0.115, txt0, color='red', relative=True, ha='right', va='bottom', weight='medium', size='medium', bbox=bbox_props) gc.savefig(pp, format='pdf') if silent == False: mylogger.info('Writing : ' + out_pdf) # Mod on 08/12/2017 pp.close() #endelse out_pdf = out_pdf_default #endfor if silent == False: mylogger.info('### End main : ' + systime())
def clean_QA(path0='', out_pdf='', silent=False, verbose=True, overwrite=False): ''' Visually compare raw and cleanir-fixed FITS images Parameters ---------- path0 : str Full path to where output PDF and inputs FITS file are located. Must end with a '/' out_pdf : str Filename for output PDF. Do NOT include full path silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True overwrite : boolean Overwrite files if they exists. Default: False Returns ------- multi-page PDF plot Notes ----- Created by Chun Ly, 8 March 2017 Modified by Chun Ly, 10 March 2017 - Call quadrant_bias_values() Modified by Chun Ly, 23 March 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 25 April 2017 - Check cN*fits files if symlink. Modified by Chun Ly, 26 April 2017 - Fix bug with call to dir_check() - Change out_pdf default name (include clean suffix) - Fix handling of symlink files; Check for False, not True Modified by Chun Ly, 2 June 2017 - Added overwrite keyword option to overwrite file. Default is not to overwrite .pdf files Modified by Chun Ly, 8 December 2017 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 17 December 2017 - Change glog logging to common path - Add begin and end QA_clean logging to glog logfile Modified by Chun Ly, 11 January 2018 - Pass mylogger to dir_check.main() ''' # Moved up on 17/12/2017 logfile = path0 + 'QA_plot.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin clean_QA : ' + systime()) # + on 23/03/2017 | Mod on 26/04/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) out_pdf_default = out_pdf # + on 15/05/2017 # Mod on 23/03/2017 for path in list_path: if out_pdf == '': out_pdf = path + 'QA_plot.clean.pdf' else: out_pdf = path + out_pdf files = glob.glob(path + 'cN*fits') # Limit to non-symlinked files | + on 25/04/2017 files = [file for file in files if os.path.islink(file) == False] n_files = len(files) if overwrite == False and exists(out_pdf): mylogger.warn('File exists!! Will not overwrite ' + out_pdf) else: pp = PdfPages(out_pdf) for nn in xrange(n_files): if silent == False: mylogger.info('Reading : ' + files[nn]) orig_file = files[nn].replace('cN', 'N') im1 = fits.getdata(orig_file) im2 = fits.getdata(files[nn]) #hdu1 = fits.open(orig_file) #im1 = hdu1[1].data #hdu1[1].header = hdu1[0].header # Copy WCS header over #hdu2 = fits.open(files[nn]) #im2 = hdu2[1].data #hdu2[1].header = hdu2[0].header # Copy WCS header over fig = plt.figure(figsize=(16, 8)) gc1 = aplpy.FITSFigure(orig_file, figure=fig, subplot=[0.05, 0.055, 0.47, 0.94]) z1, z2 = zscale.get_limits(im1) gc1.show_grayscale(invert=True, vmin=z1, vmax=z2) gc1.set_tick_color('black') gc1.set_tick_yspacing('auto') #gc1.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec #gc1.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm') gc1.add_label(0.025, 0.975, orig_file, color='red', relative=True, ha='left', va='top', weight='medium', size='medium', bbox=bbox_props) quadrant_bias_values(fits.open(orig_file), gc1) # + on 10/03/2017 gc2 = aplpy.FITSFigure(files[nn], figure=fig, subplot=[0.525, 0.055, 0.47, 0.94]) z1, z2 = zscale.get_limits(im2) gc2.show_grayscale(invert=True, vmin=z1, vmax=z2) gc2.set_tick_color('black') gc2.set_tick_yspacing('auto') gc2.hide_ytick_labels() gc2.hide_yaxis_label() gc2.add_label(0.025, 0.975, files[nn], color='red', relative=True, ha='left', va='top', weight='medium', size='medium', bbox=bbox_props) quadrant_bias_values(fits.open(files[nn]), gc2) # + on 10/03/2017 #gc2.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec #gc2.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm') fig.savefig(pp, format='pdf') if silent == False: mylogger.info('Writing : ' + out_pdf) pp.close() #endelse out_pdf = out_pdf_default #endfor if silent == False: mylogger.info('### End clean_QA : ' + systime())
def cross_check(path, cdir, dbase): ''' Check arc/OH calibration against OH/arc dataset Parameters ---------- silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True skysub : boolean Display skysubtracted or un-skysubtracted images. Default: False Returns ------- Notes ----- Created by Chun Ly, 31 May 2018 - Generate transformed image with iraf.gnirs.nstransform - Call mylogger to warn if files exist - Change gnirs logfile - Move call to iraf.gemini.nsheaders to this function - iraf.nstransform does not like suffixes. Using underscores - Call OH_check - Check for database file before running nsfitcoords and nstransform Modified by Chun Ly, 3 June 2018 - Bug fix: Call OH_check with cross_check=True - Call arc_check2 for database_OH case Modified by Chun Ly, 8 June 2018 - Exit when db_file not available Modified by Chun Ly, 19 June 2018 - Call get_database_model - Pass function and order to nsfitcoords Modified by Chun Ly, 20 June 2018 - Set nsfitcoords xorder fitting ''' logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() timestamp = systime().replace(':','.') logfile = path+'gnirs_'+timestamp+'.log' iraf.gemini.gnirs.logfile = logfile iraf.gemini.nsheaders("gnirs") mylogger.info("Raw data is located in : %s" % path) mylogger.info("GNIRS logfile : "+logfile) iraf.chdir(path) if dbase == 'database/': infile = 'OH_stack.fits' outfile = 'fOH_stack_arc.fits' lamp = 'warc_stack.fits' if dbase == 'database_OH/': infile = 'arc_stack.fits' outfile = 'farc_stack_OH.fits' lamp = 'wOH_stack.fits' db_file = dbase+'id'+lamp.replace('.fits','_SCI_1_') if not exists(db_file): mylogger.warn("Wavelength calibration file not found : "+db_file) mylogger.warn("Exiting!!!") return else: if not exists(outfile): source0 = 'OH' if '_OH' in dbase else 'arc' func0, order0 = get_database_model(path, source0) iraf.gnirs.nsfitcoords(infile, outprefix='', outspectra=outfile, lamp=lamp, database=dbase, function=func0, lyorder=order0, lxorder=xorder) else: mylogger.warn('File exists! : '+outfile) t_outfile = 't'+outfile if not exists(t_outfile): iraf.gnirs.nstransform(outfile, outprefix='', outspectra=t_outfile, database=dbase) else: mylogger.warn('File exists! : '+t_outfile) iraf.chdir(cdir) if dbase == 'database/': OH_check(path, cross_check=True) # + on 03/06/2018 if dbase == 'database_OH/': arc_check2(path, cross_check=True)
def main(path0='', out_pdf='', check_quality=True, skysub=False, silent=False, verbose=True, overwrite=False): ''' main() function to compute natural seeing (image quality) from bright alignment star Parameters ---------- path0 : str Full path to where output PDF and FITS file are located. Must end with a '/' out_pdf : str Filename for output PDF. Do NOT include full path check_quality : boolean Check whether data meets IQ requirements. Default: True silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True overwrite : boolean Overwrite files if they exists. Default: False Returns ------- multi-page PDF plot Notes ----- Created by Chun Ly, 10 March 2017 - Later modified to include check_quality keyword option - Later modified to include inset that shows the stacked line profile Modified by Chun Ly, 11 April 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 13 April 2017 - Minor bug: Check if file exists first Modified by Chun Ly, 10 May 2017 - Minor bug: When .lis file contains only one entry, problem for appending to list Modified by Chun Ly, 1 June 2017 - Added overwrite keyword option to overwrite file. Default is not to overwrite .pdf files - Bug found: No longer need sky.lis since obj.lis includes all Modified by Chun Ly, 6 June 2017 - Add skysub keyword option to operate on sky-subtracted images Modified by Chun Ly, 14 July 2017 - Fix tick mark locations - Fix y limit range for extreme outliers Modified by Chun Ly, 16 November 2017 - Change prefix: rnc to rbnc Modified by Chun Ly, 18 December 2017 - Import glog and call for stdout and ASCII logging - Pass mylogger to compute_fwhm() Modified by Chun Ly, 11 January 2018 - Pass mylogger to dir_check.main() Modified by Chun Ly, 18 April 2018 - Compute and report seeing at zenith - Show FWHM @ Zenith on right y-axis Modified by Chun Ly, 19 April 2018 - Include airmass info in plots Modified by Chun Ly, 14 May 2018 - Write ASCII table containing FWHM determination Modified by Chun Ly, 18 May 2018 - Adjust subplots_adjust to avoid labels being cut off - Handle case when extra plot window (odd numbers) is available - Handle case with extra plot window (cont'd) Modified by Chun Ly, 19 May 2018 - Include QA (PASS/USABLE/FAIL) info in table Modified by Chun Ly, 28 June 2018 - Bug troubleshooting with ValueError - Handle ValueError for avg FWHM ''' # + on 18/12/2017 logfile = path0 + 'IQ_plot.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) # + on 11/04/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) out_pdf_default = out_pdf # Mod on 11/04/2017 for path in list_path: files = [] file_lis = ['obj.lis', 'telluric.lis'] # Minor bug fix on 01/06/2017 for file0 in file_lis: # Mod on 13/04/2017 if exists(path + file0): if silent == False: mylogger.info('Reading : ' + path + file0) t_files = np.loadtxt(path + file0, dtype=type(str)).tolist() # Bug fix - 10/05/2017 if type(t_files) == str: t_files = [t_files] files += t_files else: if silent == False: mylogger.info('File not found : ' + path + file0) files.sort() n_files = len(files) # + on 14/05/2018 FWHM_avg_arr = np.zeros(n_files) # FWHM from averaging stack0_shift FWHM_avg_arr_Z = np.zeros( n_files) # FWHM from averaging stack0_shift at Zenith FWHM_avg_QA = [ '' ] * n_files # QA check from averaging stack0_shift at Zenith FWHM_med_arr = np.zeros(n_files) # FWHM from median along Y FWHM_med_arr_Z = np.zeros( n_files) # FWHM from median along Y at Zenith FWHM_med_QA = [''] * n_files # QA check from median along Y at Zenith # Mod on 06/06/2017 if skysub == True: files = ['brnc' + file0 for file0 in files] # Mod on 16/11/2017 out_pdf = path+'IQ_plot.skysub.pdf' if out_pdf == '' else \ path+out_pdf else: out_pdf = path+'IQ_plot.raw.pdf' if out_pdf == '' else \ path+out_pdf if overwrite == False and exists(out_pdf): mylogger.warn('File exists!! Will not overwrite ' + out_pdf) else: pp = PdfPages(out_pdf) for nn in xrange(n_files): if silent == False: mylogger.info('Reading : ' + files[nn]) hdr0 = fits.getheader(path + files[nn]) # Mod on 06/06/2017 if skysub == False: im0 = fits.getdata(path + files[nn]) else: im0 = fits.getdata(path + files[nn], 'sci') airmass = hdr0['AIRMASS'] # + on 18/04/2018 # Mod on 18/12/2017 bins, fwhm0, stack0_shift = compute_fwhm(im0, mylogger=mylogger) row = nn % 2 if row == 0: fig, ax0 = plt.subplots(2, 1) good = np.where(fwhm0 > 0)[0] ax0[row].plot(bins[good], fwhm0[good], marker='o', alpha=0.5, mec='none', mfc='b', linestyle='none', zorder=2) ax0[row].get_yaxis().set_tick_params(which='both', right=True, width=1, direction='in') ax0[row].get_xaxis().set_tick_params(which='both', top=True, width=1, direction='in') # Compute average line profile from stack0_shift # Later + on 10/03/2017 avg_stack = np.average(stack0_shift, axis=0) x0_avg = np.arange(-25, 25) if row == 0: axi = fig.add_axes([0.60, 0.81, 0.25, 0.15]) if row == 1: axi = fig.add_axes([0.60, 0.36, 0.25, 0.15]) axi.plot(x0_avg * pscale, avg_stack, 'k-') axi.set_ylim([-0.4, 1.1]) axi.set_xlim([-2.0, 2.0]) axi.set_xticks(range(-2, 2, 1)) axi.set_xlabel('X [arcsec]', fontsize=8) axi.tick_params(labelsize=8) axi.minorticks_on() p0 = [0.0, 1.0, 0.0, 2.0] try: popt, pcov = curve_fit(gauss1d, x0_avg, avg_stack, p0=p0) fit_good = 1 except ValueError: print len(np.where(np.isnan(x0_avg))[0]) print len(np.where(np.isnan(avg_stack))[0]) fit_good = 0 if fit_good: avg_fwhm0 = popt[3] * 2 * np.sqrt(2 * np.log(2)) * pscale avg_fwhm_Z = avg_fwhm0 / airmass**0.6 # + on 18/04/2018 axi.plot(x0_avg * pscale, gauss1d(x0_avg, *popt), 'r--') axi.annotate('FWHM = %.3f" (%.3f")' % (avg_fwhm0, avg_fwhm_Z), [0.50, 0.025], xycoords='axes fraction', ha='center', va='bottom', fontsize=8) ax0[row].axhline(y=avg_fwhm0, linewidth=2, color='r', linestyle='--', zorder=1) # Median FWHM | Later + on 10/03/2017 med_fwhm0 = np.median(fwhm0[good]) med_fwhm0_Z = med_fwhm0 / airmass**0.6 # + on 18/04/2018 ax0[row].axhline(y=med_fwhm0, linewidth=2, color='g', linestyle='--', zorder=1) # Axes labeling ax0[row].set_ylabel('FWHM [arcsec]') if row == 1 or nn == n_files - 1: ax0[row].set_xlabel('Y [pixel]') else: ax0[row].set_xticklabels([]) if nn == n_files - 1: if row == 0: ax0[row + 1].axis('off') # Annotation txt0 = files[nn] + '\nTarget: ' + hdr0[ 'OBJECT'] #.split(' ')[0] ax0[row].annotate(txt0, [0.025, 0.95], xycoords='axes fraction', ha='left', va='top') # + on 19/04/2018 ax0[row].annotate('AM = %.3f' % airmass, [0.025, 0.025], xycoords='axes fraction', ha='left', va='bottom') # Later + on 10/03/2017 if check_quality: req = hdr0['REQIQ'].replace('-percentile', '%') raw = hdr0['RAWIQ'].replace('-percentile', '%') i_raw = [ ii for ii in range(len(FWHM_IQ_C)) if raw in FWHM_IQ_C[ii] ][0] i_req = [ ii for ii in range(len(FWHM_IQ_C)) if raw in FWHM_IQ_C[ii] ][0] txt0 = 'Req. IQ: %s [%.2f"]\n' % (req, FWHM_IQ_J[i_req]) txt0 += 'Raw IQ: %s [%.2f"]\n' % (raw, FWHM_IQ_J[i_raw]) # Mod on 18/04/2018 if med_fwhm0_Z <= FWHM_IQ_J[i_raw]: txt0 += 'PASS' FWHM_med_QA[nn] = 'PASS' # + on 19/05/2018 else: if med_fwhm0_Z <= FWHM_IQ_J[i_raw] * 1.25: txt0 += 'USABLE' FWHM_med_QA[nn] = 'USABLE' # + on 19/05/2018 if med_fwhm0_Z > FWHM_IQ_J[i_raw] * 1.25: txt0 += 'FAIL' FWHM_med_QA[nn] = 'FAIL' # + on 19/05/2018 # + on 19/05/2018 if fit_good: if avg_fwhm_Z <= FWHM_IQ_J[i_raw]: FWHM_avg_QA[nn] = 'PASS' else: if avg_fwhm_Z <= FWHM_IQ_J[i_raw] * 1.25: FWHM_avg_QA[nn] = 'USABLE' if avg_fwhm_Z > FWHM_IQ_J[i_raw] * 1.25: FWHM_avg_QA[nn] = 'FAIL' ax0[row].annotate(txt0, [0.975, 0.05], ha='right', xycoords='axes fraction', va='bottom') # Aesthetics ax0[row].set_xlim([0, 1050]) if max(fwhm0[good]) > 3: ax0[row].set_ylim([0, 3.0]) else: ax0[row].set_ylim([min(fwhm0) - 0.025, max(fwhm0) + 0.075]) ax0[row].minorticks_on() # + on 18/04/2018 ax2 = ax0[row].twinx() ax2.set_ylabel(r"FWHM @ Zenith [arcsec]") ax2.set_ylim(np.array(ax0[row].get_ylim()) / airmass**0.6) ax2.minorticks_on() if row != 1: ax2.set_xticklabels([]) if row == 1 or nn == n_files - 1: subplots_adjust(left=0.11, bottom=0.10, top=0.975, right=0.875, wspace=0.03, hspace=0.05) fig.savefig(pp, format='pdf') # + on 14/05/2018 if fit_good: FWHM_avg_arr[nn] = avg_fwhm0 FWHM_avg_arr_Z[nn] = avg_fwhm_Z FWHM_med_arr[nn] = med_fwhm0 FWHM_med_arr_Z[nn] = med_fwhm0_Z #endfor if silent == False: mylogger.info('Writing : ' + out_pdf) pp.close() # + on 14/05/2018 fwhm_out_file = out_pdf.replace('.pdf', '.tbl') arr0 = [ files, FWHM_avg_arr, FWHM_avg_arr_Z, FWHM_med_arr, FWHM_med_arr_Z, FWHM_avg_QA, FWHM_med_QA ] # Mod on 19/05/2018 names0 = ('files', 'FWHM_avg', 'FWHM_avg_Z', 'FWHM_med', 'FWHM_med_Z', 'FWHM_avg_QA', 'FWHM_med_QA' ) # Mod on 19/05/2018 tab0 = Table(arr0, names=names0) if silent == False: mylogger.info('Writing : ' + fwhm_out_file) asc.write(tab0, fwhm_out_file, format='fixed_width_two_line', overwrite=True) out_pdf = out_pdf_default #endelse #endfor if silent == False: mylogger.info('### End main : ' + systime())
def create_distort_grid(rawdir, silent=False, verbose=True): ''' Create grid (and plot) of distortion for extraction Parameters ---------- None Returns ------- None Notes ----- Created by Chun Ly, 29 June 2018 Modified by Chun Ly, 10 August 2018 - Plot best fit Modified by Chun Ly, 13 August 2018 - Rewrite to work for specified rawdir path (instead of all) - Fix typo with ax.axhline indexing - Re-organize code (handle only one distort_trace.npz file) - Minor bug fix: array name - Plot aesthetics ''' if rawdir[-1] != '/': rawdir += '/' logfile = rawdir + 'distort_trace.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin ! ') dir_list, list_path = dir_check.main(rawdir, mylogger=mylogger, silent=silent, verbose=verbose) for path in list_path: mylogger.info('Working on : ' + path.split('/')[-2]) npz_files = glob(path + '/distort_trace.npz') if len(npz_files) == 0: log.warn('No files found!!!') else: fig, ax = plt.subplots(nrows=3) npz = np.load(npz_files[0]) xcen_arr = npz['xcen_arr'] fit_arr = npz['fit_arr'] best_fit = npz['best_fit'] n_peaks = xcen_arr.shape[0] for pp in range(n_peaks): ax[0].scatter(xcen_arr[pp], fit_arr[pp, :, 2], marker='o', edgecolor='k', facecolor='none') ax[0].axhline(y=best_fit[2], color='b') ax[1].scatter(xcen_arr[pp], fit_arr[pp, :, 1], marker='o', edgecolor='k', facecolor='none') ax[1].axhline(y=best_fit[1], color='b') ax[2].scatter(xcen_arr[pp], fit_arr[pp, :, 0], marker='o', edgecolor='k', facecolor='none') ax[2].axhline(y=best_fit[0], color='b') ax[0].annotate(r'x = A y$^2$ + B y + C', xy=(0.02, 0.95), xycoords='axes fraction', ha='left', va='top') ax[0].set_ylabel('C') ax[1].set_ylabel('B') ax[2].set_ylabel('A') ax[0].set_ylim([-10, 0]) ax[1].set_ylim([-0.025, 0.025]) ax[2].set_ylim([-0.001, 0.001]) ax[0].set_xticklabels([]) ax[1].set_xticklabels([]) ax[2].set_xlabel('X [pix]') plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.1) out_pdf = path + 'distort_grid.pdf' log.info('Writing : ' + out_pdf) fig.savefig(out_pdf)
def main(rawdir, line_source='', mylogger=None, silent=False, verbose=True): ''' Main function for wave_cal_script Parameters ---------- rawdir : str Path to raw files. Must end in a '/' line_source : str Type of lines to use for wavelength calibration. Option is either 'arc' or 'OH silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 7-8 October 2017 Modified by Chun Ly, 8 November 2017 - Specify lampspec and outspec Modified by Chun Ly, 16 November 2017 - Handle line_source == 'OH' - Generalized arrays: arc -> frame - Specify GNIRS log file Modified by Chun Ly, 20 November 2017 - Bug fix: prefix needed for [frames] for line_source == OH Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 9 January 2018 - Allow mylogger keyword Modified by Chun Ly, 30 May 2018 - Change order for nswavelength fitting Modified by Chun Ly, 16 June 2018 - Change coordlist for OH skylines Modified by Chun Ly, 19 June 2018 - Force function to legendre - Set fwidth in nswavelength call to depend on slitwidth Modified by Chun Ly, 21 June 2018 - Include ending print statement Modified by Chun Ly, 10 July 2018 - Modify threshold for OH lines from 50 to 25 ''' # Mod on 10/01/2018 if type(mylogger) == type(None): logfile = rawdir + 'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) timestamp = systime().replace(':', '.') logfile = rawdir + 'gnirs_' + timestamp + '.log' rawdir = check_path(rawdir) if line_source == '': mylogger.warn("Must specify line_source keyword: ") mylogger.warn("line_source='arc' or line_source='OH'") mylogger.warn("Exiting!!!") return else: out_script = rawdir + 'wave_cal_' + line_source + '.py' if silent == False: mylogger.info('Writing : ' + out_script) f0 = open(out_script, 'w') line0 = [ 'iraf.gemini(_doprint=0)', 'iraf.gemini.gnirs(_doprint=0)', 'iraf.gemini.unlearn()', 'iraf.gemini.gemtools.unlearn()', 'iraf.gemini.gnirs.unlearn()', 'iraf.set(stdimage="imt4096")', 'iraf.gemini.nsheaders("gnirs")' ] #'iraf.gemini.gnirs.logfile = "%s"' % logfile] # + on 16/11/2017 f0.writelines("\n".join(line0) + "\n") if line_source == 'arc': frame_list = rawdir + 'arc.lis' if line_source == 'OH': frame_list = rawdir + 'obj.OH.lis' frames = np.loadtxt(frame_list, dtype=type(str)) if line_source == 'OH': frames = ['rbnc' + file0 for file0 in frames] frame_hdr = fits.getheader(rawdir + frames[0]) crpix = n_sp_pix / 2.0 crval = frame_hdr['gratwave'] * 1e4 # in Angstroms if frame_hdr['FILTER2'] == 'X_G0518': cdelt = -0.094 * 1e4 / n_sp_pix if frame_hdr['FILTER2'] == 'J_G0517': cdelt = -0.113 * 1e4 / n_sp_pix mylogger.info('## CRVAL : %.1f ' % crval) mylogger.info('## CDELT : %.1f CRPIX : %.1f' % (cdelt, crpix)) line1 = ['crval = %f' % crval, 'crpix = %f' % crpix, 'cdelt = %f' % cdelt] f0.writelines("\n".join(line1) + "\n") # Get slit length | + on 19/06/2018q slitwidth = np.float(frame_hdr['SLIT'].split('arcsec')[0]) pscale = np.sqrt(frame_hdr['CD1_1']**2 + frame_hdr['CD2_1']**2) * 3600.0 * u.arcsec fwidth = 1.5 * slitwidth / pscale.to(u.arcsec).value if line_source == 'arc': coordlist = 'gnirs$data/argon.dat' database = 'database/' threshold = 50 if line_source == 'OH': coordlist = rawdir + 'rousselot2000_convl.dat' if not exists(coordlist): log.warn('File does not exists!!! : ' + coordlist) database = 'database_OH/' threshold = 25 # Mod on 08/11/2017 line2 = [ "coordlist = '%s'" % coordlist, "database = '%s'" % database, "lampspec = '%s_stack.fits'" % line_source, "outspec = 'w%s_stack.fits'" % line_source, "threshold = %f" % threshold, "logfile = '%s'" % logfile ] # + on 16/11/2017 f0.writelines("\n".join(line2) + "\n") # Mod on 08/11/2017, 16/11/2017 cmd = "iraf.gnirs.nswavelength(lampspec, outprefix='',"+\ "outspectra=outspec, crval=crval, cdelt=cdelt, crpix=crpix, "+\ "coordlist=coordlist, database=database, fl_inter='yes', "+\ "function='legendre', cradius=20, threshold=threshold, "+\ "fwidth=%.1f, " % fwidth +"order=3, logfile=logfile)" f0.write(cmd + '\n') f0.write('print("Completed! Return to other terminal and hit RETURN")\n') f0.close() if silent == False: mylogger.info('### End main : ' + systime())
def main(rawdir, silent=False, verbose=True): ''' Main function for distort_trace Parameters ---------- rawdir : str Path to FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 27 June 2018 - Write multi-page PDF file - Remove center value for middle of spectra - Plot offsets - Bug fix for curve_fit (use bb_med0); plot aesthetics (legend) - Plot aesthetics (axes labeling), margin adjustments - Call group() to get matplotlib markers and colors - Write npz file using np.savez and np.load when available - Do linear regression fit and save to npz file Modified by Chun Ly, 28 June 2018 - Change linear regression to 2nd order, plot fit Modified by Chun Ly, 29 June 2018 - Write PDF file in each datedir - Minor changes to mylogger calls Modified by Chun Ly, 31 July 2018 - Use sigma_clipped_stats; Search for multiple peaks Modified by Chun Ly, 1 August 2018 - Smooth median signal (boxcar); mask for peaks - Call group_index(); Require at leat 5 pixels for peak identification - Handle multiple peaks - Compute number of peaks only once Modified by Chun Ly, 2 August 2018 - Use combine stack for peak identification (more sensitive) - Get quick peak centers from combine stack - Use peak in med0 if no combine stack or telluric spec - Handle peak finding for both telluric and science data - Handle multiple peaks in plotting - Restrict fitting to within 10 pixels Modified by Chun Ly, 3 August 2018 - Simplify code (x0_bb -> x0) - Fix ValueError: Invalid rgba arg "" - Fix bugs with n_files and use of 'x' datapoints - Use median for x_cen_middle to deal with outliers - Compute median using sigma_clipped_stats, exclude outliers from polyfit - Switch flag to flag0 to avoid conflict - Use peak when combine stack has single line - Switch from curve_fit to weighted centering - Define p_idx for single peak Modified by Chun Ly, 7 August 2018 - Bug fix: Incorrect x0_diff - Handle case when not all skysubtracted frames are used (check for obj_rev.lis file) - Force integer for index - Avoid right edge issue (specific hack for one target) - Use DQ array to masked bad pixels and edges - Limit weighted computation within expected location - Shift x0_max1 for sub-indexing - Force x-limit range (zoom-in) - Plot aesthetics: ax annotation Modified by Chun Ly, 8 August 2018 - Plot fitting results - Bug fix with savefig location, change labeling Modified by Chun Ly, 9 August 2018 - Fix typo with wrong ax subplots use (row determination) - Annotate plot with center - Plot aesthetics: toplabel, white space, ylim - Changes for right center for telluric and single peak science data cases - Adjust ax.plot xrange Modified by Chun Ly, 10 August 2018 - Compute median of trace_arr in each bin, best fit polynomial fit - Plot best fit to median of trace_arr - Update npz savefile with best_fit results - Annotate plot with best fit - Fix IndexError issue (limit v1 >= 0) - Change fig suptitle (just the filename) ''' if rawdir[-1] != '/': rawdir += '/' logfile = rawdir + 'distort_trace.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin ! ') bin_size = 25 dir_list, list_path = dir_check.main(rawdir, mylogger=mylogger, silent=silent, verbose=verbose) for path in list_path: mylogger.info('Working on : ' + path.split('/')[-2]) out_pdf = path + 'distort_trace.pdf' out_pdf1 = path + 'distort_trace_fit.pdf' npz_file = path + 'distort_trace.npz' obj_file = path + 'obj_rev.lis' if not exists(npz_file): if not exists(obj_file): files = glob(path + 'tfrbncN????????S????.fits') else: mylogger.info('File found : ' + obj_file) npfiles = np.loadtxt(obj_file, dtype=type(str)) files = [path + 'tfrbnc' + t_file for t_file in npfiles] npfilesT = np.loadtxt(path + 'telluric.lis', dtype=type(str)) filesT = [path + 'tfrbnc' + t_file for t_file in npfilesT] files += filesT files.sort() n_files = len(files) mylogger.info('Number of files found : %i ' % n_files) if n_files > 0: pdf_pp = PdfPages(out_pdf1) hdr0 = fits.getheader(files[0], extname='SCI') n_bins = np.int(np.ceil(np.float(hdr0['NAXIS2'])) / bin_size) x0 = np.arange(hdr0['NAXIS1']) y0 = bin_size / 2.0 + bin_size * np.arange(n_bins) no_c_file = 0 c_file = glob(path + 'obj_comb.fits') if len(c_file) == 0: mylogger.warn('Combine frame not found! Using single peak') no_c_file = 1 n_peaks = 1 else: t_c_im = fits.getdata(c_file[0], extname='SCI') c_med0 = np.median(t_c_im, axis=0) # Find peaks | + on 31/07/2018 x0_max = np.argmax(c_med0) x0_min = np.argmin(c_med0) idx_mask = np.where((np.absolute(x0 - x0_max) >= 10) & (np.absolute(x0 - x0_min) >= 10))[0] sm_c_med0 = convolve(c_med0, box_kernel) t_mean, t_med, t_std = sigma_clipped_stats( sm_c_med0[idx_mask], sigma=2, iters=20) idx_det = np.where((sm_c_med0 - t_med) / t_std >= 5)[0] list_peak = np.array(list(group_index(idx_det))) peak_idx = [ xx for xx in range(len(list_peak)) if (list_peak[xx][1] - list_peak[xx][0] >= 5 and list_peak[xx][0] < 625) ] # Avoid right edge issues list_peak = list_peak[peak_idx] n_peaks = len(peak_idx) mylogger.info('Number of peaks found : ' + str(n_peaks)) peak_ctr = np.zeros(n_peaks) for pp in range(n_peaks): i1, i2 = list_peak[pp][0], list_peak[pp][1] peak_ctr[pp] = i1 + np.argmax(c_med0[i1:i2]) trace_arr = np.zeros((n_peaks, n_files, n_bins)) xcen_arr = np.zeros((n_peaks, n_files)) fit_arr = np.zeros((n_peaks, n_files, 3)) for ff in range(n_files): fig, ax = plt.subplots(nrows=nrows, ncols=ncols) t_im0, t_hdr = fits.getdata(files[ff], extname='SCI', header=True) t_dq = fits.getdata(files[ff], extname='DQ') t_im = np.ma.array(t_im0, mask=t_dq) med0 = np.ma.median(t_im, axis=0) x0_max0 = np.ma.argmax(med0) h_obj = t_hdr['OBJECT'] if no_c_file or ('HIP' in h_obj or 'HD' in h_obj): n_peak, use_peak = 1, 1 else: n_peak = n_peaks use_peak = 1 if n_peaks == 1 else 0 x0_diff = peak_ctr[0] - x0_max0 for bb in range(n_bins): row, col = bb / ncols, bb % ncols ty1, ty2 = (0 + bb) * bin_size, (1 + bb) * bin_size bb_med0 = np.ma.median(t_im[ty1:ty2], axis=0) for pp in range(n_peak): if use_peak == 1: if no_c_file or ('HIP' in h_obj or 'HD' in h_obj): v1, v2 = x0_max0 - 15, x0_max0 + 15 else: v1 = np.max([ np.int(list_peak[0][0] - x0_diff - 15), 0 ]) v2 = np.int(list_peak[0][1] - x0_diff + 15) # print ff, bb, pp, v1, v2, x0_max0 x0_max1 = v1 + np.ma.argmax(bb_med0[v1:v2]) p_idx = np.arange(x0_max1 - 10, x0_max1 + 10) # print ff, bb, pp, x0_max0, x0_max1, p_idx[0], p_idx[-1] else: p_idx = np.arange( np.int(list_peak[pp][0] - x0_diff), np.int(list_peak[pp][1] - x0_diff)) p_med0 = bb_med0[p_idx] x_cen = np.sum(p_med0 * p_idx) / np.sum(p_med0) #p0 = [0.0, y0_max, x0_max, 2.0] #try: # popt, pcov = curve_fit(gauss1d, x0, bb_med0, p0=p0) # x_cen = popt[2] #except RuntimeError: # print 'Runtime error' # x_cen = p0[2] trace_arr[pp, ff, bb] = x_cen x_off = p_idx[len(p_idx) / 2] ax[row, col].plot(p_idx - x_off, p_med0 / max(p_med0)) ax[row, col].axvline(x=x_cen - x_off) ax[row, col].annotate('%.1f' % x_cen, xy=[0.95, 0.95], ha='right', va='top', fontsize=9, xycoords='axes fraction') if row != nrows - 1: ax[row, col].set_xticklabels([]) if col != 0: ax[row, col].set_yticklabels([]) ax[row, col].set_ylim([-0.05, 1.05]) #endfor #endfor plt.subplots_adjust(left=0.08, right=0.99, bottom=0.05, top=0.95, hspace=0.02, wspace=0.02) fig.suptitle(os.path.basename(files[ff]), fontsize=12) fig.savefig(pdf_pp, format='pdf') #endfor mylogger.info('Writing : ' + out_pdf1) pdf_pp.close() flag0 = np.ones((n_peaks, n_files, n_bins)) for ff in range(n_files): for pp in range(n_peaks): t_me, t_md, t_s = sigma_clipped_stats(trace_arr[pp, ff], sigma=3, iters=10) x_cen_middle = t_md xcen_arr[pp, ff] = x_cen_middle trace_arr[pp, ff] -= x_cen_middle diff = trace_arr[pp, ff] - ((y0 - 512) * 0.019) use = np.where(np.absolute(diff) <= 5)[0] if len(use) > 0: flag0[pp, ff, use] = 0 fit = np.polyfit(y0[use], trace_arr[pp, ff][use], 2) fit_arr[pp, ff] = fit #endfor #endfor mylogger.info('Writing : ' + npz_file) np.savez(npz_file, trace_arr=trace_arr, xcen_arr=xcen_arr, fit_arr=fit_arr, y0=y0, flag0=flag0) else: mylogger.warn('Files not found !') else: mylogger.info('Reading : ' + npz_file) npz = np.load(npz_file) trace_arr = npz['trace_arr'] xcen_arr = npz['xcen_arr'] y0 = npz['y0'] fit_arr = npz['fit_arr'] flag0 = npz['flag0'] n_files = xcen_arr.shape[1] n_peaks = xcen_arr.shape[0] if n_files > 0: fig, ax = plt.subplots() xlim = [-10, 10] ctype, mtype, labels = group(xcen_arr) x_fit0 = np.zeros(len(y0)) for bb in range(len(y0)): use = np.where(flag0[:, :, bb] == 0) x_fit0[bb] = np.median(trace_arr[use[0], use[1], bb]) best_fit = np.polyfit(y0, x_fit0, 2) mylogger.info('Best fit : [%f, %f, %f]', best_fit[0], best_fit[1], best_fit[2]) for pp in range(n_peaks): for ff in range(n_files): if labels[pp, ff] != '': fc = ctype[pp, ff] if mtype[pp, ff] == 'x' else 'none' ax.scatter(trace_arr[pp, ff, :], y0, marker=mtype[pp, ff], alpha=0.5, edgecolor=ctype[pp, ff], facecolor=fc, label=labels[pp, ff]) ax.scatter(trace_arr[pp, ff, flag0[pp, ff] == 1], y0[flag0[pp, ff] == 1], marker='x', color='r') pd = np.poly1d(fit_arr[pp, ff]) ax.plot(pd(y0), y0, color=ctype[pp, ff], linewidth=0.75, alpha=0.5) ax.scatter(x_fit0, y0, marker='o', edgecolor='none', facecolor='black', linewidth=2.0, alpha=0.9) best_pd = np.poly1d(best_fit) ax.plot(best_pd(y0), y0, color='black', linewidth=2.0, alpha=0.9) b_txt = r'x = A y$^2$ + B y + C'+\ '\nA = %.3e\nB = %.3e\nC = %.3f' % (best_fit[0],best_fit[1], best_fit[2]) ax.annotate(b_txt, xy=(0.05, 0.75), xycoords='axes fraction', ha='left', va='top') out_plt = np.where((trace_arr > xlim[1]) | (trace_arr < xlim[0])) n_out_plt = len(out_plt[0]) flagged = np.where(flag0 == 1) n_flagged = len(flagged[0]) a_txt = 'N(exclude) = ' + str(n_out_plt) + '\nN(flagged) = ' + str( n_flagged) ax.annotate(a_txt, xy=(0.05, 0.95), xycoords='axes fraction', ha='left', va='top') ax.legend(loc='lower right') ax.set_ylabel('Y [pixels]', fontsize=14) ax.set_xlabel('X relative to center [pixels]', fontsize=14) ax.set_xlim(xlim) #[-100,100]) ax.set_ylim([-10, 1050]) fig.suptitle(path) ax.minorticks_on() fig.set_size_inches(8, 8) plt.subplots_adjust(left=0.1, right=0.98, bottom=0.07, top=0.98) mylogger.info('Writing : ' + out_pdf) fig.savefig(out_pdf, format='pdf') mylogger.info('Updating : ' + npz_file) np.savez(npz_file, trace_arr=trace_arr, xcen_arr=xcen_arr, fit_arr=fit_arr, y0=y0, flag0=flag0, best_fit=best_fit) #endfor if silent == False: mylogger.info('### End ! ')
def main(path0, out_pdf='', silent=False, verbose=True, overwrite=False): ''' Main function to generate PDF illustrating alignment on target Parameters ---------- path0 : str Path to FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True overwrite : boolean Overwrite files if they exists. Default: False Returns ------- Notes ----- Created by Chun Ly, 24 March 2017 Modified by Chun Ly, 01 April 2017 - Handle CRs and bad pixels using cosmicrays_lacosmic Modified by Chun Ly, 04 April 2017 - Use find_gnirs_window_mean to find center Modified by Chun Ly, 04-05 April 2017 - Adjust greyscale limits to handle slit image (make it black), and faint sources Use find_gnirs_window_mean to find center Modified by Chun Ly, 05 April 2017 - Handle alignment sequences with more than just 4 frames - Handle excess subplots for individual PDF pages (remove axes) - Compute seeing FWHM for acquisition images Modified by Chun Ly, 06 April 2017 - Get coordinates for slit in cutout Modified by Chun Ly, 11 May 2017 - Use slit image to find center when telluric data is only available Modified by Chun Ly, 3 July 2017 - Add overwrite option to prevent overwriting file Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging - Pass mylogger to find_gnirs_window_mean(), find_gnirs_window() Modified by Chun Ly, 20 April 2018 - Pass mylogger to gauss2d_fit() - Switch print statements to mylogger calls Modified by Chun Ly, 22 April 2018 - Bug fix: mylogger calls mistakes - Bug fix: mylogger calls mistakes (cont'd) ''' # + on 09/01/2018 logfile = path0 + 'align_check.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) out_pdf_default = out_pdf for path in list_path: infile = path + 'hdr_info.QA.tbl' if not exists(infile): mylogger.warning('File does not exist : ' + infile) mylogger.warning('Exiting!!! ' + systime()) return out_pdf = path + 'align_check.pdf' if out_pdf == '' else path + out_pdf # Mod on 03/07/2017 if overwrite == False and exists(out_pdf): mylogger.warn('File exists!! Will not overwrite ' + out_pdf) else: pp = PdfPages(out_pdf) if silent == False: mylogger.info('Reading: ' + infile) tab0 = asc.read(infile, format='fixed_width_two_line') align = [ii for ii in xrange(len(tab0)) if tab0['QA'][ii] == 'N/A'] if silent == False: mylogger.info('Number of alignment images found : ' + str(len(align))) ID = tab0['object'][align] ID0 = list(set(ID)) #Unique ID's if silent == False: mylogger.info('Sources found : ' + ', '.join(ID0)) # + on 04/04/2017 win_ref_idx = [ tt for tt in xrange(len(tab0)) if (tab0['QA'][tt] == 'N/A') and ('Acq' in tab0['slit'][tt]) and ('HIP' not in tab0['object'][tt]) and ( 'HD' not in tab0['object'][tt]) ] # Mod on 11/05/2017 if len(win_ref_idx) > 0: win_ref_file = path + tab0['filename'][win_ref_idx[0]] mylogger.info('Reference image for finding GNIRS window : ' + win_ref_file) x_min, x_max, y_min, y_max, x_cen, \ y_cen = find_gnirs_window_mean(win_ref_file, mylogger=mylogger) else: mylogger.info('Using telluric image as reference') win_ref_file = path + tab0['filename'][0] slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace(win_ref_file) x_min, x_max = min(slit_x0), max(slit_x0) x_cen = (x_min + x_max) / 2.0 y_cen = (np.median(slit_y0_lo) + np.median(slit_y0_hi)) / 2.0 y_min, y_max = y_cen - size2d[0].value / 2.0, y_cen + size2d[ 0].value / 2.0 pos_cen = (x_cen, y_cen) new_size = u.Quantity((y_max - y_min, x_max - x_min), u.pixel) # + on 20/04/2018, Mod on 22/04/2018 mylogger.info('pos_cen : (%f, %f) ' % (pos_cen[0], pos_cen[1])) mylogger.info('new_size : [%f, %f] pix ' % (new_size[0].value, new_size[1].value)) for ii in xrange(len(ID0)): t_idx = [ tt for tt in xrange(len(tab0)) if (tab0['object'][tt] == ID0[ii] and tab0['QA'][tt] == 'N/A') ] t_files = [path + a for a in tab0['filename'][t_idx]] ncols = 2.0 nrows = 2 # np.ceil(len(t_idx)/ncols) ncols, nrows = np.int(ncols), np.int(nrows) # Mod on 05/04/2017 if len(t_idx) <= nrows * ncols: fig, ax_arr = plt.subplots(nrows=nrows, ncols=ncols) #med0, x_min, x_max, y_min, \ # y_max, x_cen, y_cen = find_gnirs_window(t_files[1], mylogger=mylogger) # Later + on 24/03/2017 | Mod on 04/04/2017 xcen, ycen = find_star(t_files[-1], pos=pos_cen, find_size2d=new_size) # Fix to get relative coordinate for Cutout2D image #xcen -= pos_cen[0]-new_size[1].value/2.0 #ycen -= pos_cen[1]-new_size[0].value/2.0 slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace( t_files[0]) #, x_min, x_max) # Adjust values for offset that is applied # Bug: Mod on 04/04/2017 to get proper coordinate slit_x0 -= np.int64(pos_cen[0] - size2d[1].value / 2.0) slit_y0_lo -= pos_cen[1] - size2d[0].value / 2.0 slit_y0_hi -= pos_cen[1] - size2d[0].value / 2.0 for jj in xrange(len(t_idx)): jj_idx = t_idx[jj] # + on 05/04/2017 if len(t_idx) > (nrows * ncols): if jj % (nrows * ncols) == 0: fig, ax_arr = plt.subplots(nrows=nrows, ncols=ncols) im0 = fits.getdata(t_files[jj]) hdr0 = fits.getheader(t_files[jj], ext=0) # Get WCS header # + 01/04/2017 im0_clean = cosmicray_lacosmic(im0, sigclip=10)[0] cutout = Cutout2D(im0_clean, pos_cen, size2d, mode='partial', fill_value=np.nan) t_col, t_row = jj % ncols, (jj / ncols) % nrows # Mod on 04/04/2017 to handle bright and faint stars max0 = np.max(cutout.data) # Compute median within GNIRS window # + on 04-05/04/2017 temp = im0_clean[-50:-1, :] bgd0, sig0 = np.median(temp), np.std(temp) idx_y, idx_x = np.where(im0_clean > (bgd0 + 5 * sig0)) med0 = np.median(im0_clean[idx_y, idx_x]) mylogger.info('## max0 : %f med0 : %f ' % (max0, med0)) if max0 > 50000: z1, z2 = zscale.get_limits(cutout.data) z2 = max0 # Change for better stretch for telluric star else: if ('Acq_' not in tab0['slit'][jj_idx]) and \ (tab0['exptime'][jj_idx] == 3): # First frame that will show the longslit z1, z2 = 0.0, 0.5 * max0 else: # This should handle faint and bright stars z1, z2 = 0.5 * med0, max0 norm = ImageNormalize(vmin=z1, vmax=z2) t_ax = ax_arr[t_row, t_col] t_ax.imshow(cutout.data, cmap='Greys', origin='lower', norm=norm) #aplpy.FITSFigure(cutout) # Draw trace of slit t_ax.plot(slit_x0, slit_y0_lo, 'r-') t_ax.plot(slit_x0, slit_y0_hi, 'r-') t_ax.xaxis.set_ticklabels([]) t_ax.yaxis.set_ticklabels([]) fig.suptitle(path, fontsize=14) txt0 = tab0['filename'][jj_idx] + '\n' txt0 += tab0['datelabel'][jj_idx] + '\n' txt0 += tab0['UT_date'][jj_idx] + '\n' txt0 += tab0['object'][jj_idx] t_ax.annotate(txt0, [0.025, 0.95], xycoords='axes fraction', ha='left', va='top') # Plot inset | Later + on 24/03/2017 axins = zoomed_inset_axes(t_ax, 6, loc=4) norm2 = ImageNormalize(vmin=z1, vmax=z2) axins.imshow(cutout.data, cmap='Greys', origin='lower', norm=norm2) # Draw trace of slit axins.plot(slit_x0, slit_y0_lo, 'r-') axins.plot(slit_x0, slit_y0_hi, 'r-') # Mod on 04/04/2017 to get Cutout2d coordinates c_xcen = xcen - (pos_cen[0] - size2d[1].value / 2.0) c_ycen = ycen - (pos_cen[1] - size2d[0].value / 2.0) x1, x2, y1, y2 = c_xcen - 20, c_xcen + 20, c_ycen - 20, c_ycen + 20 axins.set_xlim([x1, x2]) axins.set_ylim([y1, y2]) axins.xaxis.set_ticklabels([]) axins.yaxis.set_ticklabels([]) mark_inset(t_ax, axins, loc1=1, loc2=3, fc="none", ec="b", ls='dotted', lw=0.5) # Compute FWHM of alignment star | + on 05/04/2017 if ('Acq_' not in tab0['slit'][jj_idx]) and \ (tab0['exptime'][jj_idx] == 3): mylogger.info('No source in slit : ' + tab0['filename'][jj_idx]) else: # + on 06/04/2017 c_size2d = u.Quantity((40, 40), u.pixel) c_slit_x0 = slit_x0 - (c_xcen - c_size2d[1].value / 2.0) c_slit_y0_lo = slit_y0_lo - (c_ycen - c_size2d[0].value / 2.0) c_slit_y0_hi = slit_y0_hi - (c_ycen - c_size2d[0].value / 2.0) im0_crop = Cutout2D(cutout.data, (c_xcen, c_ycen), c_size2d, mode='partial', fill_value=np.nan) gauss2d_fit(im0_crop.data, hdr0, t_ax, c_slit_x0, c_slit_y0_lo, c_slit_y0_hi, mylogger=mylogger) # Mod on 06/04/2017 # Write each page separately | + on 05/04/2017 if len(t_idx) > (nrows * ncols): # Mod later on 05/04/2017 to handle excess subplots if jj == len(t_idx) - 1: rem0 = len(t_idx) % (nrows * ncols) # remainder if rem0 != 0: for rr in range(rem0, nrows * ncols, 1): t_col, t_row = rr % ncols, (rr / ncols) % nrows ax_arr[t_row, t_col].axis('off') if (jj % (nrows * ncols) == nrows*ncols-1) or \ (jj == len(t_idx)-1): subplots_adjust(left=0.02, bottom=0.02, top=0.95, right=0.98, wspace=0.02, hspace=0.02) fig.set_size_inches(11, 8) fig.savefig(pp, format='pdf') #endfor # Mod on 05/04/2017 if len(t_idx) <= nrows * ncols: # Mod later on 05/04/2017 to handle excess subplots for rr in range(len(t_idx), nrows * ncols): t_col, t_row = rr % ncols, (rr / ncols) % nrows ax_arr[t_row, t_col].axis('off') subplots_adjust(left=0.02, bottom=0.02, top=0.95, right=0.98, wspace=0.02, hspace=0.02) fig.set_size_inches(11, 8) fig.savefig(pp, format='pdf') #endfor pp.close() #endelse out_pdf = out_pdf_default if silent == False: mylogger.info('### End main : ' + systime())
def arc_check(path, arcs=[''], out_pdf='', stack=False, silent=False, verbose=True): ''' Generate plot illustrating wavelength calibration from IRAF database Parameters ---------- path : str Full path to where output PDF and FITS file are located. Must end with a '/' arcs : str or list (Optional) List of raw filenames for the arc data (e.g., 'N20170101S0111.fits') out_pdf : str Filename for output PDF. Do NOT include full path stack : boolean Indicate whether to use stacked arc data. Default: False silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- Notes ----- Created by Chun Ly, 18 May 2017 Modified by Chun Ly, 19 May 2017 - Fix bug with scatter plot (wrong indexing) - Fix bug with l_val0 and l_val (those were flipped) - Draw dashed lines connecting the points - Adjust margins, add labels - Adjust image scale (Using zscale) - Annotate each page with the dataframe information - Label lines - Aesthetic changes for plotting - Aesthetic changes for plotting - subplots_adjust - Better labeling of lines - xlim, ylim set - Adjust PDF paper size Modified by Chun Ly, 11 November 2017 - Added stack keyword option to use stacked arc products Modified by Chun Ly, 15 November 2017 - Bug fix: Incorrect replace Modified by Chun Ly, 16 November 2017 - Change prefix: rnc to rbnc Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging ''' # + on 09/01/2018 logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin arc_check : '+systime()) # Mod on 11/11/2017 if stack == True: arcs = ['arc_stack.fits'] # + on 15/11/2017 rnc_files = [path + 'arc_stack.fits'] d_files = [path+'database/idwarc_stack_SCI_1_'] else: if arcs[0] == '': arc_list = path + 'arc.lis' if silent == False: mylogger.info('Reading : '+arc_list) arcs = np.loadtxt(arc_list, dtype=type(str)) # Mod on 16/11/2017 rnc_files = [path+'rbnc'+file0 for file0 in arcs] d_files = [path+'database/idwrbnc'+file0.replace('.fits','_SCI_1_') \ for file0 in arcs] n_arcs = len(rnc_files) # Mod on 11/11/2017 out_pdf = path+'arc_check.pdf' if out_pdf == '' else path+out_pdf pp = PdfPages(out_pdf) for nn in xrange(n_arcs): hdu0 = fits.open(rnc_files[nn]) im0 = hdu0['SCI'].data fig, ax = plt.subplots() z1, z2 = zscale.get_limits(im0) norm = ImageNormalize(vmin=z1, vmax=z2) ax.imshow(im0, cmap='Greys', origin='lower', norm=norm) f0 = open(d_files[nn], 'r') str0 = f0.readlines() beg_mark = [ii for ii in xrange(len(str0)) if 'begin' in str0[ii]] feat_mark = [ii for ii in xrange(len(str0)) if 'features' in str0[ii]] func_mark = [ii for ii in xrange(len(str0)) if 'function' in str0[ii]] n_cols = len(beg_mark) n_features = [np.int(val.split('\t')[2].replace('\n','')) for val in np.array(str0)[feat_mark]] n_max = max(n_features) x_cols, y_val = np.zeros(n_cols), np.zeros((n_cols,n_max)) l_val0, l_val = np.zeros((n_cols,n_max)), np.zeros((n_cols,n_max)) for cc in xrange(n_cols): temp = str0[beg_mark[cc]].split(' ')[1] # Mod on 16/11/2017 temp = temp.replace('wrbnc'+arcs[nn].replace('.fits','[SCI,1]['),'') temp = temp.replace('w'+arcs[nn].replace('.fits','[SCI,1]['),'') # + on 15/11/2017 x_cols[cc] = np.int(temp.split(',')[0]) for rr in xrange(n_features[cc]): #beg_mark[cc]+1, func_mark[cc]): temp1 = str0[feat_mark[cc]+1+rr].replace(' ','') temp1 = temp1.split(' ') temp1 = [val for val in temp1 if val != ''] y_val[cc,rr] = np.float(temp1[1]) try: l_val[cc,rr] = np.float(temp1[2]) except ValueError: pass try: l_val0[cc,rr] = np.float(temp1[3]) except ValueError: pass #endfor x_temp = np.repeat(x_cols[cc],n_features[cc]) y_temp = y_val[cc,0:n_features[cc]] ax.scatter(x_temp, y_temp, facecolor='none', edgecolor='red', marker='o', s=25, alpha=0.5) #endfor line_list = list(set(l_val0.reshape(n_cols*n_max).tolist())) for ll in range(len(line_list)): if line_list[ll] != 0: x_idx, y_idx = np.where(l_val0 == line_list[ll]) sort0 = np.argsort(x_cols[x_idx]) x1 = x_cols[x_idx][sort0] y1 = y_val[x_idx,y_idx][sort0] ax.plot(x1, y1, 'r--', alpha=0.5) ax.annotate(str(line_list[ll]), [690,np.average(y1)], ha='right', va='center', fontsize=10, color='red', bbox=bbox_props) ax.set_xlabel('X [pixels]') ax.set_ylabel('Y [pixels]') ax.set_xlim([0,690]) ax.set_ylim([0,1025]) ax.minorticks_on() subplots_adjust(left=0.095, bottom=0.025, top=0.995, right=0.99) str0 = (rnc_files[nn]+'\n'+d_files[nn]).replace(path,'') ax.annotate(str0, [0.025,0.975], xycoords='axes fraction', ha='left', va='top', fontsize=14, bbox=bbox_props) ax.set_title(path) fig.set_size_inches(7.65,10.5) # Scale proportionally fig.savefig(pp, format='pdf') #endfor pp.close() if silent == False: mylogger.info('### End arc_check : '+systime())
def OH_check(path, objs='', out_pdf='', skysub=False, silent=False, verbose=True, cross_check=False): ''' Generate plot illustrating expected location of OH skyline to check wavelength calibration Parameters ---------- silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True skysub : boolean Display skysubtracted or un-skysubtracted images. Default: False cross_check : boolean Check arc-based wavelength calibration against OH skylines. Default: False Returns ------- Notes ----- Created by Chun Ly, 19 May 2017 Modified by Chun Ly, 20 May 2017 - Draw OH night skyline emission on plots Modified by Chun Ly, 2 June 2017 - Add skysub keyword option to use either sky-subtracted images or those without skysubtraction (OH skylines more visible) Modified by Chun Ly, 26 June 2017 - Plot more OH skylines (change threshold for OH_mark) Modified by Chun Ly, 16 November 2017 - Change prefix: tfrnc to tfrbnc Modified by Chun Ly, 9 January 2018 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 31 May 2018 - Add cross_check keyword; Update input/outputs for cross_check == True - Bug fix: n_obj for cross_check == True Modified by Chun Ly, 10 July 2018 - Change OH skyline file to use the convolved one ''' # + on 09/01/2018 logfile = path+'QA_wave_cal.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin OH_check : '+systime()) # + on 20/05/2017 OH_file = path+'rousselot2000_convl.dat' if exists(OH_file): if silent == False: mylogger.info('Reading : '+OH_file) OH_data = np.loadtxt(OH_file) OH_lines = OH_data[:,0] OH_int = OH_data[:,1] # Mod on 02/06/2017, 31/05/2018 if cross_check == False: if objs == '': obj_list = path + 'obj.lis' if skysub == True else \ path + 'obj.OH.lis' if silent == False: mylogger.info('Reading : '+obj_list) objs = np.loadtxt(obj_list, dtype=type(str)) n_obj = len(objs) tfrnc_files = [path+'tfrbnc'+file0 for file0 in objs] # Mod on 16/11/2017 else: tfrnc_files = [path+'tfOH_stack_arc.fits'] n_obj = len(tfrnc_files) # + on 20/05/2017 chk = [file0 for file0 in tfrnc_files if exists(file0) == True] if len(chk) == 0: mylogger.warn('Files not found!!!') mylogger.warn(', '.join(file0)) mylogger.warn('Exiting!!!') return # Mod on 02/06/2017, 31/05/2018 if cross_check == False: if out_pdf == '': out_pdf = path+'OH_check.pdf' if skysub == True else \ path+'OH_check.raw.pdf' else: out_pdf = path+out_pdf else: if out_pdf == '': out_pdf = path+'OH_check_arc.pdf' else: out_pdf = path+out_pdf pp = PdfPages(out_pdf) for nn in xrange(n_obj): if exists(tfrnc_files[nn]): # Mod on 20/05/2017 hdu0 = fits.open(tfrnc_files[nn]) im0 = hdu0['SCI'].data gc0 = aplpy.FITSFigure(hdu0, hdu='SCI', figsize=(7.65,10.5)) z1, z2 = zscale.get_limits(im0) gc0.show_grayscale(invert=True, vmin=z1, vmax=z2) gc0.set_tick_color('black') # + on 20/05/2017 hdr0 = hdu0['SCI'].header crval2 = hdr0['CRVAL2'] cdelt2 = hdr0['CDELT2'] npix = hdr0['NAXIS2'] lamb_max = crval2 + cdelt2*npix OH_mark = np.where((OH_lines >= crval2) & (OH_lines <= lamb_max))[0] # Mod on 26/06/2017 OH_mark = np.where(OH_int >= 0.005*np.max(OH_int[OH_mark]))[0] line_list = [] for ll in OH_mark: line_list.append(np.array([[0,700], [OH_lines[ll],OH_lines[ll]]])) gc0.show_lines(line_list, color='red', alpha=0.5, linewidth=1.0, linestyle='dashed') #subplots_adjust(left=0.095, bottom=0.025, top=0.995, right=0.99) str0 = tfrnc_files[nn].replace(path,'') gc0.add_label(0.025, 0.975, str0, color='red', relative=True, ha='left', va='top', weight='medium', fontsize=14, bbox=bbox_props) gc0.set_axis_labels(xlabel='X [pixels]', ylabel=r'Wavelength ($\AA$)') #gc0.set_axis_label_rotation(90) gc0.savefig(pp, format='pdf') #endif #endfor if silent == False: mylogger.info('Writing : '+out_pdf) pp.close() if silent == False: mylogger.info('### End OH_check : '+systime())
def main(path0, silent=False, verbose=True, overwrite=False): ''' main() function to sort through data and create individual lists Parameters ---------- path0 : str Path to FITS file. Must include '/' at the end silent : boolean Turns off stdout messages. Default: False verbose : boolean Turns on additional stdout messages. Default: True Returns ------- ASCII files containing the filenames for each set obj.lis - List of science data frames sky.lis - List of science data frames to be used for sky-subtraction arc.lis - List of Arclamp data frames flat.lis - List of Flat data frames telluric.lis - List of Telluric data frames all.lis - List of all obj, sky, arc, flat, and telluric data frames tab0 : astropy.table.Table Astropy ASCII table with QA flag written to [path0]+'hdr_info.QA.tbl' Notes ----- Created by Chun Ly, 5 March 2017 - Later modified to output ASCII table (hdr_info.QA.tbl) containing a column called 'QA' with information of how each FITS file was classified. This is to enable quick check that each dataset if properly classified - Later modified to include all.lis output - Later modified to define r0 (for slight speed improvement Modified by Chun Ly, 23 March 2017 - Call dir_check.main() to handle multiple date directories Modified by Chun Ly, 10 April 2017 - Handle cases when arc, flat, telluric, and sci data are not available Modified by Chun Ly, 11 April 2017 - Handle case when no data for all.lis are available (i.e., the observing sequence was canceled) Modified by Chun Ly, 16 May 2017 - Change obj.lis and sky.lis to include both A-B and B-A sets - Avoid QA flag of 'sky' - Fix minor bug Modified by Chun Ly, 17 May 2017 - Minor fix for i_sky to handle skipping of frames for i_obj Modified by Chun Ly, 31 May 2017 - Minor fix for when no science data is available Modified by Chun Ly, 1 June 2017 - Added overwrite keyword option to overwrite file Modified by Chun Ly, 8 December 2017 - Import glog and call for stdout and ASCII logging Modified by Chun Ly, 10 December 2017 - Adjust glog logging to a default file in path0, move up mylogger definition - Minor mylogger text changes - Pass mylogger to dir_check.main() Modified by Chun Ly, 12 January 2018 - Handle multiple telluric datasets Modified by Chun Ly, 15 January 2018 - Handle multiple telluric datasets (cont'd) - Get indices for each dataset Modified by Chun Ly, 16 January 2018 - Handle multiple telluric datasets (cont'd) - Update prefix and index lists Modified by Chun Ly, 17 January 2018 - Create telluric.lis even for multi-telluric case Modified by Chun Ly, 23 April 2018 - Handle no telluric data case ''' logfile = path0 + 'create_list.log' mylogger = glog.log0(logfile)._get_logger() if silent == False: mylogger.info('### Begin main : ' + systime()) # + on 23/03/2017 dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent, verbose=verbose) # Mod on 23/03/2017 for path in list_path: infile = path + 'hdr_info.tbl' if not exists(infile): mylogger.warning('File does not exist : ' + infile) mylogger.warning('Exiting!!! ' + systime()) return if silent == False: mylogger.info('Reading: ' + infile) tab0 = asc.read(infile, format='fixed_width_two_line') len0 = len(tab0) r0 = xrange(len0) obstype = tab0['obstype'] object0 = tab0['object'] filter2 = tab0['filter2'] i_arc = [ii for ii in r0 if obstype[ii] == 'ARC'] i_flat = [ii for ii in r0 if obstype[ii] == 'FLAT'] i_tell = [ ii for ii in r0 if (obstype[ii] == 'OBJECT' and ( 'HIP' in object0[ii] or 'HD' in object0[ii]) and ( 'H2_' not in filter2[ii] and 'H_' not in filter2[ii])) ] # Identify when multiple sets of telluric data is available # Broken based on filename | + on 12/01/2018, Mod on 15/01/2018 # tel_name = list(set(np.array(object0[i_tell]))) multi_tell = 0 # Mod on 23/04/2018 if len(i_tell) != 0: seq_tell = np.array([ np.int(str0.replace('.fits', '')[-4:]) for str0 in tab0['filename'][i_tell] ]) seq_diff = seq_tell[1:] - seq_tell[0:-1] # +1 to factor first skip for seq_diff seq_break = [ ii + 1 for ii in range(len(seq_diff)) if seq_diff[ii] != 1 ] if len(seq_break) > 0: multi_tell = 1 i_tell0 = [] mylogger.warn('Multiple telluric star detected.') mylogger.warn('Will split into separate files.') ss_start = [0] + seq_break # relative to i_tell ss_end = seq_break + [len(i_tell)] for ss in range(len(ss_start)): ss_idx = i_tell[ss_start[ss]:ss_end[ss]] i_tell0.append(ss_idx) else: mylogger.info('Only one telluric star detected.') i_sci = [ ii for ii in r0 if (obstype[ii] == 'OBJECT' and ( 'HIP' not in object0[ii] and 'HD' not in object0[ii]) and ( 'H2_' not in filter2[ii] and 'H_' not in filter2[ii])) ] i_sci = np.array(i_sci) # Note: Assumes that dithering pattern is ABA'B' # Mod on 16/05/2017 # Mod on 17/05/2017 - Minor bug if things skip for sci frames # Mod on 31/05/2017 - Minor bug if i_sci is empty if len(i_sci) > 0: i_obj = i_sci i_off = [1, -1] * (len(i_sci) / 2) if len(i_sci) % 2 == 1: i_off.append(-1) # Odd number correction i_sky = i_sci[np.arange(len(i_sci)) + np.array(i_off)] # i_sky = [a+b for a,b in zip(i_sci,i_off)] # Mod on 10/04/2017 prefix, index = [], [] if len(i_arc) == 0: mylogger.warn('No ARC data found!') else: prefix.append('arc') index.append(i_arc) if len(i_flat) == 0: mylogger.warn('No FLAT data found!') else: prefix.append('flat') index.append(i_flat) if len(i_tell) == 0: mylogger.warn('No Telluric data found!') else: # Mod on 16/01/2018, 17/01/2018 # Always create telluric.lis for all tellurics prefix.append('telluric') index.append(i_tell) if multi_tell: for ii in range(len(i_tell0)): prefix.append('telluric' + str(ii + 1)) index.append(i_tell0[ii]) if len(i_sci) == 0: mylogger.warn('No science data found!') else: prefix.append('obj') prefix.append('sky') index.append(i_obj) index.append(i_sky) zip0 = zip(prefix, index) QA = ['N/A'] * len0 # Later + on 05/03/2017 for a, b in zip0: if a != 'sky': for idx in b: QA[idx] = a # Mod on 16/05/2017 outfile = path + a + '.lis' if overwrite == False and exists(outfile): mylogger.warn('File exists! Will not write ' + outfile + '!!') else: if silent == False: mylogger.info('Writing : ' + outfile) np.savetxt(outfile, tab0['filename'][b], fmt='%s') #asc.write will not work. Will not produce single column #asc.write(tab0[b], outfile, overwrite=True, # format='no_header') # Later + on 05/03/2017 | Mod on 11/04/2017 i_all = [ii for ii in r0 if QA[ii] != 'N/A'] if len(i_all) > 0: outfile0 = path + 'all.lis' if overwrite == False and exists(outfile0): mylogger.warn('File exists! Will not write all.lis!!') else: if silent == False: mylogger.info('Writing : ' + outfile0) np.savetxt(outfile0, tab0['filename'][i_all], fmt='%s') else: mylogger.warn('Will not write all.lis!!') # Later + on 05/03/2017 col0 = Column(QA, name='QA') tab0.add_column(col0) # Later + on 05/03/2017 outfile2 = infile.replace('.tbl', '.QA.tbl') if silent == False: if overwrite == False and exists(outfile2): mylogger.warn('File exists! Will not write ' + outfile2 + '!!') else: if not exists(outfile2): mylogger.info('Writing : ' + outfile2) else: mylogger.info('Overwriting : ' + outfile2) asc.write(tab0, outfile2, format='fixed_width_two_line', overwrite=True) if silent == False: mylogger.info('### End main : ' + systime())