def run(path0, silent=False, verbose=True):
'''
Main function to run for a given path containing the raw GNIRS data
Parameters
----------
path0 : str
Path to raw FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 22 March 2017
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 8 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to get_files()
Modified by Chun Ly, 7 May 2018
- Bug fix: Call check_path() to add '/' in path0
'''
rawdir = check_path(path0) # + on 07/05/2018
# + on 08/01/2018
logfile = path0+'symlink.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0, mylogger=mylogger,
silent=silent, verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
files, n_files = get_files(path, mylogger=mylogger, silent=silent,
verbose=verbose)
for nn in xrange(n_files):
c_file = path+'c'+files[nn]
if exists(c_file):
mylogger.warn('File exists : '+c_file)
else:
cmd0 = 'ln -fs '+files[nn]+' '+c_file
if silent == False:
mylogger.info(cmd0)
os.system(cmd0)
#endfor
#endfor
if silent == False: mylogger.info('### End run : '+systime())
def main(rawdir, line_source='', mylogger=None, silent=False, verbose=True):
'''
Main function for wave_cal_script
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
line_source : str
Type of lines to use for wavelength calibration. Option is
either 'arc' or 'OH
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 7-8 October 2017
Modified by Chun Ly, 8 November 2017
- Specify lampspec and outspec
Modified by Chun Ly, 16 November 2017
- Handle line_source == 'OH'
- Generalized arrays: arc -> frame
- Specify GNIRS log file
Modified by Chun Ly, 20 November 2017
- Bug fix: prefix needed for [frames] for line_source == OH
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 9 January 2018
- Allow mylogger keyword
Modified by Chun Ly, 30 May 2018
- Change order for nswavelength fitting
Modified by Chun Ly, 16 June 2018
- Change coordlist for OH skylines
Modified by Chun Ly, 19 June 2018
- Force function to legendre
- Set fwidth in nswavelength call to depend on slitwidth
Modified by Chun Ly, 21 June 2018
- Include ending print statement
Modified by Chun Ly, 10 July 2018
- Modify threshold for OH lines from 50 to 25
'''
# Mod on 10/01/2018
if type(mylogger) == type(None):
logfile = rawdir + 'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
timestamp = systime().replace(':', '.')
logfile = rawdir + 'gnirs_' + timestamp + '.log'
rawdir = check_path(rawdir)
if line_source == '':
mylogger.warn("Must specify line_source keyword: ")
mylogger.warn("line_source='arc' or line_source='OH'")
mylogger.warn("Exiting!!!")
return
else:
out_script = rawdir + 'wave_cal_' + line_source + '.py'
if silent == False: mylogger.info('Writing : ' + out_script)
f0 = open(out_script, 'w')
line0 = [
'iraf.gemini(_doprint=0)', 'iraf.gemini.gnirs(_doprint=0)',
'iraf.gemini.unlearn()', 'iraf.gemini.gemtools.unlearn()',
'iraf.gemini.gnirs.unlearn()', 'iraf.set(stdimage="imt4096")',
'iraf.gemini.nsheaders("gnirs")'
]
#'iraf.gemini.gnirs.logfile = "%s"' % logfile] # + on 16/11/2017
f0.writelines("\n".join(line0) + "\n")
if line_source == 'arc': frame_list = rawdir + 'arc.lis'
if line_source == 'OH': frame_list = rawdir + 'obj.OH.lis'
frames = np.loadtxt(frame_list, dtype=type(str))
if line_source == 'OH':
frames = ['rbnc' + file0 for file0 in frames]
frame_hdr = fits.getheader(rawdir + frames[0])
crpix = n_sp_pix / 2.0
crval = frame_hdr['gratwave'] * 1e4 # in Angstroms
if frame_hdr['FILTER2'] == 'X_G0518':
cdelt = -0.094 * 1e4 / n_sp_pix
if frame_hdr['FILTER2'] == 'J_G0517':
cdelt = -0.113 * 1e4 / n_sp_pix
mylogger.info('## CRVAL : %.1f ' % crval)
mylogger.info('## CDELT : %.1f CRPIX : %.1f' % (cdelt, crpix))
line1 = ['crval = %f' % crval, 'crpix = %f' % crpix, 'cdelt = %f' % cdelt]
f0.writelines("\n".join(line1) + "\n")
# Get slit length | + on 19/06/2018q
slitwidth = np.float(frame_hdr['SLIT'].split('arcsec')[0])
pscale = np.sqrt(frame_hdr['CD1_1']**2 +
frame_hdr['CD2_1']**2) * 3600.0 * u.arcsec
fwidth = 1.5 * slitwidth / pscale.to(u.arcsec).value
if line_source == 'arc':
coordlist = 'gnirs$data/argon.dat'
database = 'database/'
threshold = 50
if line_source == 'OH':
coordlist = rawdir + 'rousselot2000_convl.dat'
if not exists(coordlist):
log.warn('File does not exists!!! : ' + coordlist)
database = 'database_OH/'
threshold = 25
# Mod on 08/11/2017
line2 = [
"coordlist = '%s'" % coordlist,
"database = '%s'" % database,
"lampspec = '%s_stack.fits'" % line_source,
"outspec = 'w%s_stack.fits'" % line_source,
"threshold = %f" % threshold,
"logfile = '%s'" % logfile
] # + on 16/11/2017
f0.writelines("\n".join(line2) + "\n")
# Mod on 08/11/2017, 16/11/2017
cmd = "iraf.gnirs.nswavelength(lampspec, outprefix='',"+\
"outspectra=outspec, crval=crval, cdelt=cdelt, crpix=crpix, "+\
"coordlist=coordlist, database=database, fl_inter='yes', "+\
"function='legendre', cradius=20, threshold=threshold, "+\
"fwidth=%.1f, " % fwidth +"order=3, logfile=logfile)"
f0.write(cmd + '\n')
f0.write('print("Completed! Return to other terminal and hit RETURN")\n')
f0.close()
if silent == False: mylogger.info('### End main : ' + systime())
def main():
args1 = [('-D', '--DIR'), ('-O', '--OUT')]
args2 = [('-g', '--geno'), ('-s', '--snps'), ('-o', '--out'),
('-n', '--samples'), ('-t', '--type'), ('-a', '--alleles')]
if len(sys.argv) == 1:
from parse_args import msg
print(msg())
del msg
raise SystemExit
elif sys.argv[1] == '-h' or sys.argv[1] == '--help':
import geno2fi
geno2fi
del geno2fi
raise SystemExit
elif any(x in sys.argv[1:] for x in list(itertools.chain(*args1))):
x = set([item for item in args1 for a in sys.argv[1:] if a in item])
y = set(
[item for item in args1 for a in sys.argv[1:] if a not in item])
z = list(x.symmetric_difference(y))
if z:
bomb(f'Missing argument when trying to convert to fimpute:\n'
f' required args: {z}\n'
f' try `./snprecode -h` for complete arguments list\n')
from check_path import check_path
if not check_path():
print("File path OK...")
else:
print(check_path())
del check_path
from garbage import check_dups, file_list
if not check_dups():
print("File check complete...\nFiles to be processed...\n")
print('\n'.join(map(str, file_list)))
del check_dups
import geno2fi
geno2fi
del geno2fi
raise SystemExit
elif any(x in sys.argv[1:] for x in list(itertools.chain(*args2))):
x = set([item for item in args2 for a in sys.argv[1:] if a in item])
y = set(
[item for item in args2 for a in sys.argv[1:] if a not in item])
z = list(x.symmetric_difference(y))
if z:
bomb(f"Missing argument when trying to convert from fimpute\n"
f" required args: {z}\n"
f" try: `./snprecode -h` for complete arguments list\n")
import fi2geno
fi2geno
del fi2geno
raise SystemExit
elif [item for item in sys.argv[1:] if item.endswith(('bim', 'map'))]:
import snpinfo
snpinfo
del snpinfo
raise SystemExit
elif [item for item in sys.argv[1:] if item.endswith(('vcf', 'vcf.gz'))]:
import geno_corr
geno_corr
del geno_corr
raise SystemExit
else:
bomb('Unknown argument(s)\n try ./snprecode -h')
def check_sections(self, parser):
sectionfound = False
for section in parser.sections():
if section == "general":
continue
sectionfound = True
self.section.append(section)
temppath = check_path(parser, section, self.debug)
if not temppath:
self.path.append(None)
self.errorsoccured = True
else:
self.path.append(temppath)
tempowner = check_ownerandgroup(parser, section, "owner", self.debug)
if tempowner == False:
self.owner.append(None)
self.errorsoccured = True
else:
self.owner.append(tempowner)
tempgroup = check_ownerandgroup(parser, section, "group", self.debug)
if tempgroup == False:
self.group.append(None)
self.errorsoccured = True
else:
self.group.append(tempgroup)
tempchmodf = check_chmod(parser, section, "chmodf", self.debug)
if tempchmodf == False:
self.chmodf.append(None)
self.errorsoccured = True
else:
self.chmodf.append(tempchmodf)
tempchmodd = check_chmod(parser, section, "chmodd", self.debug)
if tempchmodd == False:
self.chmodd.append(None)
self.errorsoccured = True
else:
self.chmodd.append(tempchmodd)
tempexcludepath = check_excludepath(parser, section, self.path[-1], self.debug)
if tempexcludepath == False:
self.excludepath.append(None)
self.errorsoccured = True
else:
self.excludepath.append(tempexcludepath)
tempexcluderegex = check_excluderegex(parser, section, self.debug)
if tempexcluderegex == False:
self.excluderegex.append(None)
self.errorsoccured = True
else:
self.excluderegex.append(tempexcluderegex)
if not self.check_final():
print >>sys.stderr, ("Error in section '%s': No actions are " "configured to be performed." % section)
self.errorsoccured = True
return sectionfound
def plot_spec(rawdir, out_pdf='', silent=False, verbose=False):
'''
Plot median spectrum of OH skylines
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Produces 'OH_spec.pdf'
Notes
-----
Created by Chun Ly, 4 July 2017
- Overlay Rousselot (2000) model spectrum
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 9 January 2018
- Implement glog stdout and ASCII logging
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
rawdir = check_path(rawdir) # + on 20/09/2017
out_pdf = rawdir+'OH_spec.pdf' if out_pdf == '' else rawdir+out_pdf
im0, hdr = fits.getdata(rawdir+'tfOH_stack.fits', extname='SCI',
header=True)
OH_med0 = np.median(im0, axis=1)
crval2 = hdr['CRVAL2']
cd2_2 = hdr['CD2_2']
crpix = hdr['CRPIX2']
x0 = crval2 + cd2_2*np.arange(len(OH_med0))
x0 /= 1e4 # In microns
y0 = OH_med0/max(OH_med0)
fig, ax = plt.subplots()
if exists(OH_file):
if silent == False: mylogger.info('Reading : '+OH_file)
OH_data = np.loadtxt(OH_file)
OH_lines = OH_data[:,0] / 1E4
OH_int = OH_data[:,1]
i_max = np.where(OH_med0 == max(OH_med0))[0]
l_max = x0[i_max[0]]
p0 = [0.0, max(y0), l_max, 2.0/1e4]
popt, pcov = curve_fit(gauss1d, x0, y0, p0=p0)
fwhm = popt[3]*2*np.sqrt(2*np.log(2))
R_spec = l_max / fwhm
OH_spec_mod = np.zeros(len(x0))
in_rge = np.where((OH_lines >= x0[0]) & (OH_lines <= x0[-1]))[0]
for ii in range(len(in_rge)):
idx = in_rge[ii]
#ax.plot(np.repeat(OH_lines[in_rge[ii]],2), [0,1.1], 'r--')
temp = OH_int[idx] * gaussian_R(x0, OH_lines[idx], R_spec)
OH_spec_mod += temp
ax.plot(x0, y0, 'k-', label='OH_stack')
OH_spec_mod /= np.max(OH_spec_mod)
ax.plot(x0, OH_spec_mod, 'r--', alpha=0.75,
label='OH model (Rousselot 2000)')
ax.set_xlabel(r'Wavelength ($\mu$m)', fontsize=16)
ax.set_ylabel('Normalized Flux', fontsize=16)
ax.set_ylim([0.0, 1.10])
ax.minorticks_on()
ax.tick_params(labelsize=14)
subplots_adjust(left=0.07, right=0.99, bottom=0.07, top=0.99)
ax.legend(loc='upper right', fontsize=14, frameon=False)
fig.set_size_inches(11,8)
fig.savefig(out_pdf)
def run(rawdir, silent=False, verbose=False):
'''
Main function to combine science data to produce a 2-D FITS image
containing OH night skylines for wavelength calibration. Median
filtering is used to remove detected objects
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image containing OH skyline called 'OH_stack.fits'
Notes
-----
Created by Chun Ly, 25 June 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 16 November 2017
- Change prefix: rnc to rbnc
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
rawdir = check_path(rawdir) # + on 20/09/2017
obj_list = rawdir + 'obj.lis'
if silent == False: mylogger.info('Reading : '+obj_list)
objs = np.loadtxt(obj_list, dtype=type(str))
rbnc_files = [rawdir+'rbnc'+file0.replace('.fits','.OH.fits') for
file0 in objs] # Mod on 16/11/2017
# Mod on 16/11/2017
hdu0 = fits.open(rbnc_files[0])
hdr = fits.getheader(rbnc_files[0], extname='SCI')
naxis1 = hdr['NAXIS1']
naxis2 = hdr['NAXIS2']
arr0 = np.zeros((len(rbnc_files), naxis2, naxis1)) # Mod on 16/11/2017
for ii in range(len(rbnc_files)):
if verbose == True: mylogger.info('Reading : '+obj_list)
t_data = fits.getdata(rbnc_files[ii], extname='SCI') # Mod on 16/11/2017
t_med0 = np.median(t_data, axis=0) # Median along columns
# Remove median along columns
med_off = np.repeat(t_med0, naxis2).reshape(naxis1,naxis2).transpose()
arr0[ii] = t_data - med_off
# Compute median
med_arr0 = np.median(arr0, axis=0)
hdu0['SCI'].data = med_arr0
hdu0['VAR'].data = np.zeros((naxis2,naxis1))
outfile = rawdir+'OH_stack.fits'
stat0 = 'Overwriting' if exists(outfile) else 'Writing'
if silent == False:
mylogger.info(stat0+' : '+outfile)
hdu0.writeto(outfile, output_verify='ignore', overwrite=True)
if silent == False: mylogger.info('### End run : '+systime())
def wave_cal(rawdir, cdir, silent=False, verbose=False):
'''
Run gnirs.nswavelength on OH_stack 2-D FITS image for wavelength
calibration
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits'
Notes
-----
Created by Chun Ly, 13 July 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 16 November 2017
- Call wave_cal_script to get PyRAF code
Modified by Chun Ly, 16 November 2017
- Bug fix: indentation typo with else statement
Modified by Chun Ly, 20 November 2017
- Bug fix: Pass in cdir
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 14 June 2018
- Import and call get_OH_centers
Modified by Chun Ly, 19 June 2018
- Pass gnirs logfile to mylogger
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
rawdir = check_path(rawdir) # + on 20/09/2017
iraf.chdir(rawdir)
timestamp = systime().replace(':','.')
logfile = rawdir+'gnirs_'+timestamp+'.log'
iraf.gemini.gnirs.logfile = logfile
mylogger.info("GNIRS logfile : "+logfile)
get_OH_centers.main(rawdir)
# + on 16/11/2017
script_file = 'wave_cal_OH.py'
if not exists(script_file):
wave_cal_script.main(rawdir, line_source='OH')
else:
# Mod on 09/01/2018
mylogger.info('File exists!!! : '+script_file)
mylogger.info('Will not override!!!')
# + on 16/11/2017
do_run = 0
if not exists('wOH_stack.fits'): do_run = 1
if do_run:
# Mod on 09/01/2018
mylogger.info("In order to perform interactive calibration, open up")
mylogger.info("a PyRAF terminal in an anaconda IRAF environment")
mylogger.info("'cd' into "+rawdir)
mylogger.info("Execute the following command :")
mylogger.info("execfile('"+script_file+"')")
t_out = raw_input("## Hit RETURN when OH wavelength calibration is completed")
else:
# Mod on 09/01/2018
mylogger.warn('Files exist!!!')
mylogger.warn('Will not run nswavelength on OH stacked data')
iraf.chdir(cdir)
def transform(rawdir, silent=False, verbose=False):
'''
Transform OH_stack 2-D FITS image for wavelength calibration checks
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits'
Notes
-----
Created by Chun Ly, 4 July 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 22 November 2017
- Add file checking and log.warn calls
Modified by Chun Ly, 29 November 2017
- Bug fix: Missing else statement
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 19 June 2018
- Call QA_wave_cal.get_database_model
- Pass function and order to nsfitcoords for OH stack
Modified by Chun Ly, 20 June 2018
- Set nsfitcoords xorder fitting
Modified by Chun Ly, 22 June 2018
- Call residual_wave_cal for wavelength solution check
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
cdir = os.getcwd()+'/' # + on 06/05/2017
rawdir = check_path(rawdir) # + on 20/09/2017
iraf.chdir(rawdir)
mylogger.info("Running nsfitcoords on OH_stack") # Mod on 09/01/2018
outfile1 = rawdir + 'fOH_stack.fits'
if not exists(outfile1):
func0, order0 = QA_wave_cal.get_database_model(rawdir, 'OH')
iraf.gnirs.nsfitcoords('wOH_stack.fits', outprefix='',
outspectra='fOH_stack.fits',
lamp='wOH_stack.fits', database='database_OH/',
function=func0, lyorder=order0,
lxorder=QA_wave_cal.xorder)
else:
# Mod on 09/01/2018
mylogger.warn('File exists!!! : '+outfile1)
mylogger.warn('Will not run nsfitcoords on OH stacked data')
outfile2 = rawdir + 'tfOH_stack.fits'
if not exists(outfile2):
iraf.gnirs.nstransform('fOH_stack.fits', outprefix='',
outspectra='tfOH_stack.fits',
database='database_OH/')
else:
# Mod on 09/01/2018
mylogger.warn('File exists!!! : '+outfile2)
mylogger.warn('Will not run nstransform on OH stacked data')
iraf.chdir(cdir)
# + on 22/06/2018
QA_wave_cal.residual_wave_cal(rawdir, dataset='OH', cal='OH')
def main(rawdir, out_pdf='', silent=False, verbose=True):
'''
Main function for OH_flat_check module
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Produces 'OH_flat_check.pdf'
Notes
-----
Created by Chun Ly, 8 September 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
'''
if silent == False: log.info('### Begin main : ' + systime())
rawdir = check_path(rawdir) # + on 20/09/2017
out_pdf = rawdir + 'OH_flat_check.pdf' if out_pdf == '' else rawdir + out_pdf
infile = rawdir + 'OH_stack.fits'
if silent == False: log.info('### Reading : ' + infile)
im0, hdr = fits.getdata(infile, extname='SCI', header=True)
fig, ax = plt.subplots()
cen = np.int(np.floor(hdr['NAXIS1'] / 2))
mid_spec = np.median(im0[:, cen - 5:cen + 5], axis=1)
mid_spec /= np.max(mid_spec)
med0 = np.median(mid_spec)
mid_pind0 = np.array(find_peaks_cwt(mid_spec, np.arange(1, 3)))
mid_pind = mid_pind0[np.where(mid_spec[mid_pind0] - med0 > 0.05)[0]]
n_peaks = len(mid_pind)
med_spec = np.zeros((hdr['NAXIS1'], n_peaks))
for ii in range(n_peaks):
temp = im0[mid_pind[ii] - 9:mid_pind[ii] + 9, :]
t_spec = np.median(temp, axis=0)
mean, median0, std = sigma_clipped_stats(t_spec[250:300],
sigma=2.0,
iters=5)
t_spec /= median0 #np.max(t_spec)
med_spec[:, ii] = t_spec
#plt.plot(t_spec, color='black', linewidth=0.25)
plt.plot(np.median(med_spec, axis=1), color='black', linewidth=1) #0.25)
ax.set_xlabel('Spatial X [pixel]', fontsize=16)
ax.set_ylabel('Normalized Flux', fontsize=16)
ax.set_ylim([-0.25, 1.5])
ax.minorticks_on()
ax.tick_params(labelsize=14)
fig.set_size_inches(11, 8)
if silent == False: log.info('### Writing ' + out_pdf)
fig.savefig(out_pdf, bbox_inches='tight')
if silent == False: log.info('### End main : ' + systime())
def run(rawdir, style, mylogger=None, silent=False, verbose=True):
'''
Main function to generate median plots for various steps in the data
reduction
Parameters
----------
rawdir : str
Path to raw files.
style : str
Type of data to plot. Options are:
'orig': For original data (before any bias removal) 'ncN' files
'bias': After bias removal 'bncN' files
'flat': For flattened data 'rbncN' (Will use OH frames since those
are not sky subtracted)
'skysub': For flattened and skysub data 'rbncN'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 14 September 2017
Modified by Chun Ly, 15 September 2017
- Add if statements for style = 'flat'
- Add if statements for style = 'skysub'
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 18 December 2017
- Implement glog logging, allow mylogger keyword input
Modified by Chun Ly, 29 April 2018
- Compute median of background level
- Remove median from images, plot improved skysubtraction
Modified by Chun Ly, 30 April 2018
- Plot aesthetics (axis labels, borders)
- Write median-remove skysub images
- Modify plotting to handle style != 'skysub' (single panel)
Modified by Chun Ly, 5 May 2018
- Write skysub statistics ASCII table
- Fix skysub stats ASCII filename
Modified by Chun Ly, 8 May 2018
- Force legend in upper right corner
- Check if median skysub removal was performed (delete rnbc*.v1.fits.gz
files to rerun everything)
- Minor fix for different style settings
Modified by Chun Ly, 21 May 2018
- Bug fix: compute median from 2-D image, not column median
'''
# + on 18/12/2017
if type(mylogger) == type(None):
mylog, clog = 0, log
else:
mylog, clog = 1, mylogger
rawdir = check_path(rawdir) # + on 20/09/2017
if style == 'orig': prefix = 'nc'
if style == 'bias': prefix = 'bnc'
if style == 'flat': prefix = 'rbnc' # + on 15/09/2017
if style == 'skysub': prefix = 'rbnc' # + on 15/09/2017
if silent == False: clog.info('### Begin run : '+systime())
infile = rawdir+'obj.lis'
if silent == False: clog.info('Reading : '+infile)
files1 = np.loadtxt(infile, dtype=type(str)).tolist()
files0 = [rawdir+prefix+file1 for file1 in files1]
# + on 15/09/2017
if style == 'flat':
files0 = [file0.replace('.fits','.OH.fits') for file0 in files0]
if style == 'skysub':
fig, ax_arr = plt.subplots(nrows=2)
else:
fig, ax_arr = plt.subplots()
if style == 'skysub': # + on 05/05/2018
arr_mean = np.zeros(len(files0))
arr_med = np.zeros(len(files0))
arr_sig = np.zeros(len(files0))
for ii in range(len(files0)):
# Mod on 08/05/2018
if style == 'skysub':
out_file = files0[ii].replace('.fits','.v1.fits.gz')
if exists(out_file):
t_file = out_file.replace(rawdir+prefix,'')
clog.info('Skysub median removed. Reading original skysub file : '+t_file)
hdu0 = fits.open(out_file) # Mod on 30/04/2018
else:
hdu0 = fits.open(files0[ii]) # Mod on 30/04/2018
else:
hdu0 = fits.open(files0[ii]) # Mod on 30/04/2018
im0 = hdu0['SCI'].data
label0 = files0[ii].replace(rawdir,'')
med0 = np.median(im0, axis=0)
if style == 'skysub':
ax_arr[0].plot(med0, label=label0, linewidth=0.5)
else:
ax_arr.plot(med0, label=label0, linewidth=0.5)
if style == 'skysub': # + on 29/04/2018
c_mean0, c_med0, c_sig0 = sigma_clipped_stats(im0, sigma=2.0,
iters=10) # Mod on 21/05/2018
ax_arr[0].axhline(y=c_med0, color='black', linestyle='--',
linewidth=0.5)
# + on 05/05/2018
arr_mean[ii] = c_mean0
arr_med[ii] = c_med0
arr_sig[ii] = c_sig0
im1 = im0 - c_med0
med0 = np.median(im1, axis=0)
ax_arr[1].plot(med0, label=label0, linewidth=0.5)
# Save gnirs skysub image with v1 suffix | + on 30/04/2018
clog.info('Saving gnirs skysub file : '+out_file)
hdu0.writeto(out_file, overwrite=True)
# Write median-removed skysub image | + on 30/04/2018
hdu1 = hdu0
hdu1['SCI'].data = im1
clog.info('Write median-removed skysub file : '+files0[ii])
hdu1.writeto(files0[ii], overwrite=True)
#endif
#endfor
if style == 'skysub':
ax_arr[1].axhline(y=0, color='black', linestyle='--', linewidth=0.5)
ax_arr[0].legend(fontsize=6, loc='upper right', frameon=False)
ax_arr[1].legend(fontsize=6, loc='upper right', frameon=False)
ax_arr[0].xaxis.set_ticklabels([])
ax_arr[1].set_xlabel('X [pixels]')
ax_arr[0].set_ylabel('Flux')
ax_arr[1].set_ylabel('Flux')
# + on 05/05/2018
t_arr0 = [files1, arr_mean, arr_med, arr_sig]
tab0 = Table(t_arr0, names=('File','sky_mean','sky_med','sky_sig'))
sky_stats_outfile = rawdir+'sky_stats.tbl'
if silent == False:
clog.info('Writing : '+sky_stats_outfile)
asc.write(tab0, sky_stats_outfile, format='fixed_width_two_line')
else:
ax_arr.legend(fontsize=6, loc='upper right', frameon=False)
ax_arr.set_xlabel('X [pixels]')
ax_arr.set_ylabel('Flux')
plt.subplots_adjust(left=0.1, right=0.99, bottom=0.1, top=0.99, hspace=0.0)
out_pdf = rawdir+'median_plot_'+style+'.pdf'
if silent == False: clog.info('Writing : '+out_pdf)
fig.savefig(out_pdf)
if silent == False: clog.info('### End run : '+systime())
return goalPath.path
# Tests
if(True):
if(False):
# SIMPLE
environment = Environment('simple.yaml')
radius = 0.3
bounds = (-2, -3, 12, 8)
start = (0, 0)
goal_region = Polygon([(10,5), (10,6), (11,6), (11,5)])
path = rrt(bounds, environment, start, radius, goal_region)
check_path(path, bounds, environment, start, radius, goal_region)
if(True):
# BUGTRAP
environment = Environment('bugtrap.yaml')
radius = 0.3
bounds = (-2, -3, 12, 8)
start = (2, 2.5)
goal_region = Polygon([(10,5), (10,6), (11,6), (11,5)])
path = rrt(bounds, environment, start, radius, goal_region)
#check_path(path, bounds, environment, start, radius, goal_region)
# DIFFERENT RANDOM ENVIRONMENTS
start = (-1 , -1)
radius = 0.1
return goalPath.path
# Tests
if (True):
if (True):
# BUGTRAP
environment = Environment('bugtrap.yaml')
radius = 0.3
bounds = (-2, -3, 12, 8)
start = (2, 2.5)
goal_region = Polygon([(10, 5), (10, 6), (11, 6), (11, 5)])
path = rrt(bounds, environment, start, radius, goal_region)
# RANDOM ENVIRONMENTS
start = (-1, -1)
radius = 0.1
bounds = (-5, -4, 15, 5)
goal_region = Polygon([(12, 3), (12, 4), (13, 4), (13, 3)])
envMed = random_environment(bounds, start, radius, goal_region, 40,
(0.2, 0.4))
# Test in different environments
if (False):
print("Test medium environment")
path = rrt(bounds, envMed, start, radius, goal_region)
check_path(path, bounds, envMed, start, radius, goal_region)
plt.show() if True else print("Finished. Chosen to show no plots")
def run(rawdir, files0=[''], mylogger=None, silent=False, verbose=False):
'''
Main function to execute removal of bias in individual GNIRS images
Parameters
----------
rawdir : str
Path to raw files.
files0 : list. Optional
Name of files. No need to specify full path as rawdir is used
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 14 September 2017
- Change files0 to not include full path
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 18 December 2017
- Implement glog logging, allow mylogger keyword input
- Fix indentation for End info log
Modified by Chun Ly, 23 April 2018
- Write tab_outfile only once (outside for loop)
'''
# + on 18/12/2017
if type(mylogger) == type(None):
mylog, clog = 0, log
else:
mylog, clog = 1, mylogger
if silent == False: clog.info('### Begin run : ' + systime())
rawdir = check_path(rawdir) # + on 20/09/2017
if files0[0] == '':
files0 = glob.glob(rawdir + 'ncN????????S????.fits')
n_files0 = len(files0)
bias_offset0 = np.zeros(n_files0)
bias_offset1 = np.zeros(n_files0)
bias_offset2 = np.zeros(n_files0)
bias_offset3 = np.zeros(n_files0)
for nn in range(n_files0):
outfile = rawdir + files0[nn].replace('ncN',
'bncN') # Mod on 14/09/2017
if exists(outfile):
clog.warn('Will not overwrite file : ' + outfile)
else:
if verbose == True: clog.info('Reading : ' + outfile)
hdu = fits.open(rawdir + files0[nn]) # Mod on 14/09/2017
if ('BIAS_FIX' in hdu['SCI'].header) == False:
naxis1 = hdu['SCI'].header['NAXIS1']
naxis2 = hdu['SCI'].header['NAXIS2']
qxsize = naxis1 / 2
qysize = naxis2 / 2
im0 = hdu['SCI'].data
q1x1, q1x2, q1y1, q1y2 = 0, 160, qysize, naxis2
q2x1, q2x2, q2y1, q2y2 = 864, naxis1, qysize, naxis2
q3x1, q3x2, q3y1, q3y2 = 0, 160, 0, qysize
q4x1, q4x2, q4y1, q4y2 = 864, naxis1, 0, qysize
bias_UL = np.median(im0[q1y1:q1y2, q1x1:q1x2])
bias_UR = np.median(im0[q2y1:q2y2, q2x1:q2x2])
bias_LL = np.median(im0[q3y1:q3y2, q3x1:q3x2])
bias_LR = np.median(im0[q4y1:q4y2, q4x1:q4x2])
bias_offset0[nn] = bias_UL
bias_offset1[nn] = bias_UR
bias_offset2[nn] = bias_LL
bias_offset3[nn] = bias_LR
im0[qysize:naxis2, 0:qxsize] -= bias_UL
im0[qysize:naxis2, qxsize:naxis1] -= bias_UR
im0[0:qysize, 0:qxsize] -= bias_LL
im0[0:qysize, qxsize:naxis1] -= bias_LR
if verbose == True: clog.info('Writing : ' + outfile)
hdu['SCI'].data = im0
hdu['SCI'].header['BIAS_FIX'] = 'Yes'
hdu.writeto(outfile, overwrite=True, output_verify='ignore')
#endif
#endelse
#endfor
# Mod on 23/04/2018
arr0 = [files0, bias_offset0, bias_offset1, bias_offset2, bias_offset3]
names0 = names = ('filename', 'bias_UL', 'bias_UR', 'bias_LL', 'bias_LR')
tab0 = Table(arr0, names=names0)
# Mod on 23/04/2018
tab_outfile = rawdir + 'bias_levels.tbl'
if silent == False: clog.info('Writing : ' + tab_outfile)
asc.write(tab0, tab_outfile, format='fixed_width_two_line')
if silent == False: clog.info('### End run : ' + systime())