def update_reffile_keywords(input_dir, input_list, filetype):
'''
This function takes the files in input_dir/input_flist and updates
the reference files associated with filetype
Inputs:
input_dir: directory of files whose headers you want to update
input_list: names of files whose headers you want to update
filetype: either drk or bia - the reference file whose name you want to update
Outputs:
modifies files in input_flist
'''
#Get dates for reference file application as a function of observation mode
mode_dict = determine_correct_reference_files(input_dir, input_list, filetype)
#Determine which keyword to update
assert (filetype is 'bia') or (filetype is 'drk'), 'ERROR: Unknown filetype %s, please use bia or drk' %(filetype)
filetype_dict = {'bia':'BIASFILE', 'drk':'DARKFILE'}
keyword = filetype_dict[filetype]
#Update headers
for ifile in input_list:
hdr0 = pyfits.getheader(os.path.join(input_dir, ifile), 0)
gain = hdr0['ccdgain']
binaxis1 = hdr0['binaxis1']
binaxis2 = hdr0['binaxis2']
expstart = hdr0['texpstrt']
useafter_dates = mode_dict[gain][binaxis1][binaxis2].keys()
useafter_dates.sort()
useafter_dates = np.array(useafter_dates)
date_diff = expstart - useafter_dates
date_indx = np.where((date_diff > 0))
reffile = mode_dict[gain][binaxis1][binaxis2][useafter_dates[date_indx[0][-1]]]
pyfits.setval(os.path.join(input_dir, ifile), keyword, value = reffile, ext = 0)
def headerRenameKey2(self, key, newkey):
for filename in self._matchingFiles:
self._logger.notice('rename %s: %s= %s' % (filename, key, newkey))
try:
value = pyfits.getval(filename, key)
pyfits.setval(filename, newkey, value)
pyfits.delval(filename, key)
except:
pass
def run_cte_correction_code(input_dir, input_flist):
'''
This function will run the CTE correction code StisPixCteCorr.py on each file in
input_flist
'''
for ifile in input_flist:
filename = os.path.join(input_dir, ifile)
StisPixCteCorr.CteCorr(filename.replace('raw', 'flt'),outFits = filename.replace('raw', 'flc') )
pyfits.setval(filename.replace('raw', 'flc'), 'CTECORR', ext = 0, value = 'COMPLETE')
def background(fs=None):
iraf.cd('work')
# Get rectified science images
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images for background-subtraction."
iraf.cd('..')
return
if not os.path.exists('bkg'):
os.mkdir('bkg')
for f in fs:
print("Subtracting background for %s" % f)
# Make sure dispaxis is set correctly
pyfits.setval(f, 'DISPAXIS', value=1)
# the outfile name is very similar, just change folder prefix and
# 3-char stage substring
outfile = f.replace('nrm','bkg')
# We are going to use fit1d instead of the background task
# Go look at the code for the background task: it is literally a wrapper for 1D
# but it removes the BPM option. Annoying.
iraf.unlearn(iraf.fit1d)
iraf.fit1d(input=f + '[SCI]', output='tmpbkg.fits', bpm=f + '[BPM]',
type='difference', sample='52:949', axis=2,
interactive='no', naverage='1', function='legendre',
order=5, low_reject=1.0, high_reject=1.0, niterate=5,
grow=0.0, mode='hl')
# Copy the background subtracted frame into the rectified image
# structure.
# Save the sky spectrum as extension 3
hdutmp = pyfits.open('tmpbkg.fits')
hdu = pyfits.open(f)
skydata = hdu[1].data - hdutmp[0].data
hdu[1].data[:, :] = hdutmp[0].data[:, :]
hdu.append(pyfits.ImageHDU(skydata))
hdu[3].header['EXTNAME'] = 'SKY'
hdu[3].data[hdu['BPM'] == 1] = 0.0
# Add back in the median sky level for things like apall and lacosmicx
hdu[1].data[:, :] += np.median(skydata)
hdu[1].data[hdu['BPM'] == 1] = 0.0
hdutmp.close()
hdu.writeto(outfile, clobber=True) # saving the updated file
# (data changed)
os.remove('tmpbkg.fits')
iraf.cd('..')
def background(fs=None):
iraf.cd('work')
# Get rectified science images
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images for background-subtraction."
iraf.cd('..')
return
if not os.path.exists('bkg'):
os.mkdir('bkg')
for f in fs:
print("Subtracting background for %s" % f)
# Make sure dispaxis is set correctly
pyfits.setval(f, 'DISPAXIS', value=1)
# the outfile name is very similar, just change folder prefix and
# 3-char stage substring
outfile = f.replace('nrm','bkg')
# We are going to use fit1d instead of the background task
# Go look at the code for the background task: it is literally a wrapper for 1D
# but it removes the BPM option. Annoying.
iraf.unlearn(iraf.fit1d)
iraf.fit1d(input=f + '[SCI]', output='tmpbkg.fits', bpm=f + '[BPM]',
type='difference', sample='52:949', axis=2,
interactive='no', naverage='1', function='legendre',
order=5, low_reject=1.0, high_reject=1.0, niterate=5,
grow=0.0, mode='hl')
# Copy the background subtracted frame into the rectified image
# structure.
# Save the sky spectrum as extension 3
hdutmp = pyfits.open('tmpbkg.fits')
hdu = pyfits.open(f)
skydata = hdu[1].data - hdutmp[0].data
hdu[1].data[:, :] = hdutmp[0].data[:, :]
hdu.append(pyfits.ImageHDU(skydata))
hdu[3].header['EXTNAME'] = 'SKY'
hdu[3].data[hdu['BPM'] == 1] = 0.0
# Add back in the median sky level for things like apall and lacosmicx
hdu[1].data[:, :] += np.median(skydata)
hdu[1].data[hdu['BPM'] == 1] = 0.0
hdutmp.close()
hdu.writeto(outfile, clobber=True) # saving the updated file
# (data changed)
os.remove('tmpbkg.fits')
iraf.cd('..')
def run_cte_correction_code(input_dir, input_flist):
'''
This function will run the CTE correction code StisPixCteCorr.py on each file in
input_flist
'''
for ifile in input_flist:
filename = os.path.join(input_dir, ifile)
StisPixCteCorr.CteCorr(filename.replace('raw', 'flt'),
outFits=filename.replace('raw', 'flc'))
pyfits.setval(filename.replace('raw', 'flc'),
'CTECORR',
ext=0,
value='COMPLETE')
def update_reffile_keywords(input_dir, input_list, filetype):
'''
This function takes the files in input_dir/input_flist and updates
the reference files associated with filetype
Inputs:
input_dir: directory of files whose headers you want to update
input_list: names of files whose headers you want to update
filetype: either drk or bia - the reference file whose name you want to update
Outputs:
modifies files in input_flist
'''
#Get dates for reference file application as a function of observation mode
mode_dict = determine_correct_reference_files(input_dir, input_list,
filetype)
#Determine which keyword to update
assert (filetype is 'bia') or (
filetype is 'drk'
), 'ERROR: Unknown filetype %s, please use bia or drk' % (filetype)
filetype_dict = {'bia': 'BIASFILE', 'drk': 'DARKFILE'}
keyword = filetype_dict[filetype]
#Update headers
for ifile in input_list:
hdr0 = pyfits.getheader(os.path.join(input_dir, ifile), 0)
gain = hdr0['ccdgain']
binaxis1 = hdr0['binaxis1']
binaxis2 = hdr0['binaxis2']
expstart = hdr0['texpstrt']
useafter_dates = mode_dict[gain][binaxis1][binaxis2].keys()
useafter_dates.sort()
useafter_dates = np.array(useafter_dates)
date_diff = expstart - useafter_dates
date_indx = np.where((date_diff > 0))
reffile = mode_dict[gain][binaxis1][binaxis2][useafter_dates[
date_indx[0][-1]]]
pyfits.setval(os.path.join(input_dir, ifile),
keyword,
value=reffile,
ext=0)
def test_gsag_calibration(gsagtab):
"""Move gsagtab into TEST_DIR and calibrate with CalCOS.
Any datasets that fail calibration will be emailed to the user.
"""
print '#-------------------------#'
print 'Calibrating with %s'%(gsagtab)
print '#-------------------------#'
os.environ['lref'] = '/grp/hst/cdbs/lref/'
os.environ['testdir'] = TEST_DIR
if not os.path.exists(TEST_DIR):
os.mkdir(TEST_DIR)
shutil.copy( gsagtab ,os.path.join(TEST_DIR,'new_gsag.fits') )
test_datasets = glob.glob( os.path.join(TEST_DIR, '*rawtag_a.fits') )
#Remove products
for ext in ('*_counts*.fits','*_flt*.fits','*_x1d*.fits','*lampflash*.fits','*corrtag*.fits'):
os.system('rm '+TEST_DIR+'/'+ext)
for item in test_datasets:
pyfits.setval( item,'RANDSEED',value=8675309,ext=0 )
pyfits.setval( item,'GSAGTAB',value='testdir$new_gsag.fits',ext=0 )
failed_runs = []
for item in test_datasets:
try:
status = calcos.calcos( item,outdir=TEST_DIR )
print "CalCOS exit status is",status
except:
failed_runs.append( item )
if status != 0:
failed_runs.append( item )
if len(failed_runs):
send_email(subject='GSAGTAB Calibration Error',message='Failed calibration\n\n'+'\n'+'\n'.join(failed_runs) )
def updatecomheader(extractedfiles, objname):
airmasses = []
exptimes = []
for f in extractedfiles:
airmasses.append(float(pyfits.getval(f, 'AIRMASS')))
exptimes.append(float(pyfits.getval(f, 'EXPTIME')))
pyfits.setval(objname + '_com.fits', 'AIRMASS', value=np.mean(airmasses))
pyfits.setval(objname + '_com.fits', 'SLIT', value=pyfits.getval(extractedfiles[0], 'MASKNAME').replace('arcsec', ''))
comhdu = pyfits.open(objname + '_com.fits', mode='update')
extractedhdu = pyfits.open(extractedfiles[0])
for k in extractedhdu[0].header.keys():
if not k in comhdu[0].header.keys():
extractedhdu[0].header.cards[k].verify('fix')
comhdu[0].header.append(extractedhdu[0].header.cards[k])
comhdu.flush(output_verify='fix')
comhdu.close()
extractedhdu.close()
dateobs = pyfits.getval(objname + '_com.fits', 'DATE-OBS')
dateobs += 'T' + pyfits.getval(objname + '_com.fits', 'TIME-OBS')
pyfits.setval(objname + '_com.fits', 'DATE-OBS', value=dateobs)
def combine(doreduce=True, doshifts=True):
if doreduce:
ims = glob('oc01020[1-4]*_raw.fits')
ims += glob('oc016*_raw.fits')
# for each image
for im in ims:
print im
stistools.basic2d.basic2d(im, im.replace('raw', 'flt'))
im = im.replace('raw', 'flt')
print im
# Update the target position at 0.0
for i in range(4):
pyfits.setval(im, 'POSTARG2', value=0.0, ext=i)
# reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im, 'APERTAB', value='oref$y2r1559to_apt.fits')
# Reset the wcs to have CRPIX2 along the trace
# Run x2d on the flt frame
stistools.x2d.x2d(input=im, output=im.replace('flt', 'x2d'))
h = pyfits.open(im.replace('flt', 'x2d'), mode='update')
# Replace all of the bad pixels in the image by -10000 based on the DQ array
# save them to a new file
# Throw away bad reference file pixels and saturated pixels. None of the other error codes
# were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
h.flush()
# Trim the image
for i in range(1, 4):
h[i].data = h[i].data[100:-100, 120:-100]
h[i].header['CRPIX1'] -= 120
h[i].header['CRPIX2'] -= 100
h.close()
ims = glob('oc01020[1-4]*_x2d.fits')
ims += glob('oc01610[1-2]*_x2d.fits')
if doshifts:
init_guesses = [501, 542, 522, 523, 541, 524]
centroids = []
for i, im in enumerate(ims):
print(im)
h = pyfits.open(im)
d = average(h[1].data[:, 915:925], axis=1)
popt, _pcov = curve_fit(gauss,
arange(len(d)),
d,
p0=[10, init_guesses[i], 1.5, 0])
centroids.append(popt[1])
shift = centroids[0] - popt[1]
from matplotlib import pyplot as pl
pl.ion()
pl.clf()
pl.plot(arange(len(d)), d)
pl.plot(arange(len(d)),
gauss(arange(len(d)), popt[0], popt[1], popt[2], popt[3]))
_w = raw_input('Press return to continue')
# watch the sign convention
# This gives what you need to shift the input image by to get to the reference image
iraf.unlearn(iraf.imshift)
iraf.imshift(im + '[1]',
im[:-8] + 'shift1.fits',
0.0,
shift,
interp_type='drizzle')
# Run imcombine on the rectified (but not flux scaled) images with crreject
iraf.unlearn(iraf.imcombine)
imlist = ''
for im in ims:
imlist += im[:-8] + 'shift1.fits,'
# drop the last comma
imlist = imlist[:-1]
iraf.imcombine(input=imlist,
output='13dh_uv_com.fits',
reject='none',
lthreshold=-20,
hthreshold=300)
# run apall on the combined flux scaled image
iraf.unlearn(iraf.apall)
iraf.apall(input='13dh_uv_com.fits',
output='13dh_uv',
review='no',
line=1024,
nsum=-50,
b_order=2,
b_function='legendre',
b_niterate=30,
b_naverage=-21,
nfind=1,
t_order=2,
background='fit',
weights='variance')
def headerUpdateKey(self, key, value):
for filename in self._matchingFiles:
self._logger.notice('update %s: %s= %s' %
(filename, key, str(value)))
pyfits.setval(filename, key, value)
def combine(doreduce=True, doshifts=True):
if doreduce:
ims = glob('oc01020[1-4]*_raw.fits')
ims += glob('oc016*_raw.fits')
# for each image
for im in ims:
print im
stistools.basic2d.basic2d(im, im.replace('raw','flt'))
im = im.replace('raw', 'flt')
print im
# Update the target position at 0.0
for i in range(4):
pyfits.setval(im, 'POSTARG2', value=0.0, ext=i)
# reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im, 'APERTAB', value='oref$y2r1559to_apt.fits')
# Reset the wcs to have CRPIX2 along the trace
# Run x2d on the flt frame
stistools.x2d.x2d(input=im, output=im.replace('flt','x2d') )
h = pyfits.open(im.replace('flt','x2d'), mode='update')
# Replace all of the bad pixels in the image by -10000 based on the DQ array
# save them to a new file
# Throw away bad reference file pixels and saturated pixels. None of the other error codes
# were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
h.flush()
# Trim the image
for i in range(1,4):
h[i].data = h[i].data[100:-100, 120:-100]
h[i].header['CRPIX1'] -= 120
h[i].header['CRPIX2'] -= 100
h.close()
ims = glob('oc01020[1-4]*_x2d.fits')
ims += glob('oc01610[1-2]*_x2d.fits')
if doshifts:
init_guesses = [501, 542, 522, 523, 541, 524]
centroids = []
for i, im in enumerate(ims):
print(im)
h = pyfits.open(im)
d = average(h[1].data[:, 915:925], axis=1)
popt, _pcov = curve_fit(gauss, arange(len(d)), d, p0=[10, init_guesses[i], 1.5, 0])
centroids.append(popt[1])
shift = centroids[0] - popt[1]
from matplotlib import pyplot as pl
pl.ion()
pl.clf()
pl.plot(arange(len(d)), d)
pl.plot(arange(len(d)), gauss(arange(len(d)), popt[0], popt[1], popt[2], popt[3]))
_w = raw_input('Press return to continue')
# watch the sign convention
# This gives what you need to shift the input image by to get to the reference image
iraf.unlearn(iraf.imshift)
iraf.imshift(im + '[1]', im[:-8] + 'shift1.fits', 0.0, shift,
interp_type='drizzle')
# Run imcombine on the rectified (but not flux scaled) images with crreject
iraf.unlearn(iraf.imcombine)
imlist = ''
for im in ims:
imlist += im[:-8] + 'shift1.fits,'
# drop the last comma
imlist = imlist[:-1]
iraf.imcombine(input=imlist, output='13dh_uv_com.fits', reject='none',
lthreshold=-20, hthreshold=300)
# run apall on the combined flux scaled image
iraf.unlearn(iraf.apall)
iraf.apall(input='13dh_uv_com.fits', output='13dh_uv', review='no',
line=1024, nsum=-50, b_order=2, b_function='legendre',
b_niterate=30, b_naverage=-21, nfind=1, t_order=2,
background='fit', weights='variance')
def setval(fits, key, value, ext):
print('{}[{}]: {} -> {}'.format(fits,ext,key,value))
pyfits.setval(fits, key, value=value, ext=ext)
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-400:-200,200:400', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=100, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit1d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('trm/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
#diagnostic()
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
# print ('---hdu.data---')
# print (hdu[0].data)
w=WCS(f)
# print ('-----w-----')
# print(w)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
# print('-----npix-----')
# print(npix)
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# print('-----lam-----')
# print(lam)
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
#print ('-----specs-----')
#print (specs)
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=-2e-16)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd, comment='average of multiple exposures')
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet,comment='sum of multiple exposures')
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam,comment='average of multiple exposures')
print "avg AIRMASS = " + str(avgam)
# update this to used avg jd midpoint of all exposures?
print "barycentric velocity correction (km/s) = ",
iraf.bcvcorr(spectra=combfile,keytime='UTC-OBS',keywhen='mid',
obslong="339:11:16.8",obslat="-32:22:46.2",obsalt='1798',obsname='saao',
savebcv='yes',savejd='yes',printmode=2)
pyfits.setval(combfile,'UTMID',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'GJDN',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'HJDN',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'BCV',comment='added by RVSAO task BCVCORR (km/s)')
pyfits.setval(combfile,'HCV',comment='added by RVSAO task BCVCORR (km/s)')
iraf.dopcor(input=combfile,output='',redshift=-iraf.bcvcorr.bcv,isvelocity='yes',
add='no',dispersion='yes',flux='no',verbose='yes')
pyfits.setval(combfile,'DOPCOR01',comment='barycentric velocity correction applied')
iraf.cd('..')
dz.task_attributes['weight'] = '""'
dz.task_attributes['gain'] = 'GAIN'
dz.task_attributes['snoise'] = 'READNOIS'
#Run the task
dz.run_iraf_task('imcombine', run_externally=False)
#Pause to check task output
#raw_input("\nPress Enter to continue...")
#Add objects to data frame with the new frame_tag
dz.object_to_dataframe(dz.task_attributes['output'], data_dict)
#Setting the new airmass value
pyfits.setval(filename=dz.task_attributes['output'],
keyword='AIRMASS',
value=Airmass_combine)
#New files
if tag_to_combine == 'frame_shifted':
idx_print = (dz.reducDf.reduc_tag == 'obj_combine') | (
(dz.reducDf.reduc_tag == 'frame_shifted') &
(dz.reducDf.frame_tag.isin(dz.observation_dict['objects'])) &
(dz.target_validity_check()))
dz.generate_step_pdf(idx_print,
file_address=dz.reducFolders['reduc_data'] +
'target_combined_shiftedframes',
ext=0,
plots_type='frame_combine_shifted')
else:
idx_print = (dz.reducDf.reduc_tag == 'obj_combine') | (
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits','oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',outflat='oc0102070_nsp.fits', do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits', output='temp_nsp.fits', equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]', 'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc','oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im +'.fits','APERTAB',value='oref$y2r1559to_apt.fits')
pyfits.setval(im +'.fits', 'SPTRCTAB', value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im+ '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase+'mask1.fits', mask1.astype('i'), clobber=True)
fits.writeto(outbase+'mask2.fits', mask2.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'mask2.fits')
# Subtract off the median value
hdu = fits.open(im+ '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data, readnoise=readnoise1,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data, readnoise=readnoise2,
sigclip=5, objlim=5, sigfrac=0.8,
fsmode='median', psfmodel='gaussy',
psffwhm=2., cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits', crmask1.astype('i'), clobber=True)
fits.writeto(outbase + '_crmask2.fits', crmask2.astype('i'), clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[1]', outbase+'_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im+'[4]', outbase+'_crmask2.fits')
if outbase=='red':
iraf.mkfringeflat('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102070_frr.fits',
beg_scale=0.6, end_scale=1.5, scale_step=0.01,
beg_shift=-3.0, end_shift=3.0,shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits', 'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',output=im[:-4]+'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',output=im[:-4]+'x2d.fits')
h = pyfits.open(im[:-4]+'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(bitwise_and(d,256) == 256,bitwise_and(d,512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1,7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4]+'x2d[1],'+im[:-4]+'x2d[4]', output=outbase+'_com.fits',
reject='crreject')
hdu = pyfits.open(outbase +'_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase+'_mask.fits', mask.astype('i'), clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase+'_com.fits', outbase+'_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase+'_com',output='13dh_'+outbase, review='no',
nsum = -500, b_order = 1,
b_function='legendre',b_niterate=30, b_naverage = -21,
nfind=1,t_order=3,background='median',weights='variance',
skybox=100 )
iraf.splot(outbase+'[SCI]')
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('flx/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
w = WCS(f)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=1e-19)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GRATING','GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd)
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet)
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam)
print "avg AIRMASS = " + str(avgam)
iraf.cd('..')
return specs
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, 1)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=1, value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcom.fits' % (ga)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1],
output=combineoutname,
combine='average',
reject='sigclip',
lsigma=3.0,
hsigma=3.0,
def add_pctetab_to_headers(input_dir, input_flist):
for ifile in input_flist:
pyfits.setval(os.path.join(input_dir, ifile),
'PCTETAB',
ext=0,
value='myref$%s' % (pctetab))
def pysalt(fs=None):
# Run the pysalt pipeline on the raw data.
if fs is None:
fs = glob('P*.fits')
if len(fs) == 0:
print "WARNING: No raw files to run PySALT pre-processing."
return
# Copy the raw files into a raw directory
if not os.path.exists('raw'):
os.mkdir('raw')
if not os.path.exists('work'):
os.mkdir('work')
for f in fs:
shutil.copy2(f, 'raw/')
shutil.move(f, 'work/')
iraf.cd('work')
# Run each of the pysalt pipeline steps deleting temporary files as we go
# saltprepare
iraf.unlearn(iraf.saltprepare)
# Currently, there is not a bad pixel mask provided by SALT
# so we don't create one here.
iraf.saltprepare(images='P*.fits', clobber=True, mode='h')
for f in glob('P*.fits'):
os.remove(f)
# saltgain
iraf.unlearn(iraf.saltgain)
# Multiply by the gain so that everything is in electrons.
iraf.saltgain(images='pP*.fits',
gaindb=pysaltpath + '/data/rss/RSSamps.dat',
mult=True, usedb=True, mode='h')
for f in glob('pP*.fits'):
os.remove(f)
# write a keyword in the header keyword gain = 1 in each amplifier
fs = glob('gpP*.fits')
for f in fs:
for i in range(1, 7):
pyfits.setval(f, 'GAIN', ext=i, value=1.0)
# saltxtalk
iraf.unlearn(iraf.saltxtalk)
iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
for f in glob('gpP*.fits'):
os.remove(f)
# saltbias
iraf.unlearn(iraf.saltbias)
iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
for f in glob('xgpP*.fits'):
os.remove(f)
# Put all of the newly created files into the pysalt directory
if not os.path.exists('pysalt'):
os.mkdir('pysalt')
for f in glob('bxgpP*.fits'):
shutil.move(f, 'pysalt')
iraf.cd('..')
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('flx/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
w = WCS(f)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=-1e-17)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd)
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet)
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam)
print "avg AIRMASS = " + str(avgam)
iraf.cd('..')
return specs
def setval(fits, key, value, ext):
print('%s[%i]: %s -> %s' % (fits, ext, key, str(value)))
pyfits.setval(fits, key, value=value, ext=ext)
def add_pctetab_to_headers(input_dir, input_flist):
for ifile in input_flist:
pyfits.setval(os.path.join(input_dir, ifile), 'PCTETAB', ext = 0, value = 'myref$%s' %(pctetab))
def pysalt(fs=None):
# Run the pysalt pipeline on the raw data.
if fs is None:
fs = glob('P*.fits')
if len(fs) == 0:
print "WARNING: No raw files to run PySALT pre-processing."
return
# Copy the raw files into a raw directory
if not os.path.exists('raw'):
os.mkdir('raw')
if not os.path.exists('work'):
os.mkdir('work')
for f in fs:
shutil.copy2(f, 'raw/')
shutil.move(f, 'work/')
iraf.cd('work')
# Run each of the pysalt pipeline steps deleting temporary files as we go
# saltprepare
iraf.unlearn(iraf.saltprepare)
# Currently, there is not a bad pixel mask provided by SALT
# so we don't create one here.
iraf.saltprepare(images='P*.fits', clobber=True, mode='h')
for f in glob('P*.fits'):
os.remove(f)
# saltgain
iraf.unlearn(iraf.saltgain)
# Multiply by the gain so that everything is in electrons.
iraf.saltgain(images='pP*.fits',
gaindb=pysaltpath + '/data/rss/RSSamps.dat',
mult=True, usedb=True, mode='h')
for f in glob('pP*.fits'):
os.remove(f)
# write a keyword in the header keyword gain = 1 in each amplifier
fs = glob('gpP*.fits')
for f in fs:
for i in range(1, 7):
pyfits.setval(f, 'GAIN', ext=i, value=1.0)
# saltxtalk
iraf.unlearn(iraf.saltxtalk)
iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
for f in glob('gpP*.fits'):
os.remove(f)
# saltbias
iraf.unlearn(iraf.saltbias)
iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
for f in glob('xgpP*.fits'):
os.remove(f)
# Put all of the newly created files into the pysalt directory
if not os.path.exists('pysalt'):
os.mkdir('pysalt')
for f in glob('bxgpP*.fits'):
shutil.move(f, 'pysalt')
iraf.cd('..')
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-200:-100,100:200', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=200, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit2d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
def combine(do_cti=False, doreduce=True, doshifts=True):
if do_cti:
os.system('stis_cti --crds_update')
if doreduce:
# Defringing didn't seem to converge because of the low S/N
stistools.ocrreject.ocrreject('oc0102070_flc.fits',
'oc0102070_crc.fits')
iraf.normspflat(inflat='oc0102070_crc.fits',
outflat='oc0102070_nsp.fits',
do_cal='no')
iraf.imcalc(input='oc0102070_nsp.fits',
output='temp_nsp.fits',
equals='if(x .lt. 250) then 1 else im1')
iraf.imcopy('temp_nsp.fits[1][1:250,*]',
'oc0102070_nsp.fits[1][1:250,*]')
#iraf.defringe('oc0102060_flc.fits', 'oc0102070_nsp.fits', 'oc0102060_dfr.fits')
#for each image
for im in ['oc0102050_flc', 'oc0102060_flc']:
outbase = 'blue'
if im[:-4] == 'oc0102060':
outbase = 'red'
#reset the aperture table to the newer file (we maybe should check this)
pyfits.setval(im + '.fits',
'APERTAB',
value='oref$y2r1559to_apt.fits')
pyfits.setval(im + '.fits',
'SPTRCTAB',
value='oref$qa31608go_1dt.fits')
# fixpix any negative values. In principle some of this noise
# could be real, but I have found that is often not the case
hdu = fits.open(im + '.fits')
mask1 = hdu[1].data < -20
mask2 = hdu[4].data < -20
hdu.close()
fits.writeto(outbase + 'mask1.fits',
mask1.astype('i'),
clobber=True)
fits.writeto(outbase + 'mask2.fits',
mask2.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + 'mask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + 'mask2.fits')
# Subtract off the median value
hdu = fits.open(im + '.fits', mode='update')
hdu[1].data -= np.median(hdu[1].data)
hdu[4].data -= np.median(hdu[4].data)
readnoise1 = 1.4826 * np.median(np.abs(hdu[1].data))
readnoise2 = 1.4826 * np.median(np.abs(hdu[4].data))
# Cosmic ray reject both images using scrappy
# Make sure to treat the noise in a sensible way
crmask1, clean1 = detect_cosmics(hdu[1].data,
readnoise=readnoise1,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
crmask2, clean2 = detect_cosmics(hdu[4].data,
readnoise=readnoise2,
sigclip=5,
objlim=5,
sigfrac=0.8,
fsmode='median',
psfmodel='gaussy',
psffwhm=2.,
cleantype='idw')
hdu.flush()
hdu.close()
fits.writeto(outbase + '_crmask1.fits',
crmask1.astype('i'),
clobber=True)
fits.writeto(outbase + '_crmask2.fits',
crmask2.astype('i'),
clobber=True)
# Run fixpix on the frames
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[1]', outbase + '_crmask1.fits')
iraf.unlearn(iraf.fixpix)
iraf.fixpix(im + '[4]', outbase + '_crmask2.fits')
if outbase == 'red':
iraf.mkfringeflat('oc0102060_flc.fits',
'oc0102070_nsp.fits',
'oc0102070_frr.fits',
beg_scale=0.6,
end_scale=1.5,
scale_step=0.01,
beg_shift=-3.0,
end_shift=3.0,
shift_step=0.05)
iraf.defringe('oc0102060_flc.fits', 'oc0102070_frr.fits',
'oc0102060_dfr.fits')
#Run x2d on the flt frame
stistools.x2d.x2d(input='oc0102060_dfr.fits',
output=im[:-4] + 'x2d.fits')
else:
stistools.x2d.x2d(input='oc0102050_flc.fits',
output=im[:-4] + 'x2d.fits')
h = pyfits.open(im[:-4] + 'x2d.fits', mode='update')
#Replace all of the bad pixels in the image by -666 based on the DQ array
#save them to a new file
#Throw away bad reference file pixels and saturated pixels. None of the other error codes
#were in the first file so I haven't included them here, but we might want to
d = h[3].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[1].data[badpix] = -10000
d = h[6].data
badpix = logical_and(
bitwise_and(d, 256) == 256,
bitwise_and(d, 512) == 512)
h[4].data[badpix] = -10000
h.flush()
# Trim the images
for i in range(1, 7):
h[i].data = h[i].data[100:-100, 100:-100]
h[i].header['CRPIX1'] -= 100
h[i].header['CRPIX2'] -= 100
h.flush()
h.close()
# Combine the images
iraf.unlearn(iraf.imcombine)
iraf.imcombine(input=im[:-4] + 'x2d[1],' + im[:-4] + 'x2d[4]',
output=outbase + '_com.fits',
reject='crreject')
hdu = pyfits.open(outbase + '_com.fits')
mask = hdu[0].data == 0.0
hdu.close()
fits.writeto(outbase + '_mask.fits',
mask.astype('i'),
clobber=True)
iraf.unlearn(iraf.fixpix)
iraf.fixpix(outbase + '_com.fits', outbase + '_mask.fits')
iraf.unlearn(iraf.apall)
iraf.apall(input=outbase + '_com',
output='13dh_' + outbase,
review='no',
nsum=-500,
b_order=1,
b_function='legendre',
b_niterate=30,
b_naverage=-21,
nfind=1,
t_order=3,
background='median',
weights='variance',
skybox=100)
iraf.splot(outbase + '[SCI]')
"""
try:
import pyfits
except ImportError, err:
print 'pyfits module is required for reading and writing fits headers'
pyfits = None
raise err
expstart = pyfits.getval(filename, 'EXPSTART', ext=ext)
expend = pyfits.getval(filename, 'EXPEND', ext=ext)
focus = mean_focus(expstart, expend, with_var=with_var, **kwargs)
if not hasattr(focus, 'count'):
focus = (focus, )
comments = ('Estimated mean focus (HST Focus Model)',
'Estimated focus variance (HST Focus Model)')
for key, val, comment in zip((focus_key, var_key), focus, comments):
pyfits.setval(filename, key, value=val, comment=comment)
def _mjd_to_year_date_time(mjd):
"""
Convert Modified Julian Date number to (year, date, time) tuple
:param mjd: Modified Julian Date (e.g. from fits header)
:return: 3-tuple of (YYYY, MM/DD, HH:MM:SS) (int, string, string)
"""
# Adapted from libastro/XEphem included in pyephem, which erroneously
# calls its dates Modified Julian Date, when in fact they are Dublin
# Julian Date (Dec 31 1899 12:00 zero point).
# https://github.com/brandon-rhodes/pyephem/blob/master/libastro-3.7.5/mjd.c
_mjd_to_dublin = -15019.5 # Conversion from MJD to Dublin JD (Wikipedia)
_days_per_year = 365.25 # In JD convention, anyway
XA=stats_section[0], XB=stats_section[1], YA=stats_section[2], YB=stats_section[3]
)
dz.task_attributes["reject"] = "crreject"
dz.task_attributes["weight"] = '""'
dz.task_attributes["gain"] = "GAIN"
dz.task_attributes["snoise"] = "READNOIS"
# Run the task
dz.run_iraf_task("imcombine", run_externally=False)
# Pause to check task output
# raw_input("\nPress Enter to continue...")
# Add objects to data frame with the new frame_tag
dz.object_to_dataframe(dz.task_attributes["output"], data_dict)
# Setting the new airmass value
pyfits.setval(filename=dz.task_attributes["output"], keyword="AIRMASS", value=Airmass_combine)
# New files
idx_print = (dz.reducDf.reduc_tag == "obj_combine") | (
(dz.reducDf.reduc_tag == "cosmic_ray_removal")
& (dz.reducDf.frame_tag.isin(dz.observation_dict["objects"]))
& (dz.target_validity_check())
)
dz.generate_step_pdf(
idx_print, file_address=dz.reducFolders["reduc_data"] + "target_combinedframes", ext=0, plots_type="frame_combine"
)
print "Data treated"
