def do_psf_photometry(list_files, satmag, photfilesuffix,psfstarfile='', thresh=6.,psfcleaningradius=10.):
for datafile in list_files[:]:
print datafile
iraf.cd(os.path.dirname(datafile))
remove_previous_psfphot_files(datafile)
# extract sources.
psf_photometry_pairitel(datafile, satmag, photfilesuffix,psfstarfile,psfcleaningradius, thresh)
def remove_previous_psfphot_files(filename):
# remove old files from previous runs (pst files = psf fitting stars, sub = subtracted images for psf fitting, als and arj = detected sources with psf fitting, nrj and nst = peak finding detections)
iraf.cd(os.path.dirname(filename))
for fl in glob.glob(filename + '*.pst.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.sub.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.als.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.arj.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.nst.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.nrj.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.psg.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.psf.*.fits'):
os.remove(fl)
for fl in glob.glob(filename + '*.secondrun.mag.*'):
os.remove(fl)
for fl in glob.glob(filename + '*.mag.*'):
os.remove(fl)
def wspectext():
if not os.path.exists(Path + '/test3/'):
os.mkdir(Path + '/test3/')
fs = glob(Path + '/test/' + '/sci*.fits')
for i, f in enumerate(fs):
iraf.cd(Path + '/test3/')
iraf.wspectext(f, ('sci' + str(i) + '.dat'))
def do_psf_photometry_with_coo(list_files, satmag, photfilesuffix,psfstarlist, thresh=6.,psfcleaningradius=10.):
print '!!!!!!!!!!!!'+str(psfcleaningradius)
for i in np.arange(0, len(list_files)):
print list_files[i]
iraf.cd(os.path.dirname(list_files[i]))
remove_previous_psfphot_files(list_files[i])
# extract sources.
psf_photometry_pairitel_with_coo(list_files[i], satmag, photfilesuffix,psfstarlist[i],psfcleaningradius=psfcleaningradius, thresh=thresh)
def mask_norm(mask):
# normalizes mask.
# test if trimmed mask exists:
if os.path.isfile(mask):
iraf.cd(os.path.dirname(mask))
hdulist = pyfits.open(mask)
imdata = hdulist[0].data
hdulist.close()
outname = mask.replace('_trimmed.fits', '_normed.fits')
print imdata.max()
iraf.imarith(os.path.basename(mask),"/",imdata.max(),os.path.basename(outname))
def sorting(fs=None):
# Run the pysalt pipeline on the raw data.
if fs is None:
fs = glob('mbxgpP*.fits')
if len(fs) == 0:
fs = glob('mbxpP*.fits')
if len(fs) ==0:
print "WARNING: No raw files to run PySALT pre-processing."
return
# Copy the raw files into a raw directory
if not os.path.exists('product'):
os.mkdir('product')
if not os.path.exists('work'):
os.mkdir('work')
if not os.path.exists('work/srt'):
os.mkdir('work/srt')
for f in fs:
shutil.copy2(f, 'product/')
scifs, scigas = get_ims(fs, 'sci')
arcfs, arcgas = get_ims(fs, 'arc')
ims = np.append(scifs, arcfs)
gas = np.append(scigas, arcgas)
for i, f in enumerate(ims):
ga = gas[i]
if f in scifs:
typestr = 'sci'
else:
typestr = 'arc'
# by our naming convention, imnum should be the last 4 characters
# before the '.fits'
imnum = f[-9:-5]
outname = 'srt/' + typestr
outname += '%05.2fmos%04i.fits' % (float(ga), int(imnum))
shutil.move(f, 'work/'+outname)
iraf.cd('work')
h = pyfits.open(outname, 'update')
maskim = h[1].data.copy()
maskim[:, :] = 0.0
maskim[abs(h[1].data) < 1e-5] = 1
imhdu = pyfits.ImageHDU(maskim)
h.append(imhdu)
h[1].header['BPMEXT'] = 2
h[2].header['EXTNAME'] = 'BPM'
h[2].header['CD2_2'] = 1
h.flush()
h.close()
iraf.cd('..')
def prepare_files(list_files, resultpath, readoutnoise):
# copies data to output directory, trim data files and mask files to contain only good exposure parts, normalize masks, normalize images by masks. Yields exposure-normalized, trimmed sky images.
for datafile in list_files[:]:
#prepare directories and copy image to resultpath
filename=os.path.basename(datafile)
obsid=os.path.basename(os.path.dirname(datafile))
imagepath=os.path.join(resultpath,obsid)
image=os.path.join(imagepath,filename)
if not os.access(imagepath,os.F_OK): os.makedirs(imagepath)
print image
shutil.copy(datafile,imagepath)
os.chdir(imagepath)
iraf.cd(imagepath)
# add readoutnoise keyword to header:
iraf.hedit(images=filename, fields="RDNOISE", value=str(readoutnoise), addonly="yes", verify="no")
def trim(fs=None):
iraf.cd('work')
if fs is None:
fs=glob('flx/sci*c?.fits')
if not os.path.exists('trm'):
os.mkdir('trm')
for i,f in enumerate(fs):
outfile=f.replace('flx','trm')
w=WCS(f)
hdu= pyfits.open(f)
npix=hdu[0].data.shape[2]
W1=int(w.wcs_pix2world(2,0,0,0)[0])
W2=int(w.wcs_pix2world(npix-2,0,0,0)[0])
iraf.sarith(input1=f,op='copy',output=outfile,w1=W1,w2=W2
,clobber='yes')
iraf.cd('..')
def do_aperture_photometry(list_files, photfilesuffix):
for datafile in list_files[:]:
print datafile
iraf.cd(os.path.dirname(datafile))
# delete old aperture extractions
for fl in glob.glob(datafile + '.coo.*'):
os.remove(fl)
for fl in glob.glob(datafile + '.mag.*'):
os.remove(fl)
for fl in glob.glob(datafile + photfilesuffix):
os.remove(fl)
# extract sources.
aperture_photometry(datafile,photfilesuffix, threshold=4.,wcs='logical',mode='small', telescope='Pairitel')
def createMagFile(inputDir, inputFile, useCoordFile, outputFile, fullyAutomatic=False, aperturesString='5,10,15,20', bestSkyAperture='100'):
cr=['\n']
iraf.cd(inputDir)
iraf.noao()
iraf.digiphot()
iraf.apphot()
iraf.phot.setParam('interactive','no')
iraf.centerpars.setParam('cbox','5')
iraf.photpars.setParam('apertures',aperturesString)
iraf.fitskypars.setParam('annulus',bestSkyAperture)
iraf.fitskypars.setParam('dannulus','10')
iraf.phot.setParam('image',inputFile)
iraf.phot.setParam('coords',useCoordFile)
iraf.phot.setParam('output',outputFile)
if fullyAutomatic: ## If you use this option, it will not prompt you for any of the aperture/sky settings
iraf.phot(mode='h',Stdin=cr)
else: # Otherwise, you can change them at will, and get to press enter a bunch
iraf.phot(mode='h')
def fixpix(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images to fix."
iraf.cd('..')
return
if not os.path.exists('fix'):
os.mkdir('fix')
for f in fs:
outname = f.replace('nrm', 'fix')
# Copy the file to the fix directory
shutil.copy(f, outname)
# Set all of the BPM pixels = 0
h = pyfits.open(outname, mode='update')
h['SCI'].data[h['BPM'].data == 1] = 0
# Grab the CRM extension from the lax file
laxhdu = pyfits.open(f.replace('nrm', 'lax'))
h.append(pyfits.ImageHDU(data=laxhdu['CRM'].data.copy(),
header=laxhdu['CRM'].header.copy(),
name='CRM'))
h.flush()
h.close()
laxhdu.close()
# Run iraf's fixpix on the cosmic rays, not ideal,
# but better than nothing because apall doesn't take a bad pixel mask
iraf.unlearn(iraf.fixpix)
iraf.flpr()
iraf.fixpix(outname + '[SCI]', outname + '[CRM]', mode='hl')
iraf.cd('..')
def flatten(fs=None, masterflatdir=None):
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgpP*.fits')
if len(fs) == 0:
print "WARNING: No images to flat-field."
# Change directories to fail more gracefully
iraf.cd('..')
return
if not os.path.exists('flts'):
os.mkdir('flts')
# Make sure there are science images or arcs and what grating angles were
# used
scifs, scigas = get_ims(fs, 'sci')
arcfs, arcgas = get_ims(fs, 'arc')
ims = np.append(scifs, arcfs)
gas = np.append(scigas, arcgas)
# For each science and arc image
for i, f in enumerate(ims):
thishdu = pyfits.open(f)
ga = gas[i]
# For each chip
for c in range(1, 7):
flatfile = 'flats/flt%05.2fresc%i.fits' % (ga, c)
if len(glob(flatfile)) == 0:
if masterflatdir is None:
print("No flat field image found for %s"% f)
continue
# Check for the master flat directory
flatfile = masterflatdir+'/flt%05.2fresc%i.fits' % (ga, c)
if len(glob(flatfile)) == 0:
# Still can't find one? Abort!!
print("No flat field image found for %s"% f)
continue
# open the corresponding response file
resphdu = pyfits.open(flatfile)
# divide out the illumination correction and the flatfield
# make sure divzero = 0.0
thishdu[c].data /= resphdu[0].data.copy()
# replace the infinities with 0.0
thishdu[c].data[np.isinf(thishdu[c].data)] = 0.0
resphdu.close()
# save the updated file
if f in scifs:
typestr = 'sci'
else:
typestr = 'arc'
# get the image number
# by salt naming convention, these should be the last 4 characters
# before the '.fits'
imnum = f[-9:-5]
outname = 'flts/' + typestr + '%05.2fflt%04i.fits' % (float(ga),
int(imnum))
thishdu.writeto(outname)
thishdu.close()
iraf.cd('..')
def fixpix(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images to fix."
iraf.cd('..')
return
if not os.path.exists('fix'):
os.mkdir('fix')
for f in fs:
outname = f.replace('nrm', 'fix')
# Copy the file to the fix directory
shutil.copy(f, outname)
# Set all of the BPM pixels = 0
h = pyfits.open(outname, mode='update')
h['SCI'].data[h['BPM'].data == 1] = 0
# Grab the CRM extension from the lax file
laxhdu = pyfits.open(f.replace('nrm', 'lax'))
h.append(pyfits.ImageHDU(data=laxhdu['CRM'].data.copy(),
header=laxhdu['CRM'].header.copy(),
name='CRM'))
h.flush()
h.close()
laxhdu.close()
# Run iraf's fixpix on the cosmic rays, not ideal,
# but better than nothing because apall doesn't take a bad pixel mask
iraf.unlearn(iraf.fixpix)
iraf.flpr()
iraf.fixpix(outname + '[SCI]', outname + '[CRM]', mode='hl')
iraf.cd('..')
def split1d(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d????.fits')
if len(fs) == 0:
print "WARNING: No extracted spectra to split."
iraf.cd('..')
return
for f in fs:
hdu = pyfits.open(f.replace('x1d', 'fix'))
chipgaps = get_chipgaps(hdu)
# Throw away the first pixel as it almost always bad
chipedges = [[1, chipgaps[0][0]], [chipgaps[0][1] + 1, chipgaps[1][0]],
[chipgaps[1][1] + 1, chipgaps[2][0]]]
w = WCS(f)
# Copy each of the chips out seperately. Note that iraf is 1 indexed
# unlike python so we add 1
for i in range(3):
# get the wavelengths that correspond to each chip
lam, _apnum, _bandnum = w.all_pix2world(chipedges[i], 0, 0, 0)
iraf.scopy(f, f[:-5] + 'c%i' % (i + 1), w1=lam[0], w2=lam[1],
format='multispec', rebin='no',clobber='yes')
hdu.close()
iraf.cd('..')
def identify2d(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('mos/arc*mos*.fits')
if len(fs) == 0:
print "WARNING: No mosaiced (2D) specidentify."
# Change directories to fail gracefully
iraf.cd('..')
return
arcfs, arcgas = get_ims(fs, 'arc')
if not os.path.exists('id2'):
os.mkdir('id2')
lampfiles = {
'Th Ar': 'ThAr.salt',
'Xe': 'Xe.salt',
'Ne': 'NeAr.salt',
'Cu Ar': 'CuAr.salt',
'Ar': 'Argon_hires.salt',
'Hg Ar': 'HgAr.salt'
}
for i, f in enumerate(arcfs):
ga = arcgas[i]
# find lamp and corresponding linelist
lamp = pyfits.getval(f, 'LAMPID')
lampfn = lampfiles[lamp]
if pyfits.getval(f, 'GRATING') == 'PG0300' and lamp == 'Ar':
lampfn = 'Argon_lores.swj'
ccdsum = int(pyfits.getval(f, 'CCDSUM').split()[1])
# linelistpath is a global variable defined in beginning, path to
# where the line lists are.
lamplines = pysaltpath + '/data/linelists/' + lampfn
print(lamplines)
# img num should be right before the .fits
imgnum = f[-9:-5]
# run pysalt specidentify
idfile = 'id2/arc%05.2fid2%04i' % (float(ga), int(imgnum)) + '.db'
iraf.unlearn(iraf.specidentify)
iraf.flpr()
iraf.specidentify(
images=f,
linelist=lamplines,
outfile=idfile,
guesstype='rss',
inter=True, # automethod='FitXcor',
rstep=600 / ccdsum,
rstart=200 / ccdsum,
startext=1,
clobber='yes',
#startext=1, clobber='yes',
verbose='no',
mode='hl',
logfile='salt.log',
mdiff=2,
function='legendre')
iraf.cd('..')
def stdsensfunc(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d*c?.fits')
if len(fs) == 0:
print "WARNING: No extracted spectra to create sensfuncs from."
iraf.cd('..')
return
if not os.path.exists('std'):
os.mkdir('std')
for f in fs:
# Put the file in the std directory, but last 3 letters of sens
outfile = 'std/' + f.split('/')[1]
outfile = outfile.replace('x1d', 'sens').replace('sci', 'std')
outfile = outfile.replace('.fits', '.dat')
# if the object name is in the list of standard stars from pysalt
if isstdstar(f):
# We use pysalt here because standard requires a
# dispersion correction which was already taken care of above
# Write out an ascii file that pysalt.specsens can read
asciispec = 'std/std.ascii.dat'
spectoascii(f, asciispec)
# run specsens
stdfile = pysaltpath + '/data/standards/spectroscopic/m%s.dat' % pyfits.getval(f, 'OBJECT').lower().replace('-','_')
extfile = pysaltpath + '/data/site/suth_extinct.dat'
iraf.unlearn(iraf.specsens)
iraf.specsens(asciispec, outfile, stdfile, extfile,
airmass=pyfits.getval(f, 'AIRMASS'),
exptime=pyfits.getval(f, 'EXPTIME'), function='poly',
order=11, clobber=True, mode='h', thresh=1e10)
# delete the ascii file
os.remove(asciispec)
iraf.cd('..')
def flatten(fs=None, masterflatdir=None):
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgpP*.fits')
if len(fs) == 0:
print "WARNING: No images to flat-field."
# Change directories to fail more gracefully
iraf.cd('..')
return
if not os.path.exists('flts'):
os.mkdir('flts')
# Make sure there are science images or arcs and what grating angles were
# used
scifs, scigas = get_ims(fs, 'sci')
arcfs, arcgas = get_ims(fs, 'arc')
ims = np.append(scifs, arcfs)
gas = np.append(scigas, arcgas)
# For each science and arc image
for i, f in enumerate(ims):
thishdu = pyfits.open(f)
ga = gas[i]
# For each chip
for c in range(1, 7):
flatfile = 'flats/flt%05.2fresc%i.fits' % (ga, c)
if len(glob(flatfile)) == 0:
if masterflatdir is None:
print("No flat field image found for %s"% f)
continue
# Check for the master flat directory
flatfile = masterflatdir+'/flt%05.2fresc%i.fits' % (ga, c)
if len(glob(flatfile)) == 0:
# Still can't find one? Abort!!
print("No flat field image found for %s"% f)
continue
# open the corresponding response file
resphdu = pyfits.open(flatfile)
# divide out the illumination correction and the flatfield
# make sure divzero = 0.0
thishdu[c].data /= resphdu[0].data.copy()
# replace the infinities with 0.0
thishdu[c].data[np.isinf(thishdu[c].data)] = 0.0
resphdu.close()
# save the updated file
if f in scifs:
typestr = 'sci'
else:
typestr = 'arc'
# get the image number
# by salt naming convention, these should be the last 4 characters
# before the '.fits'
imnum = f[-9:-5]
outname = 'flts/' + typestr + '%05.2fflt%04i.fits' % (float(ga),
int(imnum))
thishdu.writeto(outname)
thishdu.close()
iraf.cd('..')
def split1d(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d????.fits')
if len(fs) == 0:
print "WARNING: No extracted spectra to split."
iraf.cd('..')
return
for f in fs:
hdu = pyfits.open(f.replace('x1d', 'fix'))
chipgaps = get_chipgaps(hdu)
# Throw away the first pixel as it almost always bad
chipedges = [[1, chipgaps[0][0]], [chipgaps[0][1] + 1, chipgaps[1][0]],
[chipgaps[1][1] + 1, chipgaps[2][0]]]
w = WCS(f)
# Copy each of the chips out seperately. Note that iraf is 1 indexed
# unlike python so we add 1
for i in range(3):
# get the wavelengths that correspond to each chip
lam, _apnum, _bandnum = w.all_pix2world(chipedges[i], 0, 0, 0)
iraf.scopy(f, f[:-5] + 'c%i' % (i + 1), w1=lam[0], w2=lam[1],
format='multispec', rebin='no',clobber='yes')
hdu.close()
iraf.cd('..')
def stdsensfunc(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d*c?.fits')
if len(fs) == 0:
print "WARNING: No extracted spectra to create sensfuncs from."
iraf.cd('..')
return
if not os.path.exists('std'):
os.mkdir('std')
for f in fs:
# Put the file in the std directory, but last 3 letters of sens
outfile = 'std/' + f.split('/')[1]
outfile = outfile.replace('x1d', 'sens').replace('sci', 'std')
outfile = outfile.replace('.fits', '.dat')
# if the object name is in the list of standard stars from pysalt
if isstdstar(f):
# We use pysalt here because standard requires a
# dispersion correction which was already taken care of above
# Write out an ascii file that pysalt.specsens can read
asciispec = 'std/std.ascii.dat'
spectoascii(f, asciispec)
# run specsens
stdfile = pysaltpath + '/data/standards/spectroscopic/m%s.dat' % pyfits.getval(f, 'OBJECT').lower().replace('-','_')
extfile = pysaltpath + '/data/site/suth_extinct.dat'
iraf.unlearn(iraf.specsens)
iraf.specsens(asciispec, outfile, stdfile, extfile,
airmass=pyfits.getval(f, 'AIRMASS'),
exptime=pyfits.getval(f, 'EXPTIME'), function='poly',
order=11, clobber=True, mode='h', thresh=1e10)
# delete the ascii file
os.remove(asciispec)
iraf.cd('..')
def fluxcal(stdsfolder='./', fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d*c*.fits')
if len(fs) == 0:
print "WARNING: No science chip spectra to flux calibrate."
iraf.cd('..')
return
if not os.path.exists('flx'):
os.mkdir('flx')
extfile = pysaltpath + '/data/site/suth_extinct.dat'
stdfiles = glob(stdsfolder + '/std/*sens*c?.dat')
print(stdfiles)
for f in fs:
outfile = f.replace('x1d', 'flx')
chip = outfile[-6]
hdu = pyfits.open(f)
ga = f.split('/')[1][3:8]
# Get the standard sensfunc with the same grating angle
stdfile = None
for stdf in stdfiles:
if np.isclose(float(ga),
float(stdf.split('/')[stdf.count('/')][3:8]),
rtol=1e-2):
# Get the right chip number
if chip == stdf[-5]:
stdfile = stdf
break
if stdfile is None:
print('No standard star with grating-angle %s' % ga)
continue
# for each extracted aperture
for i in range(hdu[0].data.shape[1]):
# create an ascii file that pysalt can read
asciiname = 'flx/sciflx.dat'
outtmpname = 'flx/scical.dat'
spectoascii(f, asciiname, i)
# Run pysalt.speccal
iraf.unlearn(iraf.speccal)
iraf.flpr()
iraf.speccal(asciiname,
outtmpname,
stdfile,
extfile,
airmass=pyfits.getval(f, 'AIRMASS'),
exptime=pyfits.getval(f, 'EXPTIME'),
clobber=True,
mode='h')
# read in the flux calibrated ascii file and copy its
# contents into a fits file
flxcal = np.genfromtxt(outtmpname).transpose()
hdu[0].data[0, i] = flxcal[1]
hdu[0].data[2, i] = flxcal[2]
# delete the ascii file
os.remove(asciiname)
os.remove(outtmpname)
hdu.writeto(outfile, clobber=True)
iraf.cd('..')
def mktelluric(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('sci_com.fits')
if len(fs) == 0:
print "WARNING: No flux-calibrated spectra to make the a telluric correction."
iraf.cd('..')
return
if not os.path.exists('tel'):
os.mkdir('tel')
# for each file
f = fs[0]
# if it is a standard star combined file
if isstdstar(f):
# read in the spectrum and calculate the wavelengths of the pixels
hdu = pyfits.open(f)
spec = hdu[0].data.copy()
hdr = hdu[0].header.copy()
hdu.close()
waves = fitshdr_to_wave(hdr)
template_spectrum = signal.savgol_filter(spec, 21, 3)
noise = np.abs(spec - template_spectrum)
noise = ndimage.filters.gaussian_filter1d(noise, 100.0)
not_telluric = np.ones(spec.shape, dtype=np.bool)
# For each telluric region
for wavereg in telluricWaves:
in_telluric_region = np.logical_and(waves >= wavereg[0],
waves <= wavereg[1])
not_telluric = np.logical_and(not_telluric,
np.logical_not(in_telluric_region))
# Smooth the spectrum so that the spline doesn't go as crazy
# Use the Savitzky-Golay filter to presevere the edges of the
# absorption features (both atomospheric and intrinsic to the star)
sgspec = signal.savgol_filter(spec, 31, 3)
#Get the number of data points to set the smoothing criteria for the
# spline
m = not_telluric.sum()
intpr = interpolate.splrep(waves[not_telluric], sgspec[not_telluric],
w = 1/noise[not_telluric], k=2, s=10 * m)
# Replace the telluric with the smoothed function
smoothedspec = interpolate.splev(waves, intpr)
smoothedspec[not_telluric] = spec[not_telluric]
# Divide the original and the telluric corrected spectra to
# get the correction factor
correction = spec / smoothedspec
# Save the correction
dout = np.ones((2, len(waves)))
dout[0] = waves
dout[1] = correction
np.savetxt('tel/telcor.dat', dout.transpose())
iraf.cd('..')
def mktelluric(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('sci_com.fits')
if len(fs) == 0:
print "WARNING: No flux-calibrated spectra to make the a telluric correction."
iraf.cd('..')
return
if not os.path.exists('tel'):
os.mkdir('tel')
# for each file
f = fs[0]
# if it is a standard star combined file
if isstdstar(f):
# read in the spectrum and calculate the wavelengths of the pixels
hdu = pyfits.open(f)
spec = hdu[0].data.copy()
hdr = hdu[0].header.copy()
hdu.close()
waves = fitshdr_to_wave(hdr)
template_spectrum = signal.savgol_filter(spec, 21, 3)
noise = np.abs(spec - template_spectrum)
noise = ndimage.filters.gaussian_filter1d(noise, 100.0)
not_telluric = np.ones(spec.shape, dtype=np.bool)
# For each telluric region
for wavereg in telluricWaves:
in_telluric_region = np.logical_and(waves >= wavereg[0],
waves <= wavereg[1])
not_telluric = np.logical_and(not_telluric,
np.logical_not(in_telluric_region))
# Smooth the spectrum so that the spline doesn't go as crazy
# Use the Savitzky-Golay filter to presevere the edges of the
# absorption features (both atomospheric and intrinsic to the star)
sgspec = signal.savgol_filter(spec, 31, 3)
#Get the number of data points to set the smoothing criteria for the
# spline
m = not_telluric.sum()
intpr = interpolate.splrep(waves[not_telluric], sgspec[not_telluric],
w = 1/noise[not_telluric], k=2, s=10 * m)
# Replace the telluric with the smoothed function
smoothedspec = interpolate.splev(waves, intpr)
smoothedspec[not_telluric] = spec[not_telluric]
# Divide the original and the telluric corrected spectra to
# get the correction factor
correction = spec / smoothedspec
# Save the correction
dout = np.ones((2, len(waves)))
dout[0] = waves
dout[1] = correction
np.savetxt('tel/telcor.dat', dout.transpose())
iraf.cd('..')
def lax(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('bkg/*bkg*.fits')
if len(fs) == 0:
print "WARNING: No background-subtracted files for Lacosmicx."
iraf.cd('..')
return
if not os.path.exists('lax'):
os.mkdir('lax')
for f in fs:
outname = f.replace('bkg','lax')
hdu = pyfits.open(f)
# Add a CRM extension
hdu.append(pyfits.ImageHDU(data=hdu['BPM'].data.copy(),
header=hdu['BPM'].header.copy(),
name='CRM'))
# Set all of the pixels in the CRM mask to zero
hdu['CRM'].data[:, :] = 0
# less aggressive lacosmic on standard star observations
if not isstdstar(f):
objl = 1.0
sigc = 4.0
else:
objl = 3.0
sigc = 10.0
chipgaps = get_chipgaps(hdu)
chipedges = [[0, chipgaps[0][0]], [chipgaps[0][1] + 1,
chipgaps[1][0]], [chipgaps[1][1] + 1, chipgaps[2][0]]]
# Run each chip separately
for chip in range(3):
# Use previously subtracted sky level = 0 as we have already added
# a constant sky value in the background task
# Gain = 1, readnoise should be small so it shouldn't matter much.
# Default value seems to work.
chipinds = slice(chipedges[chip][0], chipedges[chip][1])
crmask, _cleanarr = lacosmicx.lacosmicx(hdu[1].data[:, chipinds].copy(),
inmask=np.asarray(hdu[2].data[:, chipinds].copy(), dtype = np.uint8), sigclip=sigc,
objlim=objl, sigfrac=0.1, gain=1.0, pssl=0.0)
# Update the image
hdu['CRM'].data[:, chipinds][:, :] = crmask[:,:]
# Flag the cosmic ray pixels with a large negative number
hdu['SCI'].data[:, chipinds][crmask == 1] = -1000000
# Save the file
hdu.writeto(outname, clobber=True)
hdu.close()
iraf.cd('..')
def lax(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('bkg/*bkg*.fits')
if len(fs) == 0:
print "WARNING: No background-subtracted files for Lacosmicx."
iraf.cd('..')
return
if not os.path.exists('lax'):
os.mkdir('lax')
for f in fs:
outname = f.replace('bkg','lax')
hdu = pyfits.open(f)
# Add a CRM extension
hdu.append(pyfits.ImageHDU(data=hdu['BPM'].data.copy(),
header=hdu['BPM'].header.copy(),
name='CRM'))
# Set all of the pixels in the CRM mask to zero
hdu['CRM'].data[:, :] = 0
# less aggressive lacosmic on standard star observations
if not isstdstar(f):
objl = 1.0
sigc = 4.0
else:
objl = 3.0
sigc = 10.0
chipgaps = get_chipgaps(hdu)
chipedges = [[0, chipgaps[0][0]], [chipgaps[0][1] + 1,
chipgaps[1][0]], [chipgaps[1][1] + 1, chipgaps[2][0]]]
# Run each chip separately
for chip in range(3):
# Use previously subtracted sky level = 0 as we have already added
# a constant sky value in the background task
# Gain = 1, readnoise should be small so it shouldn't matter much.
# Default value seems to work.
chipinds = slice(chipedges[chip][0], chipedges[chip][1])
crmask, _cleanarr = lacosmicx.lacosmicx(hdu[1].data[:, chipinds].copy(),
inmask=np.asarray(hdu[2].data[:, chipinds].copy(), dtype = np.uint8), sigclip=sigc,
objlim=objl, sigfrac=0.1, gain=1.0, pssl=0.0)
# Update the image
hdu['CRM'].data[:, chipinds][:, :] = crmask[:,:]
# Flag the cosmic ray pixels with a large negative number
hdu['SCI'].data[:, chipinds][crmask == 1] = -1000000
# Save the file
hdu.writeto(outname, clobber=True)
hdu.close()
iraf.cd('..')
def correct_wcs(image,clobber=False,minmag=13, radius=20.):
'''correct wcs of fitsfile to 2MASS coordinates
This method loads the 2MASS catalog in the region covered by the fits file
and shifts,stretches, and rotates the wcs in the fits file to match the 2MASS coordinates
The initial wcs needs to be approximately correct (within a few arcsec), otherwise
the matching will fail.
image = filename
clobber = overwrite existing 2MASS catalog file?
minmag = mag of faintest object retrieved from 2MASS'''
object = iraf.hselect(image+'[0]','OBJECT', 'yes', Stdout=1)[0]
filter = iraf.hselect(image+'[0]','FILTER', 'yes', Stdout=1)[0]
catalogue=os.path.join(os.path.dirname(image),"2mass_"+str(object)+'_'+str(filter)+'_'+str(minmag)+".cat")
#Gererate catalogue, if it does not exist in this directory
if clobber or not os.path.isfile(catalogue):
get_catalogue(catalogue,iraf.hselect(image+'[0]','RA','yes', Stdout=1)[0],iraf.hselect(image+'[0]','DEC','yes', Stdout=1)[0], radius=radius, minmag=minmag)
#cfrac: maximum fraction of targets which do not center correctly
# a large number is no problem here, since often there are a few 100 sources in the catalog to match.
iraf.mscred(_doprint=0)
#matchout=iraf.msccmatch(image,catalogue,interactive=False, Stdout=1,cfrac=0.95,rms=3)
iraf.cd(os.path.dirname(image))
matchout=iraf.msccmatch(os.path.basename(image),catalogue,interactive=False, Stdout=1,cfrac=0.9,rms=10,maxshif=100., nsearch=100, nfit=5, update='yes')
try: print matchout[-5]
except IndexError: print matchout
def main():
for workdir in curr_work_dirs:
work_path = 'observations/' + workdir
if not os.path.exists(work_path):
os.makedirs(work_path)
if not os.path.exists(work_path+'/database'):
os.makedirs(work_path+'/database')
if not os.path.exists(work_path+'/cosmics'):
os.makedirs(work_path+'/cosmics')
# don't ask, we just need this
os.system("touch %s/blank.txt" % work_path)
# enter working folder, all the way to the end
iraf.cd(work_path)
# maybe useful, maybe not, not sure
iraf.prcacheOff()
# main steps, should work individually
#check_spectra(workdir, '.ec.vh')
create_masterbias(workdir)
create_masterflat(workdir)
#do_ref_apall(workdir)
#normalize_masterflat(workdir)
clean_folder_write_header(workdir)
bias_correct(workdir)
#flat_correct(workdir)
#remove_cosmics(workdir)
do_apall_for_real(workdir)
apply_apall(workdir)
wavelength_solution(workdir)
#check_wav_solution(workdir)
apply_wav_solution(workdir)
vhelio_correct(workdir)
#check_spectra(workdir, '.ec.vh')
combine_normalize_images(workdir, combine=False)
get_RV(workdir, multi_ref=False, multi_sample=False)
# export_spectrum_to_txt(workdir, suffix='.ec_wvl', export_cal=True)
# export_spectrum_to_txt(workdir, suffix='_vh_norm')
#check_spectra(workdir)
#new_flatfield_lens_problems(workdir)
# back to mother folder
iraf.cd('../')
iraf.cd('../')
def fluxcal(stdsfolder='./', fs=None):
iraf.cd('work')
if fs is None:
fs = glob('x1d/sci*x1d*c*.fits')
if len(fs) == 0:
print "WARNING: No science chip spectra to flux calibrate."
iraf.cd('..')
return
if not os.path.exists('flx'):
os.mkdir('flx')
extfile = pysaltpath + '/data/site/suth_extinct.dat'
stdfiles = glob(stdsfolder + '/std/*sens*c?.dat')
print(stdfiles)
for f in fs:
outfile = f.replace('x1d', 'flx')
chip = outfile[-6]
hdu = pyfits.open(f)
ga = f.split('/')[1][3:8]
# Get the standard sensfunc with the same grating angle
stdfile = None
for stdf in stdfiles:
if ga in stdf:
# Get the right chip number
if chip == stdf[-5]:
stdfile = stdf
break
if stdfile is None:
print('No standard star with grating-angle %s' % ga)
continue
# for each extracted aperture
for i in range(hdu[0].data.shape[1]):
# create an ascii file that pysalt can read
asciiname = 'flx/sciflx.dat'
outtmpname = 'flx/scical.dat'
spectoascii(f, asciiname, i)
# Run pysalt.speccal
iraf.unlearn(iraf.speccal)
iraf.flpr()
iraf.speccal(asciiname, outtmpname, stdfile, extfile,
airmass=pyfits.getval(f, 'AIRMASS'),
exptime=pyfits.getval(f, 'EXPTIME'),
clobber=True, mode='h')
# read in the flux calibrated ascii file and copy its
# contents into a fits file
flxcal = np.genfromtxt(outtmpname).transpose()
hdu[0].data[0, i] = flxcal[1]
hdu[0].data[2, i] = flxcal[2]
# delete the ascii file
os.remove(asciiname)
os.remove(outtmpname)
hdu.writeto(outfile, clobber=True)
iraf.cd('..')
def mosaic(fs=None):
iraf.cd('work')
# If the file list is not given, grab the default files
if fs is None:
fs = glob('flts/*.fits')
# Abort if there are no files
if len(fs) == 0:
print "WARNING: No flat-fielded images to mosaic."
iraf.cd('..')
return
if not os.path.exists('mos'):
os.mkdir('mos')
# Get the images to work with
ims, gas = get_scis_and_arcs(fs)
for i, f in enumerate(ims):
ga = gas[i]
fname = f.split('/')[1]
typestr = fname[:3]
# by our naming convention, imnum should be the last 4 characters
# before the '.fits'
imnum = fname[-9:-5]
outname = 'mos/' + typestr
outname += '%05.2fmos%04i.fits' % (float(ga), int(imnum))
# prepare to run saltmosaic
iraf.unlearn(iraf.saltmosaic)
iraf.flpr()
iraf.saltmosaic(images=f,
outimages=outname,
outpref='',
geomfile=pysaltpath + '/data/rss/RSSgeom.dat',
clobber=True,
mode='h')
# Make a bad pixel mask marking where there is no data.
h = pyfits.open(outname, 'update')
maskim = h[1].data.copy()
maskim[:, :] = 0.0
maskim[abs(h[1].data) < 1e-5] = 1
imhdu = pyfits.ImageHDU(maskim)
h.append(imhdu)
h[1].header['BPMEXT'] = 2
h[2].header['EXTNAME'] = 'BPM'
h[2].header['CD2_2'] = 1
h.flush()
h.close()
iraf.cd('..')
def background(fs=None):
iraf.cd('work')
# Get rectified science images
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images for background-subtraction."
iraf.cd('..')
return
if not os.path.exists('bkg'):
os.mkdir('bkg')
for f in fs:
print("Subtracting background for %s" % f)
# Make sure dispaxis is set correctly
pyfits.setval(f, 'DISPAXIS', value=1)
# the outfile name is very similar, just change folder prefix and
# 3-char stage substring
outfile = f.replace('nrm','bkg')
# We are going to use fit1d instead of the background task
# Go look at the code for the background task: it is literally a wrapper for 1D
# but it removes the BPM option. Annoying.
iraf.unlearn(iraf.fit1d)
iraf.fit1d(input=f + '[SCI]', output='tmpbkg.fits', bpm=f + '[BPM]',
type='difference', sample='52:949', axis=2,
interactive='no', naverage='1', function='legendre',
order=5, low_reject=1.0, high_reject=1.0, niterate=5,
grow=0.0, mode='hl')
# Copy the background subtracted frame into the rectified image
# structure.
# Save the sky spectrum as extension 3
hdutmp = pyfits.open('tmpbkg.fits')
hdu = pyfits.open(f)
skydata = hdu[1].data - hdutmp[0].data
hdu[1].data[:, :] = hdutmp[0].data[:, :]
hdu.append(pyfits.ImageHDU(skydata))
hdu[3].header['EXTNAME'] = 'SKY'
hdu[3].data[hdu['BPM'] == 1] = 0.0
# Add back in the median sky level for things like apall and lacosmicx
hdu[1].data[:, :] += np.median(skydata)
hdu[1].data[hdu['BPM'] == 1] = 0.0
hdutmp.close()
hdu.writeto(outfile, clobber=True) # saving the updated file
# (data changed)
os.remove('tmpbkg.fits')
iraf.cd('..')
def background(fs=None):
iraf.cd('work')
# Get rectified science images
if fs is None:
fs = glob('nrm/sci*nrm*.fits')
if len(fs) == 0:
print "WARNING: No rectified images for background-subtraction."
iraf.cd('..')
return
if not os.path.exists('bkg'):
os.mkdir('bkg')
for f in fs:
print("Subtracting background for %s" % f)
# Make sure dispaxis is set correctly
pyfits.setval(f, 'DISPAXIS', value=1)
# the outfile name is very similar, just change folder prefix and
# 3-char stage substring
outfile = f.replace('nrm','bkg')
# We are going to use fit1d instead of the background task
# Go look at the code for the background task: it is literally a wrapper for 1D
# but it removes the BPM option. Annoying.
iraf.unlearn(iraf.fit1d)
iraf.fit1d(input=f + '[SCI]', output='tmpbkg.fits', bpm=f + '[BPM]',
type='difference', sample='52:949', axis=2,
interactive='no', naverage='1', function='legendre',
order=5, low_reject=1.0, high_reject=1.0, niterate=5,
grow=0.0, mode='hl')
# Copy the background subtracted frame into the rectified image
# structure.
# Save the sky spectrum as extension 3
hdutmp = pyfits.open('tmpbkg.fits')
hdu = pyfits.open(f)
skydata = hdu[1].data - hdutmp[0].data
hdu[1].data[:, :] = hdutmp[0].data[:, :]
hdu.append(pyfits.ImageHDU(skydata))
hdu[3].header['EXTNAME'] = 'SKY'
hdu[3].data[hdu['BPM'] == 1] = 0.0
# Add back in the median sky level for things like apall and lacosmicx
hdu[1].data[:, :] += np.median(skydata)
hdu[1].data[hdu['BPM'] == 1] = 0.0
hdutmp.close()
hdu.writeto(outfile, clobber=True) # saving the updated file
# (data changed)
os.remove('tmpbkg.fits')
iraf.cd('..')
def mosaic(fs=None):
iraf.cd('work')
# If the file list is not given, grab the default files
if fs is None:
fs = glob('flts/*.fits')
# Abort if there are no files
if len(fs) == 0:
print "WARNING: No flat-fielded images to mosaic."
iraf.cd('..')
return
if not os.path.exists('mos'):
os.mkdir('mos')
# Get the images to work with
ims, gas = get_scis_and_arcs(fs)
for i, f in enumerate(ims):
ga = gas[i]
fname = f.split('/')[1]
typestr = fname[:3]
# by our naming convention, imnum should be the last 4 characters
# before the '.fits'
imnum = fname[-9:-5]
outname = 'mos/' + typestr
outname += '%05.2fmos%04i.fits' % (float(ga), int(imnum))
# prepare to run saltmosaic
iraf.unlearn(iraf.saltmosaic)
iraf.flpr()
iraf.saltmosaic(images=f, outimages=outname, outpref='',
geomfile=pysaltpath + '/data/rss/RSSgeom.dat',
clobber=True, mode='h')
# Make a bad pixel mask marking where there is no data.
h = pyfits.open(outname, 'update')
maskim = h[1].data.copy()
maskim[:, :] = 0.0
maskim[abs(h[1].data) < 1e-5] = 1
imhdu = pyfits.ImageHDU(maskim)
h.append(imhdu)
h[1].header['BPMEXT'] = 2
h[2].header['EXTNAME'] = 'BPM'
h[2].header['CD2_2'] = 1
h.flush()
h.close()
iraf.cd('..')
def identify2d(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('mos/arc*mos*.fits')
if len(fs) == 0:
print "WARNING: No mosaiced (2D) specidentify."
# Change directories to fail gracefully
iraf.cd('..')
return
arcfs, arcgas = get_ims(fs, 'arc')
if not os.path.exists('id2'):
os.mkdir('id2')
lampfiles = {'Th Ar': 'ThAr.salt', 'Xe': 'Xe.salt', 'Ne': 'NeAr.salt',
'Cu Ar': 'CuAr.salt', 'Ar': 'Argon_hires.salt',
'Hg Ar': 'HgAr.salt'}
for i, f in enumerate(arcfs):
ga = arcgas[i]
# find lamp and corresponding linelist
lamp = pyfits.getval(f, 'LAMPID')
lampfn = lampfiles[lamp]
if pyfits.getval(f,'GRATING') == 'PG0300' and lamp == 'Ar':
lampfn = 'Argon_lores.swj'
ccdsum = int(pyfits.getval(f, 'CCDSUM').split()[1])
# linelistpath is a global variable defined in beginning, path to
# where the line lists are.
lamplines = pysaltpath + '/data/linelists/' + lampfn
print(lamplines)
# img num should be right before the .fits
imgnum = f[-9:-5]
# run pysalt specidentify
idfile = 'id2/arc%05.2fid2%04i' % (float(ga), int(imgnum)) + '.db'
iraf.unlearn(iraf.specidentify)
iraf.flpr()
iraf.specidentify(images=f, linelist=lamplines, outfile=idfile,
guesstype='rss', inter=True, # automethod='FitXcor',
rstep=600 / ccdsum,
rstart=200 / ccdsum, startext=1, clobber='yes',
#startext=1, clobber='yes',
verbose='no', mode='hl', logfile='salt.log',
mdiff=2, function='legendre')
iraf.cd('..')
def slitnormalize(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('rec/*rec*.fits')
if len(fs) == 0:
print "WARNING: No rectified files for slitnormalize."
# Change directories to fail gracefully
iraf.cd('..')
return
if not os.path.exists('nrm'):
os.mkdir('nrm')
for f in fs:
outname = f.replace('rec', 'nrm')
iraf.unlearn(iraf.specslitnormalize)
iraf.specslitnormalize(images=f, outimages=outname, outpref='',
order=5, clobber=True, mode='h')
iraf.cd('..')
def slitnormalize(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('rec/*rec*.fits')
if len(fs) == 0:
print "WARNING: No rectified files for slitnormalize."
# Change directories to fail gracefully
iraf.cd('..')
return
if not os.path.exists('nrm'):
os.mkdir('nrm')
for f in fs:
outname = f.replace('rec', 'nrm')
iraf.unlearn(iraf.specslitnormalize)
iraf.specslitnormalize(images=f, outimages=outname, outpref='',
order=5, clobber=True, mode='h')
iraf.cd('..')
def telluric(stdsfolder='./', fs=None):
iraf.cd('work')
if fs is None:
fs = glob('sci_com.fits')
if len(fs) == 0:
print "WARNING: No flux-calibrated spectra to telluric-correct."
iraf.cd('..')
return
f = fs[0]
outfile = 'final.fits'
# Get the standard to use for telluric correction
stdfile = glob(stdsfolder + '/tel/telcor.dat')[0]
hdu = pyfits.open(f)
spec = hdu[0].data.copy()
hdr = hdu[0].header.copy()
hdu.close()
waves = fitshdr_to_wave(hdr)
telwave, telspec = np.genfromtxt(stdfile).transpose()
# Cross-correlate the standard star and the sci spectra
# to find wavelength shift of standard star.
p = fitxcor(waves, spec, telwave, telspec)
if abs(p[0] - 1.0) > 0.02 or abs(p[1]) > 10.0:
print "Cross-correlation scale/shift too large; won't do it:"
print p
print " reset to [1.0, 0.0]"
p = [1.0, 0.0]
# shift and stretch standard star spectrum to match science
# spectrum.
telcorr = interp(waves, p[0] * telwave + p[1], telspec)
# In principle, we should scale by the proper airmasses, but SALT
# always observes at ~same airmass
# Divide science spectrum by transformed standard star sub-spectrum
correct_spec = spec / telcorr
# Copy telluric-corrected data to new file.
tofits(outfile, correct_spec, hdr=hdr, clobber=True)
print "Telluric correction applied; output is " + outfile
iraf.cd('..')
def telluric(stdsfolder='./', fs=None):
iraf.cd('work')
if fs is None:
fs = glob('sci_com.fits')
if len(fs) == 0:
print "WARNING: No flux-calibrated spectra to telluric-correct."
iraf.cd('..')
return
f = fs[0]
outfile = 'final.fits'
# Get the standard to use for telluric correction
stdfile = glob(stdsfolder + '/tel/telcor.dat')[0]
hdu = pyfits.open(f)
spec = hdu[0].data.copy()
hdr = hdu[0].header.copy()
hdu.close()
waves = fitshdr_to_wave(hdr)
telwave, telspec = np.genfromtxt(stdfile).transpose()
# Cross-correlate the standard star and the sci spectra
# to find wavelength shift of standard star.
p = fitxcor(waves, spec, telwave, telspec)
if abs(p[0] - 1.0) > 0.02 or abs(p[1]) > 10.0:
print "Cross-correlation scale/shift too large; won't do it:"
print p
print " reset to [1.0, 0.0]"
p = [1.0, 0.0]
# shift and stretch standard star spectrum to match science
# spectrum.
telcorr = interp(waves, p[0] * telwave + p[1], telspec)
# In principle, we should scale by the proper airmasses, but SALT
# always observes at ~same airmass
# Divide science spectrum by transformed standard star sub-spectrum
correct_spec = spec / telcorr
# Copy telluric-corrected data to new file.
tofits(outfile, correct_spec, hdr=hdr, clobber=True)
print "Telluric correction applied; output is " + outfile
iraf.cd('..')
def rectify(ids=None, fs=None):
iraf.cd('work')
if ids is None:
ids = np.array(glob('id2/arc*id2*.db'))
if fs is None:
fs = glob('srt/*mos*.fits')
if len(ids) == 0:
print "WARNING: No wavelength solutions for rectification."
iraf.cd('..')
return
if len(fs) == 0:
print "WARNING: No images for rectification."
iraf.cd('..')
return
# Get the grating angles of the solution files
idgas = []
for i, thisid in enumerate(ids):
f = open(thisid)
idlines = np.array(f.readlines(), dtype=str)
f.close()
idgaline = idlines[np.char.startswith(idlines, '#graang')][0]
idgas.append(float(idgaline.split('=')[1]))
ims, gas = get_scis_and_arcs(fs)
if not os.path.exists('rec'):
os.mkdir('rec')
for i, f in enumerate(ims):
fname = f.split('/')[1]
typestr = fname[:3]
ga, imgnum = gas[i], fname[-9:-5]
outfile = 'rec/' + typestr + '%05.2frec' % (ga) + imgnum + '.fits'
iraf.unlearn(iraf.specrectify)
iraf.flpr()
print('_____idgas_____')
print (np.array(idgas))
print('_____ga_____')
print (ga)
idfile = ids[np.array(idgas) == ga][0]
iraf.specrectify(images=f, outimages=outfile, solfile=idfile,
outpref='', function='legendre', order=3,
inttype='interp', conserve='yes', clobber='yes',
verbose='yes')
# Update the BPM to mask any blank regions
h = pyfits.open(outfile, 'update')
# Cover the chip gaps. The background task etc do better if the chip
# gaps are straight
# To deal with this we just throw away the min and max of each side of
# the curved chip gap
chipgaps = get_chipgaps(h)
print(" -- chipgaps --")
print(chipgaps)
# Chip 1
h[2].data[:, chipgaps[0][0]:chipgaps[0][1]] = 1
# Chip 2
h[2].data[:, chipgaps[1][0]:chipgaps[1][1]] = 1
# edge of chip 3
h[2].data[:, chipgaps[2][0]:chipgaps[2][1]] = 1
# Cover the other blank regions
h[2].data[[h[1].data == 0]] = 1
# Set all of the data to zero in the BPM
h[1].data[h[2].data == 1] = 0.0
h.flush()
h.close()
iraf.cd('..')
def pysalt(fs=None):
# Run the pysalt pipeline on the raw data.
if fs is None:
fs = glob('P*.fits')
if len(fs) == 0:
print "WARNING: No raw files to run PySALT pre-processing."
return
# Copy the raw files into a raw directory
if not os.path.exists('raw'):
os.mkdir('raw')
if not os.path.exists('work'):
os.mkdir('work')
for f in fs:
shutil.copy2(f, 'raw/')
shutil.move(f, 'work/')
iraf.cd('work')
# Run each of the pysalt pipeline steps deleting temporary files as we go
# saltprepare
iraf.unlearn(iraf.saltprepare)
# Currently, there is not a bad pixel mask provided by SALT
# so we don't create one here.
iraf.saltprepare(images='P*.fits', clobber=True, mode='h')
for f in glob('P*.fits'):
os.remove(f)
# saltgain
iraf.unlearn(iraf.saltgain)
# Multiply by the gain so that everything is in electrons.
iraf.saltgain(images='pP*.fits',
gaindb=pysaltpath + '/data/rss/RSSamps.dat',
mult=True, usedb=True, mode='h')
for f in glob('pP*.fits'):
os.remove(f)
# write a keyword in the header keyword gain = 1 in each amplifier
fs = glob('gpP*.fits')
for f in fs:
for i in range(1, 7):
pyfits.setval(f, 'GAIN', ext=i, value=1.0)
# saltxtalk
iraf.unlearn(iraf.saltxtalk)
iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
for f in glob('gpP*.fits'):
os.remove(f)
# saltbias
iraf.unlearn(iraf.saltbias)
iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
for f in glob('xgpP*.fits'):
os.remove(f)
# Put all of the newly created files into the pysalt directory
if not os.path.exists('pysalt'):
os.mkdir('pysalt')
for f in glob('bxgpP*.fits'):
shutil.move(f, 'pysalt')
iraf.cd('..')
#import pyfits
import astropy.io.fits as pyfits
from pyraf import iraf
iraf.imexamine.unlearn()
# If zenity is not working on you machine, use the cat command instead by uncommnting it.
DISPLAY_MENU_CMD = 'zenity --text-info --title "Images in {imgdir}" --filename={imglist} --width 600 --height 250 &'
#DISPLAY_MENU_CMD = 'echo "Images in {imgdir}" ; cat {imglist}'
IMGListFile = 'imgs.text' # This file will be overwritten
pwd=os.getcwd()
wildcard=sys.argv[-1]
dirs=sys.argv[1:-1]
for imgdir in dirs :
iraf.cd(imgdir)
while 1 :
print imgdir
images=glob.glob(wildcard)
images.sort()
imgs_txt=open(IMGListFile,'w')
i=0
for img in images:
hdulist=pyfits.open(img)
Exptime=hdulist[0].header.get('ITIME')
Commentx=hdulist[0].header.get('TCOMMENT')
Obj=hdulist[0].header.get('TARGET')
Comment=hdulist[0].header.get('DATE')
Filter=hdulist[0].header.get('UPPER')
hdulist.close()
imgs_txt.write(str(i)+' '+img+' '+str(Obj)+' '+str(Exptime)+' '+str(Commentx)+' '+str(Comment)+' '+str(Filter)+'\n')
# These statements import python modules that will be used below...
import glob
import os
import string
from pyraf import iraf
from shutil import copyfile
# Load necessary IRAF (pyraf) modules...
iraf.stsdas()
iraf.hst_calib()
iraf.synph
# cd to current working directory
iraf.cd(os.getcwd())
# This creates a list of filter file names...
print "Here is the list of CCDs:"
filterFileNameList = glob.glob('des_band_20150601a_syn.average.list')
filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*.list')
filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*_amp1.list')
filterFileNameList += glob.glob('des_band_20150601a_syn.ccd*_amp2.list')
filterFileNameList.sort()
print filterFileNameList
# This is the file containing the list of calspec files,
# which you created previously using the pyraf "files"
# command (see top of this script)...
calspecListFileName = 'pickles.list'
# Open and the calspec list file...
fd = open(calspecListFileName, 'r')
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
iraf.centerpars.maxshift = 1.
# now test visually in ds9 for images that have not been properly aligned.
# put info for those which have not been properly aligned into the manual wcs correction file.
if input_info.badwcs == "yes":
corrbyhand = ascii.read(input_info.wcs_corrbyhand_file)
for i in np.arange(0, len(corrbyhand)):
# remove file if beyond saving
if corrbyhand[i]["beyond_saving"] == "yes":
print("removing files which are beyond saving...")
if os.path.isfile(corrbyhand[i]["filename_incl_path"]): os.remove(corrbyhand[i]["filename_incl_path"])
else:
# if saveable, update header with manual coordinates.
print("updating headers with manual coordinates...")
iraf.cd(os.path.dirname(corrbyhand[i]["filename_incl_path"]))
f = os.path.basename(corrbyhand[i]["filename_incl_path"])
iraf.hedit(f, "CRVAL1", corrbyhand[i]["ra_good"], verify="no")
iraf.hedit(f, "CRPIX1", corrbyhand[i]["x_pixel"], verify="no")
iraf.hedit(f, "CRVAL2", corrbyhand[i]["dec_good"], verify="no")
iraf.hedit(f, "CRPIX2", corrbyhand[i]["y_pixel"], verify="no")
# run the automatic coordinate correction again:
print("running refined coordinate matching for updated files...")
iraf.centerpars.cbox = 10.
iraf.centerpars.maxshift = 10.
photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.)
photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.)
photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.)
photometry.correct_coordinates([corrbyhand[i]["filename_incl_path"]], radius=10., twomassmag = 17.)
iraf.centerpars.cbox = 9.
iraf.centerpars.maxshift = 1.
shift_temp.append([name_temp[id2], 0., 0., 0.])
#f = open(path+galname+'/shift_pix_'+targname+'_lam'+str(lam_temp[id2])+'_n'+str(id1)+'.txt','a')
#f.write('\n'+str(0.)+' '+str(0.)+' '+str(refpa)+' '+str(name_temp[id2]))
#f.close()
shift_fin.append(shift_temp)
shift_temp = []
######################################################### IMSHIFT + IMCOMBINE ############################################################################
#
# Perform the shifts with iraf/pyraf
# We need the imshift task: imshift input output xshift yshift (or file with shifts in cols 1 & 2)
# The initial list will be *x2d.fits[1] and the output list will be *shift.fits
#
list_temp = []
list_total = []
iraf.cd(path + galname + '/shifts')
for index in range(len(shift_fin)):
arr1 = shift_fin[index]
for index2 in range(len(arr1)):
i_nam = arr1[index2][0] # Initial name for imshift
xs = arr1[index2][1]
ys = arr1[index2][2]
posa = arr1[index2][3]
list_in = i_nam.replace('.fits', '.fits[1,overwrite+]')
list_ot = list_in.replace('x2d', 'shift')
list_out = list_ot.replace('.fits[1,overwrite+]', '.fits')
list_temp.append(list_out)
if not os.path.exists(list_out):
iraf.imshift(input=list_in, output=list_out, xshift=xs, yshift=ys)
print('File ' + str(i_nam) + ' changed to ' + str(list_out) + '!')
list_total.append(list_temp)
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-200:-100,100:200', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=200, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit2d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
def pysalt(fs=None):
# Run the pysalt pipeline on the raw data.
if fs is None:
fs = glob('P*.fits')
if len(fs) == 0:
print "WARNING: No raw files to run PySALT pre-processing."
return
# Copy the raw files into a raw directory
if not os.path.exists('raw'):
os.mkdir('raw')
if not os.path.exists('work'):
os.mkdir('work')
for f in fs:
shutil.copy2(f, 'raw/')
shutil.move(f, 'work/')
iraf.cd('work')
# Run each of the pysalt pipeline steps deleting temporary files as we go
# saltprepare
iraf.unlearn(iraf.saltprepare)
# Currently, there is not a bad pixel mask provided by SALT
# so we don't create one here.
iraf.saltprepare(images='P*.fits', clobber=True, mode='h')
for f in glob('P*.fits'):
os.remove(f)
# saltgain
iraf.unlearn(iraf.saltgain)
# Multiply by the gain so that everything is in electrons.
iraf.saltgain(images='pP*.fits',
gaindb=pysaltpath + '/data/rss/RSSamps.dat',
mult=True, usedb=True, mode='h')
for f in glob('pP*.fits'):
os.remove(f)
# write a keyword in the header keyword gain = 1 in each amplifier
fs = glob('gpP*.fits')
for f in fs:
for i in range(1, 7):
pyfits.setval(f, 'GAIN', ext=i, value=1.0)
# saltxtalk
iraf.unlearn(iraf.saltxtalk)
iraf.saltxtalk(images='gpP*.fits', clobber=True, usedb=True,
xtalkfile=pysaltpath + '/data/rss/RSSxtalk.dat', mode='h')
for f in glob('gpP*.fits'):
os.remove(f)
# saltbias
iraf.unlearn(iraf.saltbias)
iraf.saltbias(images='xgpP*.fits', clobber=True, mode='h')
for f in glob('xgpP*.fits'):
os.remove(f)
# Put all of the newly created files into the pysalt directory
if not os.path.exists('pysalt'):
os.mkdir('pysalt')
for f in glob('bxgpP*.fits'):
shutil.move(f, 'pysalt')
iraf.cd('..')
def extract(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('fix/*fix*.fits')
if len(fs) == 0:
print "WARNING: No fixpixed images available for extraction."
iraf.cd('..')
return
if not os.path.exists('x1d'):
os.mkdir('x1d')
print "Note: No continuum? Make nsum small (~-5) with 'line' centered on an emission line."
for f in fs:
# Get the output filename without the ".fits"
outbase = f.replace('fix', 'x1d')[:-5]
# Get the readnoise, right now assume default value of 5 but we could
# get this from the header
readnoise = 5
# If interactive open the rectified, background subtracted image in ds9
ds9display(f.replace('fix', 'bkg'))
# set dispaxis = 1 just in case
pyfits.setval(f, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apall)
iraf.flpr()
iraf.apall(input=f + '[SCI]', output=outbase, interactive='yes',
review='no', line='INDEF', nsum=-1000, lower=-5, upper=5,
b_function='legendre', b_order=5,
b_sample='-400:-200,200:400', b_naverage=-10, b_niterate=5,
b_low_reject=3.0, b_high_reject=3.0, nfind=1, t_nsum=15,
t_step=15, t_nlost=100, t_function='legendre', t_order=5,
t_niterate=5, t_low_reject=3.0, t_high_reject=3.0,
background='fit', weights='variance', pfit='fit1d',
clean='no', readnoise=readnoise, gain=1.0, lsigma=4.0,
usigma=4.0, mode='hl')
# Copy the CCDSUM keyword into the 1d extraction
pyfits.setval(outbase + '.fits', 'CCDSUM',
value=pyfits.getval(f, 'CCDSUM'))
# Extract the corresponding arc
arcname = glob('nrm/arc' + f.split('/')[1][3:8] + '*.fits')[0]
# set dispaxis = 1 just in case
pyfits.setval(arcname, 'DISPAXIS', extname='SCI', value=1)
iraf.unlearn(iraf.apsum)
iraf.flpr()
iraf.apsum(input=arcname + '[SCI]', output='auxext_arc',
references=f[:-5] + '[SCI]', interactive='no', find='no',
edit='no', trace='no', fittrace='no', extras='no',
review='no', background='no', mode='hl')
# copy the arc into the 5 column of the data cube
arcfs = glob('auxext_arc*.fits')
for af in arcfs:
archdu = pyfits.open(af)
scihdu = pyfits.open(outbase + '.fits', mode='update')
d = scihdu[0].data.copy()
scihdu[0].data = np.zeros((5, d.shape[1], d.shape[2]))
scihdu[0].data[:-1, :, :] = d[:, :, :]
scihdu[0].data[-1::, :] = archdu[0].data.copy()
scihdu.flush()
scihdu.close()
archdu.close()
os.remove(af)
# Add the airmass, exptime, and other keywords back into the
# extracted spectrum header
kws = ['AIRMASS','EXPTIME',
'PROPID','PROPOSER','OBSERVER','OBSERVAT','SITELAT','SITELONG',
'INSTRUME','DETSWV','RA','PM-RA','DEC','PM-DEC','EQUINOX',
'EPOCH','DATE-OBS','TIME-OBS','UTC-OBS','TIMESYS','LST-OBS',
'JD','MOONANG','OBSMODE','DETMODE','SITEELEV','BLOCKID','PA',
'TELHA','TELRA','TELDEC','TELPA','TELAZ','TELALT','DECPANGL',
'TELTEM','PAYLTEM','MASKID','MASKTYP','GR-ANGLE','GRATING',
'FILTER']
for kw in kws:
pyfits.setval(outbase + '.fits', kw, value=pyfits.getval(f,kw))
iraf.cd('..')
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('flx/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
w = WCS(f)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=-1e-17)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd)
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet)
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam)
print "avg AIRMASS = " + str(avgam)
iraf.cd('..')
return specs
@author: cmccully ''' import os, shutil from glob import glob import pyfits import numpy as np from astroscrappy import detect_cosmics from pyraf import iraf from scipy import interpolate, ndimage, signal, optimize import pf_model as pfm import statsmodels as sm from astropy.modeling import models, fitting import astropy iraf.cd(os.getcwd()) iraf.gemini() iraf.gmos() iraf.onedspec() bluecut = 3450 iraf.gmos.logfile = "log.txt" iraf.gmos.mode = 'h' iraf.set(clobber='yes') iraf.set(stdimage='imtgmos') dooverscan = False is_GS = False
def extract(arcs, objs,ccd,mode=1):
"""
mode=1: 1D extraction
mode=2: 2D extraction
"""
#apall na vseh luckah
print " + identifing arcs"
for i in arcs:
iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
if mode==1:
pass
else:
iraf.cd("tmp/arcs")
geometry_prep(arcs,ccd)
os.system("cp transformations* ../objs/")
#sys.exit(0)
iraf.cd("../..")
for i in arcs:
os.system("rm tmp/arcs/%s.ms.fits" % (i[1][:-5]))
os.system("rm tmp/arcs/%s_cut*.fits" % (i[1][:-5]))
os.system("rm tmp/arcs/%s_t*.fits" % (i[1][:-5]))
pass
os.system("rm tmp/arcs/calibrations/idarcs_cut*")
os.system("rm tmp/arcs/arcs_cut*")
#extract 2d arcs and objects
for i in arcs:
iraf.hedit(images='tmp/arcs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
#forget apertures
for j in range(8,1000):
iraf.hedit(images='tmp/arcs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null")
for i in objs:
iraf.hedit(images='tmp/objs/'+i[1], fields='DISPAXIS', value=1, add='yes', verify='no', update='yes',Stdout="/dev/null")
iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=7, pfit="fit1d", Stdout="/dev/null")
#forget apertures
for j in range(8,1000):
iraf.hedit(images='tmp/objs/%s.ms.fits' % (i[1][:-5]), fields="APNUM%s" % (str(j)), value='', delete="yes", verify="no", Stdout="/dev/null")
iraf.cd("tmp/arcs")
for ii in arcs:
geometry_transform(ii)
iraf.cd("../..")
iraf.cd("tmp/objs")
for ii in objs:
geometry_transform(ii)
iraf.cd("../..")
#make normal 1d extraction and copy results into it
for i in arcs:
os.system("rm -f tmp/arcs/%s" % (i[1][:-5]+".ms.fits"))
iraf.apall(input='tmp/arcs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null")
os.system("cp tmp/arcs/%s" % (i[1][:-5])+".ms.fits tmp/arcs/%s" % (i[1][:-5])+".ms2.fits")
if mode==1:
pass
else:
for j in range(1,393):
os.system("rm -f tmp/copy_tmp.fits")
try:
iraf.blkavg(input="tmp/arcs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null")
iraf.imcopy(input="tmp/copy_tmp", output="tmp/arcs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null")
except:
pass
for i in objs:
os.system("rm -f tmp/objs/%s" % (i[1][:-5]+".ms.fits"))
iraf.apall(input='tmp/objs/%s' % (i[1]), referen='tmp/masterflat.fits', format='multispec', interac='no', find='no', recenter='no', resize='no', edit='no', trace='no', fittrac='no', extract='yes', extras='no', review='yes', lower=-3.0, upper=3.0, nsubaps=1, pfit="fit1d", Stdout="/dev/null")
os.system("cp tmp/objs/%s" % (i[1][:-5])+".ms.fits tmp/objs/%s" % (i[1][:-5])+".ms2.fits")
if mode==1:
pass
else:
for j in range(1,393):
os.system("rm -f tmp/copy_tmp.fits")
try:
iraf.blkavg(input="tmp/objs/"+i[1][:-5]+"_t%s.fits" % (str(j)), output="tmp/copy_tmp", option='sum', b1=1, b2=7, Stdout="/dev/null")
iraf.imcopy(input="tmp/copy_tmp", output="tmp/objs/"+i[1][:-5]+".ms.fits[*,%s]" % (j), Stdout="/dev/null")
except:
pass
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('flx/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
w = WCS(f)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=1e-19)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GRATING','GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd)
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet)
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam)
print "avg AIRMASS = " + str(avgam)
iraf.cd('..')
return specs
def speccombine(fs=None):
iraf.cd('work')
if fs is None:
fs = glob('trm/sci*c?.fits')
if len(fs)==0:
print("No flux calibrated images to combine.")
iraf.cd('..')
return
#diagnostic()
nsteps = 8001
lamgrid = np.linspace(2000.0, 10000.0, nsteps)
nfs = len(fs)
# for each aperture
# get all of the science images
specs = np.zeros((nfs, nsteps))
specerrs = np.zeros((nfs, nsteps))
ap = 0
for i, f in enumerate(fs):
hdu = pyfits.open(f)
# print ('---hdu.data---')
# print (hdu[0].data)
w=WCS(f)
# print ('-----w-----')
# print(w)
# get the wavelengths of the pixels
npix = hdu[0].data.shape[2]
# print('-----npix-----')
# print(npix)
lam = w.all_pix2world(np.linspace(0, npix - 1, npix), 0, 0, 0)[0]
# print('-----lam-----')
# print(lam)
# interpolate each spectrum onto a comman wavelength scale
specs[i] = interp(lamgrid, lam, hdu[0].data[0][ap],
left=0.0, right=0.0)
# Also calculate the errors. Right now we assume that the variances
# interpolate linearly. This is not stricly correct but it should be
# close. Also we don't include terms in the variance for the
# uncertainty in the wavelength solution.
specerrs[i] = interp(lamgrid, lam, hdu[0].data[3][ap] ** 2.0) ** 0.5
#print ('-----specs-----')
#print (specs)
# minimize the chi^2 given free parameters are multiplicative factors
# We could use linear or quadratic, but for now assume constant
p0 = np.ones(nfs)
results = optimize.minimize(combine_spec_chi2, p0,
args=(lamgrid, specs, specerrs),
method='Nelder-Mead',
options={'maxfev': 1e5, 'maxiter': 1e5})
# write the best fit parameters into the headers of the files
# Dump the list of spectra into a string that iraf can handle
iraf_filelist = str(fs).replace('[', '').replace(']', '').replace("'", '')
# write the best fit results into a file
lines = []
for p in results['x']:
lines.append('%f\n' % (1.0 / p))
f = open('flx/scales.dat', 'w')
f.writelines(lines)
f.close()
# run scombine after multiplying the spectra by the best fit parameters
combfile = 'sci_com.fits'
if os.path.exists(combfile):
os.remove(combfile)
iraf.scombine(iraf_filelist, combfile, scale='@flx/scales.dat',
reject='avsigclip', lthreshold=-2e-16)
# Remove the other apertures [TBD]
# remove the sky and arc bands from the combined spectra. (or add back?? TBD)
# remove some header keywords that don't make sense in the combined file
delkws = ['GR-ANGLE','FILTER','BANDID2','BANDID3','BANDID4']
for kw in delkws:
pyfits.delval(combfile,kw)
# combine JD (average), AIRMASS (average), EXPTIME (sum)
# we assume there is a c1.fits file for each image
c1fs = [f for f in fs if 'c1.fits' in f]
avgjd = np.mean([pyfits.getval(f,'JD') for f in c1fs])
pyfits.setval(combfile,'JD',value=avgjd, comment='average of multiple exposures')
print "average JD = " + str(avgjd)
sumet = np.sum([pyfits.getval(f,'EXPTIME') for f in c1fs])
pyfits.setval(combfile,'EXPTIME',value=sumet,comment='sum of multiple exposures')
print "total EXPTIME = " + str(sumet)
avgam = np.mean([pyfits.getval(f,'AIRMASS') for f in c1fs])
pyfits.setval(combfile,'AIRMASS',value=avgam,comment='average of multiple exposures')
print "avg AIRMASS = " + str(avgam)
# update this to used avg jd midpoint of all exposures?
print "barycentric velocity correction (km/s) = ",
iraf.bcvcorr(spectra=combfile,keytime='UTC-OBS',keywhen='mid',
obslong="339:11:16.8",obslat="-32:22:46.2",obsalt='1798',obsname='saao',
savebcv='yes',savejd='yes',printmode=2)
pyfits.setval(combfile,'UTMID',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'GJDN',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'HJDN',comment='added by RVSAO task BCVCORR')
pyfits.setval(combfile,'BCV',comment='added by RVSAO task BCVCORR (km/s)')
pyfits.setval(combfile,'HCV',comment='added by RVSAO task BCVCORR (km/s)')
iraf.dopcor(input=combfile,output='',redshift=-iraf.bcvcorr.bcv,isvelocity='yes',
add='no',dispersion='yes',flux='no',verbose='yes')
pyfits.setval(combfile,'DOPCOR01',comment='barycentric velocity correction applied')
iraf.cd('..')
def cd(target):
"""Changes to a directory, pyraf included"""
os.chdir(target)
iraf.cd(target)
def reduce_science(rawdir, rundir, flat, arc, twilight, twilight_flat, sciimg,
starimg, bias, overscan, vardq, observatory, lacos,
apply_lacos, lacos_xorder, lacos_yorder, lacos_sigclip,
lacos_objlim, bpm, instrument, slits, fl_gscrrej,
wltrim_frac, grow_gap, cube_bit_mask):
"""
Reduction pipeline for standard star.
Parameters
----------
rawdir: string
Directory containing raw images.
rundir: string
Directory where processed files are saved.
flat: string
Names of the files containing flat field images.
arc: string
Arc images.
twilight: string
Twilight flat images.
twilight_flat: string
Flat field for twilight image.
starimg: string
Name of the file containing the image to be reduced.
bias: string
Bias images.
grow_gap: number
Number of pixels by which to grow the bad pixel mask around
the chip gaps.
"""
iraf.set(stdimage='imtgmos')
iraf.gemini()
iraf.unlearn('gemini')
iraf.gmos()
iraf.unlearn('gmos')
iraf.gemtools()
iraf.unlearn('gemtools')
# os.path.isfile('arquivo')
iraf.unlearn('gemini')
iraf.unlearn('gmos')
iraf.task(lacos_spec=lacos)
# set directories
iraf.set(caldir=rawdir) #
iraf.set(rawdir=rawdir) # raw files
iraf.set(procdir=rundir) # processed files
iraf.gmos.logfile = 'logfile.log'
iraf.gfextract.verbose = 'no'
iraf.cd('procdir')
flat = flat.replace('.fits', '')
twilight = twilight.replace('.fits', '')
twilight_flat = twilight_flat.replace('.fits', '')
arc = arc.replace('.fits', '')
starimg = starimg.replace('.fits', '')
sciimg = sciimg.replace('.fits', '')
mdffile = 'mdf' + flat + '.fits'
iraf.gfreduce.bias = 'caldir$' + bias
iraf.gfreduce.fl_fulldq = 'yes'
iraf.gfreduce.fl_fixgaps = 'yes'
iraf.gfreduce.grow = grow_gap
iraf.gireduce.bpm = 'rawdir$' + bpm
iraf.gfextract.verbose = 'no'
cal_reduction(rawdir=rawdir,
rundir=rundir,
flat=flat,
arc=arc,
twilight=twilight,
bias=bias,
bpm=bpm,
overscan=overscan,
vardq=vardq,
instrument=instrument,
slits=slits,
twilight_flat=twilight_flat,
grow_gap=grow_gap)
#
# Actually reduce science
#
image_name = 'rg' + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfreduce(sciimg,
slits='header',
rawpath='rawdir$',
fl_inter='no',
fl_addmdf='yes',
key_mdf='MDF',
mdffile=mdffile,
weights='no',
fl_over=overscan,
fl_trim='yes',
fl_bias='yes',
trace='no',
recenter='no',
fl_fulldq='yes',
fl_flux='no',
fl_gscrrej='no',
fl_extract='no',
fl_gsappwave='no',
fl_wavtran='no',
fl_novl='yes',
fl_skysub='no',
fl_vardq=vardq,
mdfdir='procdir$')
prefix = 'rg'
# Gemfix
image_name = 'p' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gemfix(prefix + sciimg,
out='p' + prefix + sciimg,
method='fit1d',
bitmask=65535,
axis=1)
prefix = 'p' + prefix
# LA Cosmic - cosmic ray removal
if apply_lacos:
image_name = 'l' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gemcrspec(prefix + sciimg,
out='l' + prefix + sciimg,
sigfrac=0.5,
niter=4,
fl_vardq=vardq,
xorder=lacos_xorder,
yorder=lacos_yorder,
sigclip=lacos_sigclip,
objlim=lacos_objlim)
prefix = 'l' + prefix
if fl_gscrrej:
image_name = 'ex' + prefix + sciimg + '.fits'
else:
image_name = 'e' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfreduce(prefix + sciimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej=fl_gscrrej,
fl_extract='yes',
fl_gsappwave='yes',
fl_wavtran='no',
fl_novl='no',
fl_skysub='no',
grow=grow_gap,
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq,
fl_fulldq='yes',
fl_fixgaps='yes')
if fl_gscrrej:
prefix = 'ex' + prefix
else:
prefix = 'e' + prefix
# if wl2 > 7550.0:
# wl2 = 7550.0
#
# Apply wavelength transformation
#
wl1, wl2 = wl_lims(prefix + sciimg + '.fits', wltrim_frac)
image_name = 't' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gftransform(
prefix + sciimg,
wavtraname='eprg' + arc,
fl_vardq=vardq,
w1=wl1,
w2=wl2,
)
prefix = 't' + prefix
#
# Sky subtraction
#
image_name = 's' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gfskysub(
prefix + sciimg,
expr='default',
combine='median',
reject='avsigclip',
scale='none',
zero='none',
weight='none',
sepslits='yes',
fl_inter='no',
lsigma=1,
hsigma=1,
)
prefix = 's' + prefix
#
# Apply flux calibration to galaxy
#
image_name = 'c' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
iraf.gscalibrate(prefix + sciimg,
sfuncti=starimg,
extinct='onedstds$ctioextinct.dat',
observatory=observatory,
fluxsca=1,
fl_vardq=vardq)
prefix = 'c' + prefix
#
# Remove spurious data with PCA
#
image_name = 'x' + prefix + sciimg + '.fits'
print(os.getcwd())
print(image_name)
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
t = pca.Tomography(prefix + sciimg + '.fits')
t.decompose()
t.remove_cosmic_rays(sigma_threshold=10.0)
t.write(image_name)
prefix = 'x' + prefix
#
# Create data cubes
#
image_name = 'd' + prefix + sciimg + '.fits'
if os.path.isfile(image_name):
pipe.skipwarn(image_name)
else:
data_cube = CubeBuilder(prefix + sciimg + '.fits')
data_cube.build_cube()
data_cube.fit_refraction_function()
data_cube.fix_atmospheric_refraction()
data_cube.write(image_name)
import os,string,sys
import pylab
import numpy as numarray
from numpy import zeros,where,add,sqrt,array,any,nan,arange
thisdir = os.getcwd()
os.chdir('/home/durant')
from pyraf import iraf
os.chdir(thisdir)
iraf.cd(thisdir)
iraf.stsdas()
iraf.digiphot()
iraf.daophot()
import glob
import pyfits
import astLib
class star_photometry(object):
"""reads photometry information from IRAF .als or .mag file, and
returns information from it as arrays of magnitudes etc, including
conversions of co-ordinates etc.
self.data is a float array of the data, containing:
id,x,y,mag,magerr,chi,ra,dec.
self.image is the corresponding FITS file with WCS information
self.photfile is the original file
self.nstar is number of stars (=len(self.data))"""
mynames={"id":0,"x":1,"y":2,"mag":3,"magerr":4,
"sky":5,"chi":6,"ra":7,"dec":8} #record structure
def __init__(self,photfile,imagefile=None,calcradec=True):
"""load data from photfile, and store in self.stars,
associate with imagefile if it exists, or try to find it"""
def identify_edge(infile, overwrite=False):
print('\n#############################')
print('Identifying the edges.')
binfct1 = fits.getval(infile, 'BIN-FCT1')
coordlist = fi.filibdir + 'edge' + str(binfct1) + '.dat'
section = 'middle line'
verbose = 'yes'
nsum = 50
match = -10.
fwidth = 6. / binfct1
cradius = 20. / binfct1
threshold = 0.
function = 'chebyshev'
order = 2
niter = 0
autowrite = 'yes'
newaps = 'yes'
override = 'yes'
refit = 'no'
trace = 'yes'
step = 50
shift = 0
nlost = 0
minsep = 60. / binfct1
addfeatures = 'no'
database = 'database'
logfile = 'identify_edge.log'
# Not to display items in IRAF packages
sys.stdout = open('/dev/null', 'w')
iraf.noao()
iraf.twodspec()
iraf.longslit()
sys.stdout = sys.__stdout__ # Back to the stadard output
# entering the channel image directory.
# os.chdir() does not change the directory for pyraf only in this function.
print('\t Entering the channel image directory, \"' + fi.chimagedir +
'\".')
iraf.cd(fi.chimagedir)
basename = fits.getval('../' + infile, 'FRAMEID')
idfile = database + '/id' + basename + '.ch01edge'
if os.path.isfile(idfile) and not overwrite:
print('\t Edge identification files already exist, ' + idfile \
+ '. Skipping.')
else:
if os.path.isfile(idfile) and overwrite:
print('\t Removing ' + idfile)
try:
os.remove(idfile)
except:
pass
print('\t Identifying: ' + basename + '.ch01edge.fits')
iraf.identify(basename + '.ch01edge',
section=section,
database=database,
coordlist=coordlist,
units='',
nsum=nsum,
match=match,
maxfeat=2,
ftype='emission',
fwidth=fwidth,
cradius=cradius,
threshold=threshold,
function=function,
order=order,
sample='*',
niter=niter,
autowrite=autowrite)
print('\t Reidentifying: ' + basename + '.ch01edge.fits')
iraf.reidentify(basename + '.ch01edge',
basename + '.ch01edge',
interac='no',
section=section,
newaps=newaps,
override=override,
refit=refit,
trace=trace,
step=step,
nsum=nsum,
shift=shift,
nlost=nlost,
cradius=cradius,
threshold=threshold,
addfeatures=addfeatures,
coordlist=coordlist,
match=match,
maxfeat=2,
minsep=minsep,
database=database,
logfile=logfile,
plotfile='',
verbose=verbose,
cursor='')
for i in range(2, 25):
print('\t Reidentifying: ' + basename + '.ch%02dedge.fits' % i)
idfile = database + '/id' + basename + '.ch%02dedge' % i
if os.path.isfile(idfile) and not overwrite:
print('\t Edge identification files already exist, ' + idfile +
'. Skipping.')
else:
if os.path.isfile(idfile) and overwrite:
print('\t Removing ' + idfile)
try:
os.remove(idfile)
except:
pass
# treatment for VPH650
if i == 12:
disperser = fits.getval(basename + '.ch12edge.fits',
'DISPERSR')
if disperser == 'SCFCGRHD65':
nlost = 1
if i == 13:
disperser = fits.getval(basename + '.ch12edge.fits',
'DISPERSR')
if disperser == 'SCFCGRHD65':
nlost = 0
iraf.reidentify(basename+'.ch%02dedge'%(i-1), \
basename+'.ch%02dedge'%i, \
interac='no', section=section, newaps=newaps, \
override=override, refit=refit, trace=trace, \
step=0.0, nsum=nsum, shift=shift, nlost=nlost, \
cradius=cradius, threshold=threshold, \
addfeatures=addfeatures, coordlist=coordlist, \
match=match, maxfeat=2, minsep=minsep, \
database=database, logfile=logfile, \
plotfile='', verbose=verbose, cursor='')
#Check the result
iraf.identify(basename+'.ch%02dedge'%i, section=section, \
database=database, coordlist=coordlist, units='', \
nsum=nsum, match=match, maxfeat=2,ftype='emission', \
fwidth=fwidth, cradius=cradius, threshold=threshold, \
function=function, order=order, sample='*', \
niter=niter, autowrite=autowrite)
iraf.reidentify(basename+'.ch%02dedge'%i, \
basename+'.ch%02dedge'%i, \
interac='no', section=section, newaps=newaps, \
override=override, refit=refit, trace=trace, \
step=step, nsum=nsum, shift=shift, nlost=nlost, \
cradius=cradius, threshold=threshold, \
addfeatures=addfeatures, coordlist=coordlist, \
match=match, maxfeat=2, minsep=minsep, \
database=database, logfile=logfile, \
plotfile='', verbose=verbose, cursor='')
print('\t Go back to the original directory.')
iraf.cd('..')
disperser = fits.getval(fi.chimagedir + basename + '.ch12edge.fits',
'DISPERSR')
if disperser == 'SCFCGRHD65':
correct_ch12_edge(basename, overwrite=overwrite)
return
def reduce_stdstar(rawdir,
rundir,
caldir,
starobj,
stdstar,
flat,
arc,
twilight,
twilight_flat,
starimg,
bias,
overscan,
vardq,
lacos,
observatory,
apply_lacos,
lacos_xorder,
lacos_yorder,
lacos_objlim,
lacos_sigclip,
bpm,
instrument,
slits,
fl_gscrrej,
wltrim_frac=0.03,
sens_order=6,
sens_function='spline3',
apsum_radius=1):
"""
Reduction pipeline for standard star.
Parameters
----------
rawdir: string
Directory containing raw images.
rundi: string
Directory where processed files are saved.
caldir: string
Directory containing standard star calibration files.
starobj: string
Object keyword for the star image.
stdstar: string
Star name in calibration file.
flat: list
Names of the files containing flat field images.
arc: list
Arc images.
twilight: list
Twilight flat images.
starimg: string
Name of the file containing the image to be reduced.
bias: list
Bias images.
"""
iraf.set(stdimage='imtgmos')
iraf.task(lacos_spec=lacos)
iraf.gemini()
iraf.unlearn('gemini')
iraf.gmos()
iraf.unlearn('gmos')
iraf.gemtools()
iraf.unlearn('gemtools')
iraf.gmos.logfile = 'logfile.log'
iraf.gemtools.gloginit.logfile = 'logfile.log'
iraf.gfextract.verbose = 'no'
# set directories
iraf.set(caldir=rawdir) #
iraf.set(rawdir=rawdir) # raw files
iraf.set(procdir=rundir) # processed files
# os.path.isfile('arquivo')
iraf.cd('procdir')
flat = flat.strip('.fits')
twilight = twilight.strip('.fits')
twilight_flat = twilight_flat.strip('.fits')
arc = arc.strip('.fits')
starimg = starimg.strip('.fits')
mdffile = 'mdf' + flat + '.fits'
iraf.gfreduce.bias = 'rawdir$' + bias
iraf.gfreduce.fl_fulldq = 'yes'
iraf.gfreduce.fl_fixgaps = 'yes'
iraf.gireduce.bpm = 'rawdir$' + bpm
cal_reduction(rawdir=rawdir,
rundir=rundir,
flat=flat,
arc=arc,
twilight=twilight,
bias=bias,
bpm=bpm,
overscan=overscan,
vardq=vardq,
instrument=instrument,
slits=slits,
twilight_flat=twilight_flat)
#
# Actually reduce star
#
imageName = 'rg' + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
imageName = 'g' + starimg + '.fits'
if os.path.isfile(imageName):
iraf.printlog(
'GIREDS: WARNING: Removing file {:s}'.format(imageName),
'logfile.log', 'yes')
iraf.delete(imageName)
iraf.gfreduce(starimg,
slits='header',
rawpath='rawdir$',
fl_inter='no',
fl_addmdf='yes',
key_mdf='MDF',
mdffile=mdffile,
weights='no',
fl_over=overscan,
fl_trim='yes',
fl_bias='yes',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej='no',
fl_extract='no',
fl_gsappwave='no',
fl_wavtran='no',
fl_novl='yes',
fl_skysub='no',
fl_vardq=vardq,
mdfdir='procdir$')
prefix = 'rg'
#
# Gemfix
#
imageName = 'p' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gemfix(prefix + starimg,
out='p' + prefix + starimg,
method='fit1d',
bitmask=1,
axis=1)
prefix = 'p' + prefix
#
# LA Cosmic
#
if apply_lacos:
imageName = 'l' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
if apply_lacos:
iraf.gemcrspec(prefix + starimg,
out='l' + prefix + starimg,
sigfrac=0.32,
niter=4,
fl_vardq=vardq,
xorder=lacos_xorder,
yorder=lacos_yorder,
objlim=lacos_objlim,
sigclip=lacos_sigclip)
prefix = 'l' + prefix
#
# Extraction and Gemini's comsmic ray rejection
#
if fl_gscrrej:
imageName = 'x' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
fl_gscrrej = False
else:
imageName = 'ex' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gfreduce(prefix + starimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej=fl_gscrrej,
fl_extract='yes',
fl_gsappwave='yes',
fl_wavtran='no',
fl_novl='no',
fl_skysub='no',
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq)
prefix = 'ex' + prefix
else:
imageName = 'e' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gfreduce(prefix + starimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej=fl_gscrrej,
fl_extract='yes',
fl_gsappwave='yes',
fl_wavtran='no',
fl_novl='no',
fl_skysub='no',
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq)
prefix = 'e' + prefix
#
# Wavelength transform
#
wl1, wl2 = wl_lims(prefix + starimg + '.fits', wltrim_frac)
imageName = 't' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gfreduce(prefix + starimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej='no',
fl_extract='no',
fl_gsappwave='no',
fl_wavtran='yes',
fl_novl='no',
fl_skysub='no',
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq,
w1=wl1,
w2=wl2)
prefix = 't' + prefix
#
# Sky subtraction
#
imageName = 's' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gfreduce(prefix + starimg,
slits='header',
rawpath='./',
fl_inter='no',
fl_addmdf='no',
key_mdf='MDF',
mdffile=mdffile,
fl_over='no',
fl_trim='no',
fl_bias='no',
trace='no',
recenter='no',
fl_flux='no',
fl_gscrrej='no',
fl_extract='no',
fl_gsappwave='no',
fl_wavtran='no',
fl_novl='no',
fl_skysub='yes',
reference='eprg' + flat,
weights='no',
wavtraname='eprg' + arc,
response='eprg' + flat + '_response.fits',
fl_vardq=vardq,
w1=wl1,
w2=wl2)
prefix = 's' + prefix
#
# Apsumming the stellar spectra
#
xinst = pf.getdata(prefix + starimg + '.fits', ext=1)['XINST']
if instrument == 'GMOS-N':
x0 = np.average(xinst[xinst < 10])
elif instrument == 'GMOS-S':
x0 = np.average(xinst[xinst > 10])
ap_expression = '((XINST-{:.2f})**2 + '\
'(YINST-2.45)**2)**0.5 < {:.2f}'.format(x0, apsum_radius)
imageName = 'a' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gfapsum(prefix + starimg,
fl_inter='no',
lthreshold='INDEF',
hthreshold='INDEF',
reject='avsigclip',
expr=ap_expression)
#
# Building sensibility function
#
if os.path.isfile('std' + starimg)\
and os.path.isfile('sens' + starimg + '.fits'):
skipwarn('std{0:s} and sens{0:s}.fits'.format(starimg))
else:
imageName = 'std' + starimg
if os.path.isfile(imageName):
iraf.printlog(
'GIREDS: WARNING: Removing file {:s}'.format(imageName),
'logfile.log', 'yes')
iraf.delete(imageName)
imageName = 'sens' + starimg + '.fits'
if os.path.isfile(imageName):
iraf.printlog(
'GIREDS: WARNING: Removing file {:s}'.format(imageName),
'logfile.log', 'yes')
iraf.delete(imageName)
iraf.gsstandard('a' + prefix + starimg,
starname=stdstar,
observatory=observatory,
sfile='std' + starimg,
sfunction='sens' + starimg,
caldir=caldir,
order=sens_order,
function=sens_function)
#
# Apply flux calibration to star
#
imageName = 'c' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gscalibrate(prefix + starimg,
sfuncti='sens' + starimg,
extinct='onedstds$ctioextinct.dat',
observatory=observatory,
fluxsca=1,
fl_vardq=vardq)
#
# Create data cubes
#
imageName = 'dc' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
data_cube = CubeBuilder('c' + prefix + starimg + '.fits')
data_cube.build_cube()
data_cube.fit_refraction_function()
data_cube.fix_atmospheric_refraction()
data_cube.write(imageName)
#
# Test calibration
#
iraf.cd(caldir)
caldata = np.loadtxt(stdstar + '.dat', unpack=True)
iraf.cd('procdir')
calflux = mag2flux(caldata[0], caldata[1])
imageName = 'ca' + prefix + starimg + '.fits'
if os.path.isfile(imageName):
skipwarn(imageName)
else:
iraf.gscalibrate('a' + prefix + starimg,
sfuncti='sens' + starimg,
extinct='onedstds$ctioextinct.dat',
observatory=observatory,
fluxsca=1)
sumflux = pf.getdata('ca' + prefix + starimg + '.fits', ext=2)
sumhead = pf.getheader('ca' + prefix + starimg + '.fits', ext=2)
sumwl = sumhead['crval1'] + np.arange(
sumhead['naxis1']) * sumhead['cdelt1']
plt.close('all')
plt.plot(sumwl, sumflux, 'b', lw=.5)
plt.plot(caldata[0], calflux, 'r', lw=1.5)
plt.xlim(sumwl[0] * .99, sumwl[-1] * 1.01)
plt.ylim(min(calflux) * .8, max(calflux) * 1.2)
plt.savefig('calib' + starimg + '.eps')
def makeflats(fs=None):
# Note the list of files need to not include any paths relative to
# the work directory.
# Maybe not the greatest convention, but we can update this later
iraf.cd('work')
if fs is None:
fs = glob('pysalt/bxgp*.fits')
if len(fs) == 0:
print "WARNING: No flat-fields to combine and normalize."
# Fail gracefully by going up a directory
iraf.cd('..')
return
# make a flats directory
if not os.path.exists('flats'):
os.mkdir('flats')
# Figure out which images are flats and which grating angles were used
allflats, grangles = get_ims(fs, 'flat')
# For each grating angle
for ga in np.unique(grangles):
# grab the flats for this gr angle
flats = allflats[grangles == ga]
# For each chip
for c in range(1, 7):
# run imcombine with average and sigclip, weighted by exposure time
flatlist = ''
for f in flats:
flatlist += '%s[%i],' % (f, c)
# Add the exptime keyword to each extension
pyfits.setval(f, 'EXPTIME', ext=c,
value=pyfits.getval(f, 'EXPTIME'))
# set the output combined file name
combineoutname = 'flats/flt%05.2fcomc%i.fits' % (ga, c)
if os.path.exists(combineoutname):
os.remove(combineoutname)
# initialize the iraf command
iraf.unlearn(iraf.imcombine)
print(flatlist)
# don't forget to remove the last comma in the filelist
iraf.imcombine(input=flatlist[:-1], output=combineoutname,
combine='average', reject='sigclip', lsigma=3.0,
hsigma=3.0, weight='exposure', expname='EXPTIME')
pyfits.setval(combineoutname, 'DISPAXIS', value=1)
# We want to make an illumination correction file
# before running response:
illumoutname = 'flats/flt%05.2fillc%i.fits' % (ga, c)
iraf.unlearn(iraf.illumination)
iraf.illumination(images=combineoutname,
illuminations=illumoutname, interactive=False,
naverage=-40, order=11, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
# Flag any pixels in the illumination correction< 0.1
illumhdu = pyfits.open(illumoutname, mode='update')
illumhdu[0].data[illumhdu[0].data <= 0.1] = 0.0
illumhdu.flush()
# Get 40 pixels out of the middle of the image and
# median them to run response
combinehdu = pyfits.open(combineoutname)
ny = combinehdu[0].data.shape[0]
# divide out the illumination correction before running response
flat1d = np.median(combinehdu[0].data[ny / 2 - 21: ny / 2 + 20, :]
/ illumhdu[0].data[ny / 2 - 21: ny / 2 + 20, :],
axis=0)
# close the illumination file because we don't need it anymore
illumhdu.close()
# File stage m1d for median 1-D
flat1dfname = 'flats/flt%05.2fm1dc%i.fits' % (ga, c)
tofits(flat1dfname, flat1d, hdr=combinehdu[0].header.copy())
# run response
# r1d = response1d
resp1dfname = 'flats/flt%05.2fr1dc%i.fits' % (ga, c)
iraf.response(flat1dfname, flat1dfname, resp1dfname, order=31,
interactive=False, naverage=-5, low_reject=3.0,
high_reject=3.0, niterate=5, mode='hl')
resp1dhdu = pyfits.open(resp1dfname)
resp1d = resp1dhdu[0].data.copy()
resp1dhdu.close()
# After response divide out the response function
# normalize the 1d resp to its median
resp1d /= np.median(resp1d)
# Chuck any outliers
flatsig = np.std(resp1d - 1.0)
resp1d[abs(resp1d - 1.0) > 5.0 * flatsig] = 1.0
resp = flat1d / resp1d
resp2dfname = 'flats/flt%05.2fresc%i.fits' % (ga, c)
resp2d = combinehdu[0].data.copy() / resp
tofits(resp2dfname, resp2d, hdr=combinehdu[0].header.copy())
combinehdu.close()
# close the combined flat because we don't need it anymore
combinehdu.close()
pyfits.setval(resp2dfname, 'DISPAXIS', value=1)
# Reset any pixels in the flat field correction< 0.1
# We could flag bad pixels here if we want, but not right now
flathdu = pyfits.open(resp2dfname, mode='update')
flathdu[0].data[flathdu[0].data <= 0.1] = 0.0
flathdu.flush()
flathdu.close()
# Step back up to the top directory
iraf.cd('..')
#!/usr/bin/python # IMP: Keep ds9 open # This script will guide you through the V R and I images directories. # Tell the script which combination of images to align , combine etc. # After that you can do photometry. # Enjoy!!!---------------------- [email protected] import os import glob from pyraf import iraf import pyfits import sys, traceback dirs = open('directories', 'r') for night in dirs.readlines(): night = night.rstrip() iraf.cd(night) print night filters = glob.glob('image[RVI]') #Only going to image[RVI] filters.sort() for imgdir in filters: iraf.cd(imgdir) while 1: print night + ' ' + imgdir images = glob.glob('[sn][nz]*.fits') images.sort() imgs_txt = open('imgs.txt', 'w') i = 0 for img in images: hdulist = pyfits.open(img) Exptime = hdulist[0].header.get('EXPTIME')
def merge_cubes(rawdir, rundir, name, observatory, imgcube, xoff, yoff, crval3,
cdelt3, cdelt1):
"""
Merge cubes.
Parameters
----------
rawdir: string
Directory containing raw images.
rundir: string
Directory where processed files are saved.
name: string
Name of the object.
observatory: string
Gemini-South/Gemini-North.
imgcube: list of strings
Cube file for each object cube.
xoff: list of floats
x-offset for each object cube.
yoff: list of floats
y-offset for each object cube.
crval3: list of floats
crval3 for each object cube.
cdelt3: list of floats
cdelt3 for each object cube.
cdelt1: list of floats
cdelt1 for each object cube.
"""
rundir = rundir + '/'
iraf.set(stdimage='imtgmos')
iraf.gemini()
iraf.unlearn('gemini')
iraf.gmos()
iraf.unlearn('gmos')
iraf.gemtools()
iraf.unlearn('gemtools')
iraf.gmos.logfile = 'logfile.log'
iraf.gemtools.gloginit.logfile = 'logfile.log'
# set directories
iraf.set(caldir=rawdir) #
iraf.set(rawdir=rawdir) # raw files
iraf.set(procdir=rundir) # processed files
iraf.cd('procdir')
#
# Creation of file/offset files
#
nCubes = len(imgcube)
in_filesSCI = 'files_' + name + '_SCI'
in_filesVAR = 'files_' + name + '_VAR'
in_offset = 'offsets_' + name
with open(in_filesSCI, 'w') as f:
for img in imgcube:
f.write(rundir + img + '[1]' + '\n')
with open(in_filesVAR, 'w') as f:
for img in imgcube:
f.write(rundir + img + '[2]' + '\n')
# Invert (x,y)offsets if in gemini-north
sign = -1 if (observatory.lower() == 'gemini-north') else 1
with open(in_offset, 'w') as f:
for k in range(nCubes):
f.write("{:.5f} {:.5f} {:.5f}\n".format(
sign * (xoff[k] - xoff[0]) / cdelt1[k],
sign * (yoff[k] - yoff[0]) / cdelt1[k],
(crval3[k] - crval3[0]) / cdelt3[k]))
#
# Definition of in/output files. And header modification.
#
in_sci = [img + '[1]' for img in imgcube]
in_var = [img + '[2]' for img in imgcube]
in_dq = [img + '[3]' for img in imgcube]
out_sci = name + '_SCI.fits'
out_var = name + '_VAR.fits'
out_sigIN = name + '_SIGIN.fits'
out_exp = name + '_EXP'
# Convert DQ extension to 'pl' and add the its filename to 'bpm' key
# --- Change to other key. (Other rotines use this key) - Improve
# --- Change also the key for bpm used by 'fixpix' ------ Improve
out_dqPL = [img[:-5] + '_DQ.pl' for img in imgcube]
for k in range(nCubes):
print(in_sci[k], in_var[k], in_dq[k], out_dqPL[k])
iraf.imcopy(in_dq[k], out_dqPL[k])
iraf.hedit(in_sci[k], 'BPM', out_dqPL[k], add='yes', verify='no')
iraf.hedit(in_var[k], 'BPM', out_dqPL[k], add='yes', verify='no')
#
# Merge sci/var cubes
#
iraf.imcombine("@" + in_filesSCI,
out_sci,
offsets=in_offset,
combine='average',
reject='avsigclip',
masktype='goodvalue',
maskvalue=0,
expmasks=out_exp,
sigmas=out_sigIN)
iraf.imcombine("@" + in_filesVAR,
out_var,
offsets=in_offset,
combine='sum',
reject='none',
masktype='goodvalue',
maskvalue=0)
#
# Criate correct error cube
#
iraf.imcopy(out_exp + '.pl', out_exp.replace('.pl', '.fits'))
# Read cubes
sci_cube = pf.getdata(out_sci)
var_cube = pf.getdata(out_var)
sigIN_cube = pf.getdata(out_sigIN)
exp_cube = pf.getdata(out_exp + '.fits')
# --- Identify problem with negative values ---- Improve
# RuntimeWarning: invalid value encountered in divide
exp_MASK = np.ma.array(exp_cube, mask=(exp_cube == 0))
err_cube = np.sqrt(abs(var_cube / exp_MASK**2).data)
#
# Criate hypercube
#
# ---- Maybe don't need header for each extension -- Improve
pry = pf.PrimaryHDU(header=pf.getheader(out_sci))
hdu1 = pf.ImageHDU(sci_cube, header=pf.getheader(out_sci), name='SCI')
hdu2 = pf.ImageHDU(err_cube, header=pf.getheader(out_var), name='ERR')
hdu4 = pf.ImageHDU(sigIN_cube,
header=pf.getheader(out_sigIN),
name='SIG_IN')
hdu3 = pf.ImageHDU(exp_cube,
header=pf.getheader(out_exp + '.fits'),
name='NCUBE')
hdu = pf.HDUList([pry, hdu1, hdu2, hdu3, hdu4])
hdu.writeto(name + '_HYPERCUBE.fits')
def rectify(ids=None, fs=None):
iraf.cd('work')
if ids is None:
ids = np.array(glob('id2/arc*id2*.db'))
if fs is None:
fs = glob('mos/*mos*.fits')
if len(ids) == 0:
print "WARNING: No wavelength solutions for rectification."
iraf.cd('..')
return
if len(fs) == 0:
print "WARNING: No images for rectification."
iraf.cd('..')
return
# Get the grating angles of the solution files
idgas = []
for i, thisid in enumerate(ids):
f = open(thisid)
idlines = np.array(f.readlines(), dtype=str)
f.close()
idgaline = idlines[np.char.startswith(idlines, '#graang')][0]
idgas.append(float(idgaline.split('=')[1]))
ims, gas = get_scis_and_arcs(fs)
if not os.path.exists('rec'):
os.mkdir('rec')
for i, f in enumerate(ims):
fname = f.split('/')[1]
typestr = fname[:3]
ga, imgnum = gas[i], fname[-9:-5]
outfile = 'rec/' + typestr + '%05.2frec' % (ga) + imgnum + '.fits'
iraf.unlearn(iraf.specrectify)
iraf.flpr()
idfile = ids[np.array(idgas) == ga][0]
iraf.specrectify(images=f, outimages=outfile, solfile=idfile,
outpref='', function='legendre', order=3,
inttype='interp', conserve='yes', clobber='yes',
verbose='yes')
# Update the BPM to mask any blank regions
h = pyfits.open(outfile, 'update')
# Cover the chip gaps. The background task etc do better if the chip
# gaps are straight
# To deal with this we just throw away the min and max of each side of
# the curved chip gap
chipgaps = get_chipgaps(h)
# Chip 1
h[2].data[:, chipgaps[0][0]:chipgaps[0][1]] = 1
# Chip 2
h[2].data[:, chipgaps[1][0]:chipgaps[1][1]] = 1
# edge of chip 3
h[2].data[:, chipgaps[2][0]:chipgaps[2][1]] = 1
# Cover the other blank regions
h[2].data[[h[1].data == 0]] = 1
# Set all of the data to zero in the BPM
h[1].data[h[2].data == 1] = 0.0
h.flush()
h.close()
iraf.cd('..')
