def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines', preprocess=False):
#set up the files
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
#check the value of dy
#apply the mask to the data sets
for i in range(len(infile_list)):
specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile=slitmask,
outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy,
inter=False, clobber=True, logfile=logfile, verbose=True)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None):
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='s'+os.path.basename(infile_list[i])
if lamp == 'NONE': lamp='CuAr'
lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype,
guessfile=guessfile, automethod=automethod, function='legendre', order=3,
rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
inter=True, clobber=True, preprocess=True, logfile=logfile, verbose=True)
#specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
# function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
spec_list=[]
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and \
(obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None) and \
obs_dict['OBSMODE'][i].count('SPECTROSCOPY'):
img = infile_list[i]
##rectify it
specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines'):
#set up the files
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
#apply the mask to the data sets
for i in range(len(infile_list)):
specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile='../../P001423N01.xml',
outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy,
inter=False, clobber=True, logfile=logfile, verbose=True)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='s'+os.path.basename(infile_list[i])
if lamp == 'NONE': lamp='CuAr'
lampfile=iraf.osfn("../../%s.salt" % lamp)
specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype,
guessfile=guessfile, automethod=automethod, function='legendre', order=3,
rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
inter=False, clobber=True, logfile=logfile, verbose=True)
#specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
# function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
spec_list=[]
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and \
obs_dict['PROPID'][i].upper().strip()==propcode and \
obs_dict['OBSMODE'][i].count('SPECTROSCOPY'):
img = infile_list[i]
##rectify it
specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
def specred(infile_list, target, propcode, calfile=None, inter=True, automethod='Matchlines'):
#set up the files
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage=os.path.basename(infile_list[i])
if lamp == 'NONE': lamp='CuAr'
lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
inter=False, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
spec_list=[]
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and obs_dict['PROPID'][i].upper().strip()==propcode:
img = infile_list[i]
##rectify it
specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
#extract the spectra
spec_list.append(extract_spectra('x'+img, yc=1030, calfile=calfile, findobject=True, smooth=False, maskzeros=True, clobber=True))
#combine the results
w,f,e = speccombine(spec_list, obsdate)
outfile = "%s_%s.spec" % (target, obsdate)
write_spectra(outfile, w,f,e)
def quickspec(profile, lampid=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True):
"""From mosaicked data, produce wavelength calibrated files"""
profile = os.path.basename(profile)
#fill in the mosaic
fillgaps(profile)
#specrectify
specrectify(profile, outimages='', outpref='s', solfile=None, caltype='rss',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
y1,y2=quickap('s' + profile, lampid, findobj, objsection, skysection, clobber, logfile, verbose)
if skysection is None:
ylen=100
skysection='[%i:%i]' % (y2+0.1*ylen,y2+0.2*ylen)
specsky('s'+profile, outimages='s'+profile, outpref='', method='normal', section=skysection, function='polynomial', order=3, clobber=clobber, logfile=logfile, verbose=verbose)
return y1,y2
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'):
#set up the files
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage=os.path.basename(infile_list[i])
if lamp == 'NONE': lamp='CuAr'
lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
#lampfile='/Users/crawford/research/kepler/Xe.salt'
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
inter=inter, clobber=True, logfile=logfile, verbose=True)
#specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
# function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
spec_list=[]
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and obs_dict['PROPID'][i].upper().strip()==propcode:
img = infile_list[i]
##rectify it
specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
def quickspec(profile, lampid=None, solfile=None, findobj=False, objsection=None, skysection=None, clobber=False, logfile='saltclean.log', verbose=True):
"""From mosaicked data, produce wavelength calibrated files"""
profile = os.path.basename(profile)
#fill in the mosaic
#fillgaps(profile)
#specrectify
caltype='rss'
if solfile:
soldict = entersolution(solfile)
hdu = fits.open(profile)
instrume = hdu[0].header['INSTRUME']
grating = hdu[0].header['GRATING']
grang = hdu[0].header['GRTILT']
arang = hdu[0].header['CAMANG']
filtername = hdu[0].header['FILTER']
slitid=None
for sol in soldict:
if matchobservations( soldict[sol], instrume, grating, grang, arang, filtername, slitid):
caltype='line'
break
specrectify(profile, outimages='', outpref='s', solfile=solfile, caltype=caltype,
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
y1,y2=quickap('s' + profile, lampid, findobj, objsection, skysection, clobber, logfile, verbose)
if skysection is None:
ylen=100
skysection='[%i:%i]' % (y2+0.1*ylen,y2+0.2*ylen)
specsky('s'+profile, outimages='s'+profile, outpref='', method='normal', section=skysection, function='polynomial', order=3, clobber=clobber, logfile=logfile, verbose=verbose)
return y1,y2
def specred(infile_list, propcode=None, inter=True, guessfile = None, automethod='Matchlines', preprocess=False):
#set up the files
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
dbfile='spec%s.db' % obsdate
sdbfile='spec_straight_%s.db' % obsdate
straight_function = 'legendre'
straight_order = 2
dcoef = [0.50, 1.0, 0.0]
#create the observation log
obs_dict=obslog(infile_list)
for i in range(len(infile_list)):
print infile_list[i], obs_dict['OBJECT'][i].upper().strip(), obs_dict['PROPID'][i].upper().strip()
if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode):
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage=os.path.basename(infile_list[i])
if lamp == 'NONE': lamp='CuAr'
lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
#lampfile='/Users/crawford/research/kepler/Xe.salt'
#straighten the arc
#specarcstraighten(arcimage, sdbfile , function=straight_function, order=straight_order, rstep=20,
# rstart='middlerow', nrows=1, dcoef=dcoef, ndstep=10,
# startext=0, clobber=False, logfile='salt.log', verbose=True)
#rectify it
#specrectify(arcimage, outimages='', outpref='s', solfile=sdbfile, caltype='line',
# function=straight_function, order=straight_order, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# nearest=True, blank=0.0, clobber=True, logfile=logfile, verbose=True)
#idnetify the line
if guessfile is None:
guesstype='rss'
guessfile=None
else:
guesstype='file'
specidentify(arcimage, lampfile, dbfile, guesstype=guesstype,
guessfile=guessfile, automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=20, thresh=5, niter=5, smooth=3,
inter=inter, clobber=True, preprocess=preprocess, logfile=logfile, verbose=True)
#apply the final rectification
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
spec_list=[]
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS') and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode):
img = infile_list[i]
# rectify it
#specrectify(img, outimages='', outpref='s', solfile=sdbfile, caltype='line',
# function='legendre', order=2, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# blank=0.0, clobber=True, logfile=logfile, verbose=True)
specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
def specred(infile_list,
target,
propcode,
calfile=None,
inter=True,
automethod='Matchlines'):
#set up the files
infiles = ','.join(['%s' % x for x in infile_list])
obsdate = os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile = 'spec' + obsdate + '.log'
dbfile = 'spec%s.db' % obsdate
#create the observation log
obs_dict = obslog(infile_list)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip(
) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode:
lamp = obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage = os.path.basename(infile_list[i])
if lamp == 'NONE': lamp = 'CuAr'
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specidentify(arcimage,
lampfile,
dbfile,
guesstype='rss',
guessfile='',
automethod=automethod,
function='legendre',
order=3,
rstep=100,
rstart='middlerow',
mdiff=20,
thresh=3,
niter=5,
smooth=3,
inter=False,
clobber=True,
logfile=logfile,
verbose=True)
specrectify(arcimage,
outimages='',
outpref='x',
solfile=dbfile,
caltype='line',
function='legendre',
order=3,
inttype='interp',
w1=None,
w2=None,
dw=None,
nw=None,
blank=0.0,
clobber=True,
logfile=logfile,
verbose=True)
objimages = ''
spec_list = []
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][
i].count('RSS') and obs_dict['PROPID'][i].upper().strip(
) == propcode:
img = infile_list[i]
##rectify it
specrectify(img,
outimages='',
outpref='x',
solfile=dbfile,
caltype='line',
function='legendre',
order=3,
inttype='interp',
w1=None,
w2=None,
dw=None,
nw=None,
blank=0.0,
clobber=True,
logfile=logfile,
verbose=True)
#extract the spectra
spec_list.append(
extract_spectra('x' + img,
yc=1030,
calfile=calfile,
findobject=True,
smooth=False,
maskzeros=True,
clobber=True))
#combine the results
w, f, e = speccombine(spec_list, obsdate)
outfile = "%s_%s.spec" % (target, obsdate)
write_spectra(outfile, w, f, e)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'):
#set up the files
infiles = ','.join(['%s' % x for x in infile_list])
obsdate = os.path.basename(infile_list[0])[7:15]
#set up some files that will be needed
logfile = 'spec' + obsdate + '.log'
dbfile = 'spec%s.db' % obsdate
#create the observation log
obs_dict = obslog(infile_list)
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip(
) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode:
lamp = obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage = os.path.basename(infile_list[i])
if lamp == 'NONE': lamp = 'CuAr'
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
#lampfile='/Users/crawford/research/kepler/Xe.salt'
specidentify(arcimage,
lampfile,
dbfile,
guesstype='rss',
guessfile='',
automethod=automethod,
function='legendre',
order=3,
rstep=100,
rstart='middlerow',
mdiff=20,
thresh=3,
niter=5,
smooth=3,
inter=inter,
clobber=True,
logfile=logfile,
verbose=True)
#specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
# function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
# blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages = ''
spec_list = []
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][
i].count('RSS') and obs_dict['PROPID'][i].upper().strip(
) == propcode:
img = infile_list[i]
##rectify it
specrectify(img,
outimages='',
outpref='x',
solfile=dbfile,
caltype='line',
function='legendre',
order=3,
inttype='interp',
w1=None,
w2=None,
dw=None,
nw=None,
blank=0.0,
nearest=True,
clobber=True,
logfile=logfile,
verbose=True)
infile_list = glob.glob('m*fits')
infiles=','.join(['%s' % x for x in infile_list])
obsdate=os.path.basename(infile_list[0])[7:15]
obs_dict=obslog(infile_list)
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].upper().strip()=='ARC':
arc_image = infile_list[i]
auto_arc_lens(arc_image, dbfile=dbfile, ndstep=20, logfile=logfile)
obj_dict = {}
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img = infile_list[i]
specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
obj = obs_dict['OBJECT'][i].strip()
if obj in obj_dict.keys():
obj_dict[obj].append('x'+img)
else:
obj_dict[obj] = ['x'+img]
# combine the data if necessary
red_list = []
for obj in obj_dict.keys():
if len(obj_dict[obj])>1:
print(obj_dict[obj])
def science_red(rawdir, prodir, imreduce=True, specreduce=True, bpmfile=None, calfile=None, lampfile='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'P*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
add_variance('bpP*fits', bpmfile)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if 0: #len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=False, blank=0, scale=None, statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=0.8, allext=False, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='fxgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
#lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=50, thresh=2, niter=5, smooth=3,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
return
def specred(rawdir, prodir, imreduce=True, specreduce=True, calfile=None, lamp='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('cxgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
lampfile=iraf.osfn("pysalt$data/linelists/%s.txt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=5,
rstep=100, rstart='middlerow', mdiff=10, thresh=3, niter=5,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
#create the spectra text files for all of our objects
spec_list=[]
for img in objimages.split(','):
spec_list.extend(createspectra('x'+img, obsdate, smooth=False, skysection=skysection, clobber=True))
print spec_list
#determine the spectrophotometric standard
extfile=iraf.osfn('pysalt$data/site/suth_extinct.dat')
for spec, am, et, pc in spec_list:
if pc=='CAL_SPST':
stdstar=spec.split('.')[0]
print stdstar, am, et
stdfile=iraf.osfn('pysalt$data/standards/spectroscopic/m%s.dat' % stdstar.lower().replace('-', '_'))
print stdfile
ofile=spec.replace('txt', 'sens')
calfile=ofile #assumes only one observations of a SP standard
specsens(spec, ofile, stdfile, extfile, airmass=am, exptime=et,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
clobber=True, logfile='salt.log',verbose=True)
for spec, am, et, pc in spec_list:
if pc!='CAL_SPST':
ofile=spec.replace('txt', 'spec')
speccal(spec, ofile, calfile, extfile, airmass=am, exptime=et,
clobber=True, logfile='salt.log',verbose=True)
#clean up the spectra for bad pixels
cleanspectra(ofile)
