def saltgain(images,
outimages,
outpref,
gaindb=None,
usedb=False,
mult=True,
clobber=True,
logfile='salt.log',
verbose=True):
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# read in the database file if usedb is true
if usedb:
gaindb = gaindb.strip()
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct = saltio.openfits(img)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message = 'SALTGAIN: %s has already been gain-corrected' % img
raise SaltError(message)
# gain correct the data
struct = gain(struct,
mult=mult,
usedb=usedb,
dblist=dblist,
log=log,
verbose=verbose)
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], keygain,
'Images have been gain corrected', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def saltslot(images,
outimages,
outpref,
gaindb='',
xtalkfile='',
usedb=False,
clobber=False,
logfile='salt.log',
verbose=True):
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# are input and output lists the same length?
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# does crosstalk coefficient data exist
if usedb:
dblist = saltio.readgaindb(gaindb)
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
dblist = []
xdict = None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct = saltio.openfits(img)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message = '%s has already been reduced' % img
raise SaltError(message)
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], keyslot,
'Images have been slotmode reduced', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def saltgain(images,outimages, outpref, gaindb=None,usedb=False, mult=True,
clobber=True, logfile='salt.log',verbose=True):
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# read in the database file if usedb is true
if usedb:
gaindb = gaindb.strip()
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message='SALTGAIN: %s has already been gain-corrected' % img
raise SaltError(message)
# gain correct the data
struct = gain(struct,mult=mult, usedb=usedb, dblist=dblist, log=log, verbose=verbose)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keygain, 'Images have been gain corrected', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def saltslot(images,outimages,outpref,gaindb='',xtalkfile='',usedb=False, clobber=False,logfile='salt.log',verbose=True):
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
# are input and output lists the same length?
saltio.comparelists(infiles,outfiles,'Input','output')
# does crosstalk coefficient data exist
if usedb:
dblist= saltio.readgaindb(gaindb)
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
dblist=[]
xdict=None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message='%s has already been reduced' % img
raise SaltError(message)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyslot, 'Images have been slotmode reduced', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def quickclean(filename, interp='linear', cleanup=True, clobber=False, logfile='saltclean.log', verbose=True):
"""Start the process to reduce the data and produce a single mosaicked image"""
print filename
#create the input file name
status=0
infile=os.path.basename(filename)
rawpath=os.path.dirname(filename)
outpath='./'
outfile=outpath+'mbxp'+infile
print infile, rawpath, outpath
#check to see if it exists and return if clobber is no
if os.path.isfile(outfile) and not clobber: return
#set up the files needed
if infile[0]=='P':
gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
elif infile[0]=='S':
gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
#verify the file
struct=saltio.openfits(rawpath+'/'+infile)
struct.verify('exception')
#check to see if detmode is there
if not saltkey.found('DETMODE', struct[0]):
return
#reduce the file
struct=prepare(struct, createvar=False, badpixelstruct=None)
#reset the names in the structures
for i in range(1,len(struct)):
struct[i].name=struct[i].header['EXTNAME']
#gain correct the files
usedb=True
dblist= saltio.readgaindb(gaindb)
log=open(logfile, 'a')
ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
struct=gain(struct, mult=True,usedb=usedb, dblist=dblist, ampccd=ampccd, log=None, verbose=verbose)
struct=bias(struct, subover=True,trim=True,subbias=False,
median=False,function='polynomial',order=5,rej_lo=3,rej_hi=3,niter=10,
plotover=False,log=None, verbose=verbose)
if struct[0].header['CCDTYPE']=='OBJECT' and struct[0].header['EXPTIME']>90:
struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)
pinfile=outpath+'bxp'+infile
saltio.writefits(struct, pinfile, clobber)
saltred.saltmosaic(images=pinfile,
outimages='',outpref=outpath+'m',geomfile=geomfile, fill=True,
interp=interp,cleanup=cleanup,clobber=clobber,logfile=logfile,
verbose=verbose)
profile=outpath+'mbxp'+infile
#remove intermediate steps
if cleanup:
if os.path.isfile(pinfile): os.remove(pinfile)
return
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def quickclean(filename,
interp='linear',
cleanup=True,
clobber=False,
logfile='saltclean.log',
verbose=True):
"""Start the process to reduce the data and produce a single mosaicked image"""
print filename
#create the input file name
status = 0
infile = os.path.basename(filename)
rawpath = os.path.dirname(filename)
outpath = './'
outfile = outpath + 'mbxp' + infile
print infile, rawpath, outpath
#check to see if it exists and return if clobber is no
if os.path.isfile(outfile) and not clobber: return
#set up the files needed
if infile[0] == 'P':
gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
elif infile[0] == 'S':
gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
#verify the file
struct = saltio.openfits(rawpath + '/' + infile)
struct.verify('exception')
#check to see if detmode is there
if not saltkey.found('DETMODE', struct[0]):
return
#reduce the file
struct = prepare(struct, createvar=False, badpixelstruct=None)
#reset the names in the structures
for i in range(1, len(struct)):
struct[i].name = struct[i].header['EXTNAME']
#gain correct the files
usedb = True
dblist = saltio.readgaindb(gaindb)
log = open(logfile, 'a')
ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
struct = gain(struct,
mult=True,
usedb=usedb,
dblist=dblist,
ampccd=ampccd,
log=None,
verbose=verbose)
struct = bias(struct,
subover=True,
trim=True,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=10,
plotover=False,
log=None,
verbose=verbose)
if struct[0].header[
'CCDTYPE'] == 'OBJECT' and struct[0].header['EXPTIME'] > 90:
struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)
pinfile = outpath + 'bxp' + infile
saltio.writefits(struct, pinfile, clobber)
saltred.saltmosaic(images=pinfile,
outimages='',
outpref=outpath + 'm',
geomfile=geomfile,
fill=True,
interp=interp,
cleanup=cleanup,
clobber=clobber,
logfile=logfile,
verbose=verbose)
profile = outpath + 'mbxp' + infile
#remove intermediate steps
if cleanup:
if os.path.isfile(pinfile): os.remove(pinfile)
return