def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def imred(rawdir, prodir, cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='imred'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
#saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
def imred(infile_list, prodir, bpmfile=None, gaindb = None, cleanup=True):
#get the name of the files
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='im'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
with logging(logfile, debug) as log:
log.message('Pysalt Version: '+pysalt.verno, with_header=False)
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
for img in infile_list:
hdu = pyfits.open('p'+os.path.basename(img), 'update')
# for backwards compatibility
if not hdu[1].header.has_key('XTALK'):
hdu[1].header.update('XTALK',1474)
hdu[2].header.update('XTALK',1474)
hdu[3].header.update('XTALK',1166)
hdu[4].header.update('XTALK',1111)
hdu[5].header.update('XTALK',1377)
hdu[6].header.update('XTALK',1377)
hdu.close()
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
add_variance('bpP*fits', bpmfile)
#gain correct the data
usedb = False
if gaindb: usedb = True
saltgain('bpP*fits', '', 'g', gaindb=gaindb, usedb=usedb, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if (obs_dict['CCDTYPE'][i].count('OBJECT') \
and obs_dict['LAMPID'][i].count('NONE') \
and obs_dict['INSTRUME'][i].count('RSS')):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#mosaic the data
#khn: attempt to use most recent previous geometry to obsdate.
#NOTE: mosaicing does not do this correctly
#geomdb = open(datadir+"RSSgeom.dat",'r')
#for geomline in geomdb:
# if geomline[0]=='#': continue
# if (int(obsdate) > int(geomline.split(' ')[0].replace('-',''))): break
#geomfile = "RSSgeom_obsdate.dat"
#open(geomfile,'w').write(geomline)
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
try:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
except:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#khn: fix mosaiced VAR and BPM extensions
#khn: fix mosaiced bpm missing some of gap
for img in infile_list:
filename = 'mxgbp'+os.path.basename(img)
hdu = pyfits.open(filename, 'update')
hdu[2].header.update('EXTNAME','VAR')
hdu[3].header.update('EXTNAME','BPM')
bpm_rc = (hdu[3].data>0).astype('uint8')
zeroscicol = hdu['SCI'].data.sum(axis=0) == 0
bpmgapcol = bpm_rc.mean(axis=0) == 1
addbpmcol = zeroscicol & ~bpmgapcol
addbpmcol[np.argmax(addbpmcol)-4:np.argmax(addbpmcol)] = True # allow for chip tilt
bpm_rc[:,addbpmcol] = 1
hdu[3].data = bpm_rc
hdu.writeto(filename,clobber=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def imred(rawdir, prodir, cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list = glob.glob(rawdir + '*.fits')
infiles = ','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate = os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile = 'imred' + obsdate + '.log'
flatimage = 'FLAT%s.fits' % (obsdate)
dbfile = 'spec%s.db' % obsdate
#create the observation log
obs_dict = obslog(infile_list)
#prepare the data
saltprepare(infiles,
'',
'p',
createvar=False,
badpixelimage='',
clobber=True,
logfile=logfile,
verbose=True)
#bias subtract the data
saltbias('pP*fits',
'',
'b',
subover=True,
trim=True,
subbias=False,
masterbias='',
median=False,
function='polynomial',
order=5,
rej_lo=3.0,
rej_hi=5.0,
niter=10,
plotover=False,
turbo=False,
clobber=True,
logfile=logfile,
verbose=True)
#gain correct the data
saltgain('bpP*fits',
'',
'g',
usedb=False,
mult=True,
clobber=True,
logfile=logfile,
verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits',
'',
'x',
xtalkfile="",
usedb=False,
clobber=True,
logfile=logfile,
verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count(
'OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img = 'xgbp' + os.path.basename(infile_list[i])
saltcrclean(img,
img,
'',
crtype='edge',
thresh=5,
mbox=11,
bthresh=5.0,
flux_ratio=0.2,
bbox=25,
gain=1.0,
rdnoise=5.0,
fthresh=5.0,
bfactor=2,
gbox=3,
maxiter=5,
multithread=True,
clobber=True,
logfile=logfile,
verbose=True)
#flat field correct the data
flat_imgs = ''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp' + os.path.basename(infile_list[i])
if len(flat_imgs) != 0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
#saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits',
'',
'f',
flatimage,
minflat=500,
clobber=True,
logfile=logfile,
verbose=True)
else:
flats = None
imfiles = glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f' + f)
#mosaic the data
geomfile = iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits',
'',
'm',
geomfile,
interp='linear',
cleanup=True,
geotran=True,
clobber=True,
logfile=logfile,
verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'):
os.remove(f)
for f in glob.glob('bp*fits'):
os.remove(f)
for f in glob.glob('gbp*fits'):
os.remove(f)
for f in glob.glob('xgbp*fits'):
os.remove(f)
for f in glob.glob('fxgbp*fits'):
os.remove(f)
def science_red(rawdir, prodir, imreduce=True, specreduce=True, bpmfile=None, calfile=None, lampfile='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'P*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
add_variance('bpP*fits', bpmfile)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if 0: #len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=False, blank=0, scale=None, statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=0.8, allext=False, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('xgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='fxgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
#lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=3,
rstep=100, rstart='middlerow', mdiff=50, thresh=2, niter=5, smooth=3,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
return
def imred(infilelist,
prodir,
bpmfile=None,
crthresh='',
gaindb=None,
cleanup=True):
#get the name of the files
infiles = ','.join(['%s' % x for x in infilelist])
#get the current date for the files
obsdate = os.path.basename(infilelist[0])[1:9]
print "Observation Date: ", obsdate
#set up some files that will be needed
logfile = 'im' + obsdate + '.log'
flatimage = 'FLAT%s.fits' % (obsdate)
dbfile = 'spec%s.db' % obsdate
#create the observation log
# obs_dict=obslog(infilelist)
verbose = True
with logging(logfile, debug) as log:
log.message('Pysalt Version: ' + pysalt.verno, with_header=False)
#prepare the data
for img in infilelist:
hdu = pyfits.open(img)
# for backwards compatibility
hdu = remove_duplicate_keys(hdu)
if not 'XTALK' in hdu[1].header:
hdu[1].header['XTALK'] = 1474
hdu[2].header['XTALK'] = 1474
hdu[3].header['XTALK'] = 1166
hdu[4].header['XTALK'] = 1111
hdu[5].header['XTALK'] = 1377
hdu[6].header['XTALK'] = 1377
img = os.path.basename(img)
hdu = prepare(hdu, createvar=False, badpixelstruct=None)
if not cleanup: hdu.writeto('p' + img, clobber=True)
hdu = bias(hdu,
subover=True,
trim=True,
subbias=False,
bstruct=None,
median=False,
function='polynomial',
order=5,
rej_lo=5.0,
rej_hi=5.0,
niter=10,
plotover=False,
log=log,
verbose=verbose)
if not cleanup: hdu.writeto('bp' + img, clobber=True)
# put windowed data into full image
exts = len(hdu)
if exts > 7:
rows, cols = hdu[1].data.shape
cbin, rbin = [
int(x) for x in hdu[0].header['CCDSUM'].split(" ")
]
ampsecO = hdu[1].header["AMPSEC"].strip("[]").split(",")
ampsecE = hdu[7].header["AMPSEC"].strip("[]").split(",")
rO = int((float(ampsecO[1].split(":")[0]) - 1.) / rbin)
rE = int((float(ampsecE[1].split(":")[0]) - 1.) / rbin)
keylist = ['BIASSEC', 'DATASEC', 'AMPSEC', 'CCDSEC', 'DETSEC']
oldlist = [
hdu[1].header[key].strip("[]").split(",")[1]
for key in keylist
]
newlist = 2 * ['1:' + str(int(0.5 + 4102 / rbin))
] + 3 * [str(int(rbin / 2)) + ':4102']
for amp in range(6):
hduO = hdu[amp + 1].copy()
hdu[amp + 1].data = np.zeros((4102 / rbin, cols))
hdu[amp + 1].data[rO:rO + rows] = hduO.data
hdu[amp + 1].data[rE:rE + rows] = hdu[amp + 7].data
hdu[amp + 1].update_header
for k, key in enumerate(keylist):
hdu[amp+1].header[key] = \
hdu[amp+1].header[key].replace(oldlist[k],newlist[k])
del hdu[7:]
hdu[0].header['NSCIEXT'] = 6
badpixelstruct = saltio.openfits(bpmfile)
hdu = add_variance(hdu, badpixelstruct)
#gain correct the data
if gaindb:
usedb = True
dblist = saltio.readgaindb(gaindb.strip())
else:
usedb = False
dblist = ''
hdu = gain(hdu,
mult=True,
usedb=usedb,
dblist=dblist,
log=log,
verbose=verbose)
if not cleanup: hdu.writeto('gbp' + img, clobber=True)
#cross talk correct the data
hdu = xtalk(hdu, [], log=log, verbose=verbose)
#cosmic ray clean the data
#only clean the object data
if crthresh == '':
thresh = 5.0
if hdu[0].header['GRATING'].strip() == 'PG0300': thresh = 7.0
else: thresh = crthresh
if hdu[0].header['CCDTYPE']=='OBJECT' and \
hdu[0].header['LAMPID']=='NONE' and \
hdu[0].header['INSTRUME']=='RSS':
if crthresh != False:
log.message('Cleaning CR using thresh={}'.format(thresh))
hdu = multicrclean(hdu,
crtype='edge',
thresh=thresh,
mbox=11,
bthresh=5.0,
flux_ratio=0.2,
bbox=25,
gain=1.0,
rdnoise=5.0,
fthresh=5.0,
bfactor=2,
gbox=3,
maxiter=5,
log=log,
verbose=verbose)
for ext in range(13, 19):
hdu[ext].data = hdu[ext].data.astype('uint8')
hdu[0].header.add_history(
'CRCLEAN: multicrclean, thresh = ', thresh)
else:
hdu[0].header.add_history('CRCLEAN: None')
hdu.writeto('xgbp' + img, clobber=True)
hdu.close()
#mosaic the data
#khn: attempt to use most recent previous geometry to obsdate.
#NOTE: mosaicing does not do this correctly
#geomdb = open(datadir+"RSSgeom.dat",'r')
#for geomline in geomdb:
# if geomline[0]=='#': continue
# if (int(obsdate) > int(geomline.split(' ')[0].replace('-',''))): break
#geomfile = "RSSgeom_obsdate.dat"
#open(geomfile,'w').write(geomline)
geomfile = iraf.osfn("pysalt$data/rss/RSSgeom.dat")
try:
saltmosaic('xgbpP*fits',
'',
'm',
geomfile,
interp='linear',
cleanup=True,
geotran=True,
clobber=True,
logfile=logfile,
verbose=True)
except:
saltmosaic('xgbpP*fits',
'',
'm',
geomfile,
interp='linear',
cleanup=True,
geotran=True,
clobber=True,
logfile=logfile,
verbose=True)
#khn: fix mosaiced VAR and BPM extensions
#khn: fix mosaiced bpm missing some of gap
for img in infilelist:
filename = 'mxgbp' + os.path.basename(img)
hdu = pyfits.open(filename, 'update')
hdu[2].header['EXTNAME'] = 'VAR'
hdu[3].header['EXTNAME'] = 'BPM'
bpm_rc = (hdu[3].data > 0).astype('uint8')
zeroscicol = hdu['SCI'].data.sum(axis=0) == 0
bpmgapcol = bpm_rc.mean(axis=0) == 1
addbpmcol = zeroscicol & ~bpmgapcol
addbpmcol[np.argmax(addbpmcol) -
4:np.argmax(addbpmcol)] = True # allow for chip tilt
bpm_rc[:, addbpmcol] = 1
hdu[3].data = bpm_rc
hdu.writeto(filename, clobber=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'):
os.remove(f)
for f in glob.glob('bp*fits'):
os.remove(f)
for f in glob.glob('gbp*fits'):
os.remove(f)
for f in glob.glob('xgbp*fits'):
os.remove(f)
def specred(rawdir, prodir, imreduce=True, specreduce=True, calfile=None, lamp='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
print rawdir
print prodir
#get the name of the files
infile_list=glob.glob(rawdir+'*.fits')
infiles=','.join(['%s' % x for x in infile_list])
#get the current date for the files
obsdate=os.path.basename(infile_list[0])[1:9]
print obsdate
#set up some files that will be needed
logfile='spec'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
#create the observation log
obs_dict=obslog(infile_list)
if imreduce:
#prepare the data
saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)
#bias subtract the data
saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',
median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0,
niter=10, plotover=False, turbo=False,
clobber=True, logfile=logfile, verbose=True)
#gain correct the data
saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)
#cross talk correct the data
saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)
#cosmic ray clean the data
#only clean the object data
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
img='xgbp'+os.path.basename(infile_list[i])
saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, multithread=True, clobber=True, logfile=logfile, verbose=True)
#flat field correct the data
flat_imgs=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('FLAT'):
if flat_imgs: flat_imgs += ','
flat_imgs += 'xgbp'+os.path.basename(infile_list[i])
if len(flat_imgs)!=0:
saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False, \
weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3, \
hthresh=3, clobber=True, logfile=logfile, verbose=True)
saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)
saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
else:
flats=None
imfiles=glob.glob('cxgbpP*fits')
for f in imfiles:
shutil.copy(f, 'f'+f)
#mosaic the data
geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
for f in glob.glob('fxgbp*fits'): os.remove(f)
#set up the name of the images
if specreduce:
for i in range(len(infile_list)):
if obs_dict['OBJECT'][i].upper().strip()=='ARC':
lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
arcimage='mfxgbp'+os.path.basename(infile_list[i])
lampfile=iraf.osfn("pysalt$data/linelists/%s.txt" % lamp)
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod=automethod, function='legendre', order=5,
rstep=100, rstart='middlerow', mdiff=10, thresh=3, niter=5,
inter=True, clobber=True, logfile=logfile, verbose=True)
specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
objimages=''
for i in range(len(infile_list)):
if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
if objimages: objimages += ','
objimages+='mfxgbp'+os.path.basename(infile_list[i])
if specreduce:
#run specidentify on the arc files
specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
blank=0.0, clobber=True, logfile=logfile, verbose=True)
#create the spectra text files for all of our objects
spec_list=[]
for img in objimages.split(','):
spec_list.extend(createspectra('x'+img, obsdate, smooth=False, skysection=skysection, clobber=True))
print spec_list
#determine the spectrophotometric standard
extfile=iraf.osfn('pysalt$data/site/suth_extinct.dat')
for spec, am, et, pc in spec_list:
if pc=='CAL_SPST':
stdstar=spec.split('.')[0]
print stdstar, am, et
stdfile=iraf.osfn('pysalt$data/standards/spectroscopic/m%s.dat' % stdstar.lower().replace('-', '_'))
print stdfile
ofile=spec.replace('txt', 'sens')
calfile=ofile #assumes only one observations of a SP standard
specsens(spec, ofile, stdfile, extfile, airmass=am, exptime=et,
stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
clobber=True, logfile='salt.log',verbose=True)
for spec, am, et, pc in spec_list:
if pc!='CAL_SPST':
ofile=spec.replace('txt', 'spec')
speccal(spec, ofile, calfile, extfile, airmass=am, exptime=et,
clobber=True, logfile='salt.log',verbose=True)
#clean up the spectra for bad pixels
cleanspectra(ofile)
