def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=0
if lastrssnum>1 or lastscmnum>1:
message = 'Copy of data is complete'
log.message(message)
else:
message = 'No data was taken on %s' % obsdate
log.message(message)
#process the data RSS data
if lastrssnum>1:
preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose)
#process the SCAM data
if lastscmnum>1:
preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose)
#process the HRS data
if lasthrsnum>1:
preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose)
preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose)
#check that all data was given a proper proposal id
#only do it for semesters after the start of science operations
if int(obsdate)>=20110901:
# Check to see that the PROPID keyword exists and if not add it
message = '\nSALTPIPE -- Checking for PROPID keyword'
log.message(message)
#check rss data
rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log)
#check scam data
scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log)
#check hrsB data
hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log)
#check hrsB data
hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log)
if not rssstatus or not scmstatus or not hrsbstatus or not hrsrstatus:
msg='The PROPIDs for these files needs to be updated and re-start the pipeline'
raise SaltError("Invalid PROPID in images:"+msg)
#process the RSS data
rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#advance process the data
if rssrawnum > 0:
advanceprocess('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose)
#process the SCAM data
scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#process the HRS data
hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose)
#upload the data to the database
img_list=glob.glob(workpath+'scam/product/*bxgp*.fits')
img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits'))
img_list.extend(glob.glob(workpath+'hrs/raw/*.fits'))
if img_list:
img=','.join('%s' % (k) for k in img_list)
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
#add junk sources to the database
raw_list=glob.glob(workpath+'scam/raw/S*.fits')
raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits'))
if raw_list:
img=''
for img in raw_list:
hdu=pyfits.open(img)
if hdu[0].header['PROPID'].strip()=='JUNK':
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
hdu.close()
# construct observation and pipeline documentation
if lastrssnum > 1 and rssrawnum>0:
rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits'
else:
rssobslog = 'none'
if lastscmnum > 1 and scmrawnum>0:
scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits'
else:
scmobslog = 'None'
if lasthrsnum > 1 and hrsrawnum>0:
hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits'
else:
hrsobslog = 'None'
if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0:
msg='No data processed for %s' % obsdate
email=False
ftp=False
log.message(msg)
htmlpath = '.'
nightlog = '../nightlogfiles/' + obsdate + '.log'
readme = iraf.osfn('pipetools$html/readme.template')
if not os.path.isfile(nightlog):
nightlog = ''
message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found'
log.warning(message)
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath,
nightlog=nightlog,readme=readme,clobber=True,logfile=logfile,
verbose=verbose)
#add a pause to allow syncing of the databases
time.sleep(10)
#Add in the environmental information
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
propids=saltmysql.getpropcodes(sdb, obsdate)
for pid in propids:
saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose)
try:
outfile='%s_%s_elsdata.fits' % (pid, obsdate)
outdir='%s/doc/' % (pid)
shutil.move(outfile, outdir)
except:
os.remove(outfile)
#ftp the data
beachdir='/salt/ftparea/'
if ftp:
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=False,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)
#run with the splitting of files
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=True,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrbnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrbnum=0
try:
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits')))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=lasthrbnum
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None,
subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3,
niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct=saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('OBJECT')
biaslist=filename[ccdtype=='Bias']
masterbias_dict={}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bgph'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose, with_header=False)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=False, imstack=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list)
#create the master bias frame
for i in list(masterbias_dict.keys()):
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'mbgph'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list)
if masterbias:
subbias=True
bstruct=saltio.openfits(masterbias)
else:
subbias=False
bstruct=None
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=subbias, imstack=True,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
return
def hrsclean(images,
outpath,
obslogfile=None,
subover=True,
trim=True,
masbias=None,
subbias=True,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile)
and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct = saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('OBJECT')
biaslist = filename[ccdtype == 'Bias']
masterbias_dict = {}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = pyfits.open(img)
bimg = outpath + 'bgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message,
with_stdout=verbose,
with_header=False)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=False,
imstack=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=hrsbiasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(
blist, bkeys, x1=5, x2=13)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg = os.path.basename(img)
if not nimg in biaslist:
#open the image
struct = pyfits.open(img)
simg = outpath + 'mbgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias = get_masterbias(struct,
masterbias_dict,
keylist=hrsbiasheader_list)
if masterbias:
subbias = True
bstruct = saltio.openfits(masterbias)
else:
subbias = False
bstruct = None
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=subbias,
imstack=True,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
return
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
"""Split data into their different data directories
"""
with logging(logfile,debug) as log:
# are the arguments defined
pids = saltio.argunpack('propcode',propcode)
# check observation log file exists
obslog = obslog.strip()
saltio.fileexists(obslog)
#open the observing log
obstruct = saltio.openfits(obslog)
obstab = saltio.readtab(obstruct[1],obslog)
saltio.closefits(obstruct)
#read in the file information
filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
propids = saltstring.listfunc(obstab.field('propid'),'clean')
ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
filters = saltstring.listfunc(obstab.field('filter'),'clean')
gratings = saltstring.listfunc(obstab.field('grating'),'clean')
gr_angles = obstab.field('gr-angle')
ar_angles = obstab.field('ar-angle')
# Create the list of proposals
try:
pids=saltio.cleanpropcode(pids, propids)
except SaltIOError:
#throw a warning adn exit if not data needs to be filterd
log.warning('No data to filter\n', with_stdout=verbose)
return
# check paths exist, end with a "/" and convert them to absolute paths
rawpath = saltio.abspath(rawpath)
prodpath = saltio.abspath(prodpath)
outpath = saltio.abspath(outpath)
#create the symlink raw path
rawsplit=rawpath.strip().split('/')
symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
prodsplit=prodpath.strip().split('/')
symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
# create PI directories
for pid in pids:
saltio.createdir(outpath+pid)
saltio.createdir(outpath+pid+'/raw')
saltio.createdir(outpath+pid+'/product')
#copy the data that belongs to a pid into that directory
log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)
#copy data for a given proposal to the raw and produce directories
for i in range(len(obstab)):
if os.path.exists(outpath+obstab[i]['propid']):
if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
fname=obstab[i]['filename']
pdir=obstab[i]['propid']
detmode=obstab[i]['detmode']
linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#look through the bias/flat/arc/standard data to see if there is any relavent data
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
caldata=['ZERO', 'FLAT', 'ARC']
biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
calproplist=['CAL_SPST']
#Include bias frames
log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a zero, check to see what other data have the same settings
if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], biasheader_list):
prop_list.append(obstab[i]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include calibration frames
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a flat, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a arc, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], archeader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a calibration standard, see what other data have the same settings
if obstab[i]['PROPID'].strip().upper() in calproplist:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
if pdir!=obstab[i]['propid']:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include master (bias or flat) frames
log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
for bimg in masterlist:
struct=pyfits.open(bimg)
bdict={}
prop_list=[]
for k in biasheader_list:
bdict[k]=saltkey.get(k, struct[0])
for i in range(len(obstab)):
if comparefiles(obstab[i], bdict, biasheader_list):
prop_list.append(obstab[i]['PROPID'])
struct.close()
#copy the files over to the directory
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
fname=os.path.basename(bimg)
infile = symprodpath+fname
link = outpath+pdir+'/product/'+fname
saltio.symlink(infile,link,clobber)
message='Copying %s to %s' % (fname ,pdir)
log.message(message, with_header=False, with_stdout=verbose)