def saltdataquality(obsdate, sdbhost = "sdb.saao", sdbname = "sdb", sdbuser = "",
password = '', clobber=False, logfile='salt.log',verbose=True):
"""For a given SALT data. Move the data to /salt/data after it has been run by the pipeline"""
#make a dataquality directory
dqdir='dq%s' % obsdate
if os.path.isdir(dqdir) and clobber:
saltio.deletedir(dqdir)
saltio.createdir(dqdir)
saltio.changedir(dqdir)
with logging(logfile,debug) as log:
#check the entries
saltio.argdefined('obsdate',str(obsdate))
#open up the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#run for each instrument
for instrume in ['rss', 'scam']:
log.message('Measuring Data Quality for %s observations' % instrume.upper())
dataquality(str(obsdate), sdb, instrume, clobber, logfile, verbose)
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrbnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrbnum=0
try:
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits')))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=lasthrbnum
time_unit = ' min'
else:
processing_time = float(int(processing_time / 360 + 0.5)) / 10
time_unit = ' hrs'
#output the message with this information
message = 'Pipeline Statistics:\n'
message += 'Processing time: %s %s\n' % (str(processing_time), time_unit)
message += 'Number of files: %i\n' % (rssrawnum+scmrawnum+hrsrawnum)
message += 'Total Raw Data: %s %s\n' % (rawsize, rawunit)
message += 'Total Product Data: %s %s\n' % (prodsize, produnit)
#message += 'Number of Proposals: \n' % (1)
log.message(message)
sdb.close()
#return to the original working directory
saltio.changedir(basedir)
def calcsizeunit(size):
if (size < 1.e6):
rawsize = str(int(size / 1e3))
rawunit = 'KB'
elif (size >= 1.e6 and size < 1.e9):
rawsize = str(int(size / 1e6))
rawunit = 'MB'
else:
rawsize = str(float(int(size / 1e8 + 0.5)) / 10)
rawunit = 'GB'
return rawsize, rawunit
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=0
if lastrssnum>1 or lastscmnum>1:
message = 'Copy of data is complete'
log.message(message)
else:
message = 'No data was taken on %s' % obsdate
log.message(message)
#process the data RSS data
if lastrssnum>1:
preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose)
#process the SCAM data
if lastscmnum>1:
preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose)
#process the HRS data
if lasthrsnum>1:
preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose)
preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose)
#check that all data was given a proper proposal id
#only do it for semesters after the start of science operations
if int(obsdate)>=20110901:
# Check to see that the PROPID keyword exists and if not add it
message = '\nSALTPIPE -- Checking for PROPID keyword'
log.message(message)
#check rss data
rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log)
#check scam data
scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log)
#check hrsB data
hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log)
#check hrsB data
hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log)
if not rssstatus or not scmstatus or not hrsbstatus or not hrsrstatus:
msg='The PROPIDs for these files needs to be updated and re-start the pipeline'
raise SaltError("Invalid PROPID in images:"+msg)
#process the RSS data
rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#advance process the data
if rssrawnum > 0:
advanceprocess('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose)
#process the SCAM data
scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#process the HRS data
hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose)
#upload the data to the database
img_list=glob.glob(workpath+'scam/product/*bxgp*.fits')
img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits'))
img_list.extend(glob.glob(workpath+'hrs/raw/*.fits'))
if img_list:
img=','.join('%s' % (k) for k in img_list)
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
#add junk sources to the database
raw_list=glob.glob(workpath+'scam/raw/S*.fits')
raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits'))
if raw_list:
img=''
for img in raw_list:
hdu=pyfits.open(img)
if hdu[0].header['PROPID'].strip()=='JUNK':
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
hdu.close()
# construct observation and pipeline documentation
if lastrssnum > 1 and rssrawnum>0:
rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits'
else:
rssobslog = 'none'
if lastscmnum > 1 and scmrawnum>0:
scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits'
else:
scmobslog = 'None'
if lasthrsnum > 1 and hrsrawnum>0:
hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits'
else:
hrsobslog = 'None'
if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0:
msg='No data processed for %s' % obsdate
email=False
ftp=False
log.message(msg)
htmlpath = '.'
nightlog = '../nightlogfiles/' + obsdate + '.log'
readme = iraf.osfn('pipetools$html/readme.template')
if not os.path.isfile(nightlog):
nightlog = ''
message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found'
log.warning(message)
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath,
nightlog=nightlog,readme=readme,clobber=True,logfile=logfile,
verbose=verbose)
#add a pause to allow syncing of the databases
time.sleep(10)
#Add in the environmental information
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
propids=saltmysql.getpropcodes(sdb, obsdate)
for pid in propids:
saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose)
try:
outfile='%s_%s_elsdata.fits' % (pid, obsdate)
outdir='%s/doc/' % (pid)
shutil.move(outfile, outdir)
except:
os.remove(outfile)
#ftp the data
beachdir='/salt/ftparea/'
if ftp:
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=False,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)
#run with the splitting of files
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=True,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)