示例#1
0
def saltdataquality(obsdate, sdbhost = "sdb.saao", sdbname = "sdb", sdbuser = "", 
                    password = '', clobber=False, logfile='salt.log',verbose=True):
   """For a given SALT data.  Move the data to /salt/data after it has been run by the pipeline"""
   #make a dataquality directory
   dqdir='dq%s' % obsdate
   if os.path.isdir(dqdir) and clobber:
      saltio.deletedir(dqdir)
   saltio.createdir(dqdir)
   saltio.changedir(dqdir)

   with logging(logfile,debug) as log:

       #check the entries
       saltio.argdefined('obsdate',str(obsdate))

       #open up the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)


       #run for each instrument
       for instrume in ['rss', 'scam']:
           log.message('Measuring Data Quality for %s observations' % instrume.upper())
           dataquality(str(obsdate), sdb, instrume, clobber, logfile, verbose)
示例#2
0
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
             ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, 
             elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
             clobber, runstatus, logfile,verbose):

   # set up

   basedir=os.getcwd()
   propcode=pinames
   sender = emuser + '@salt.ac.za'
   recipient = sender
   emessage = ''
   emailfile = '../piemaillist/email.lis'

   # check the observation date is sensible
   if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
       emessage = 'Observation date does not look sensible - YYYYMMDD\n'
       raise SaltError(emessage)


   # stop if the obsdate temporary directory already exists

   obsdir='%s' % obsdate
   if os.path.exists(obsdir):
       emessage += 'The temporary working directory ' + os.getcwd() + '/'
       emessage += obsdate + ' already exists. '
       raise SaltError(emessage)

   # create a temporary working directory and move to it
   saltio.createdir(obsdir)
   saltio.changedir(obsdir)
   workpath = saltio.abspath('.')

   # test the logfile
   logfile = workpath+logfile
   logfile = saltio.logname(logfile)

   #note the starttime
   starttime = time.time()

   #start logging
   with logging(logfile,debug) as log:

       #connect to the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
       
       #get the nightinfo id   
       nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)

       #Get the list of proposal codes
       state_select='Proposal_Code'
       state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
       state_logic="current=1"
       records=saltmysql.select(sdb, state_select, state_tables, state_logic)
       propids=[k[0] for k in records]

       # Calculate the current date
       currentdate=salttime.currentobsdate()

       # are the arguments defined
       saltio.argdefined('obsdate',obsdate)

       # check email and ftp arguments are consistent
       if email and not ftp:
           message =  'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
           message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
           raise SaltError(message)

       # identify a potential list of keyword edits
       keyfile = '../newheadfiles/list_newhead_' + obsdate
       if not os.path.isfile(keyfile):
           message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
           log.message(message)


       # check directories for the raw RSS data
       rssrawpath = makerawdir(obsdate, 'rss')

       # check directories for the raw SALTICAM data
       scmrawpath = makerawdir(obsdate, 'scam')


       # check raw directories for the disk.file record and find last file number
       #check rss data
       lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
       #check scame data
       lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
       #check for HRS Data--not filedata yet, so cannot check
   
       if lastrssnum == 1 and lastscmnum == 1:
           message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
           emessage += '\n' + message + '\n'
           log.message(message)

       #copy the data to the working directory
       if lastrssnum > 1:
           message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(rssrawpath,'rss/raw')
       if lastscmnum > 1:
           message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(scmrawpath,'scam/raw')

       #copy and pre-process the HRS data
       try:
           hrsbrawpath = makerawdir(obsdate, 'hbdet')
           saltio.createdir('hrs')
           saltio.createdir('hrs/raw')
           message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)

           lasthrbnum=len(glob.glob('hrs/raw/*fits'))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrbnum=0

       try:
           hrsrrawpath = makerawdir(obsdate, 'hrdet')
           message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
           
           lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits')))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrsnum=lasthrbnum
示例#3
0
          time_unit = ' min'
       else:
          processing_time = float(int(processing_time / 360 + 0.5)) / 10
          time_unit = ' hrs'
       #output the message with this information
       message = 'Pipeline Statistics:\n'
       message += 'Processing time: %s %s\n' %  (str(processing_time), time_unit)
       message += 'Number of files: %i\n' %  (rssrawnum+scmrawnum+hrsrawnum)
       message += 'Total Raw Data: %s %s\n' % (rawsize, rawunit)
       message += 'Total Product Data: %s %s\n' % (prodsize, produnit)
       #message += 'Number of Proposals: \n'  % (1)
       log.message(message)
   sdb.close()

   #return to the original working directory
   saltio.changedir(basedir)


def calcsizeunit(size):
    if (size < 1.e6):
        rawsize = str(int(size / 1e3))
        rawunit = 'KB'
    elif (size >= 1.e6 and size < 1.e9):
        rawsize = str(int(size / 1e6))
        rawunit = 'MB'
    else:
        rawsize = str(float(int(size / 1e8 + 0.5)) / 10)
        rawunit = 'GB'
    return rawsize, rawunit

 
示例#4
0
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
             ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, 
             elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
             clobber, runstatus, logfile,verbose):

   # set up

   basedir=os.getcwd()
   propcode=pinames
   sender = emuser + '@salt.ac.za'
   recipient = sender
   emessage = ''
   emailfile = '../piemaillist/email.lis'

   # check the observation date is sensible
   if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
       emessage = 'Observation date does not look sensible - YYYYMMDD\n'
       raise SaltError(emessage)


   # stop if the obsdate temporary directory already exists

   obsdir='%s' % obsdate
   if os.path.exists(obsdir):
       emessage += 'The temporary working directory ' + os.getcwd() + '/'
       emessage += obsdate + ' already exists. '
       raise SaltError(emessage)

   # create a temporary working directory and move to it
   saltio.createdir(obsdir)
   saltio.changedir(obsdir)
   workpath = saltio.abspath('.')

   # test the logfile
   logfile = workpath+logfile
   logfile = saltio.logname(logfile)

   #note the starttime
   starttime = time.time()

   #start logging
   with logging(logfile,debug) as log:

       #connect to the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
       
       #get the nightinfo id   
       nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)

       #Get the list of proposal codes
       state_select='Proposal_Code'
       state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
       state_logic="current=1"
       records=saltmysql.select(sdb, state_select, state_tables, state_logic)
       propids=[k[0] for k in records]

       # Calculate the current date
       currentdate=salttime.currentobsdate()

       # are the arguments defined
       saltio.argdefined('obsdate',obsdate)

       # check email and ftp arguments are consistent
       if email and not ftp:
           message =  'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
           message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
           raise SaltError(message)

       # identify a potential list of keyword edits
       keyfile = '../newheadfiles/list_newhead_' + obsdate
       if not os.path.isfile(keyfile):
           message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
           log.message(message)


       # check directories for the raw RSS data
       rssrawpath = makerawdir(obsdate, 'rss')

       # check directories for the raw SALTICAM data
       scmrawpath = makerawdir(obsdate, 'scam')


       # check raw directories for the disk.file record and find last file number
       #check rss data
       lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
       #check scame data
       lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
       #check for HRS Data--not filedata yet, so cannot check
   
       if lastrssnum == 1 and lastscmnum == 1:
           message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
           emessage += '\n' + message + '\n'
           log.message(message)

       #copy the data to the working directory
       if lastrssnum > 1:
           message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(rssrawpath,'rss/raw')
       if lastscmnum > 1:
           message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(scmrawpath,'scam/raw')

       #copy and pre-process the HRS data
       try:
           hrsbrawpath = makerawdir(obsdate, 'hbdet')
           saltio.createdir('hrs')
           saltio.createdir('hrs/raw')
           message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)

           hrsrrawpath = makerawdir(obsdate, 'hrdet')
           message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
           
           lasthrsnum=len(glob.glob('hrs/raw/*fits'))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrsnum=0

       if lastrssnum>1 or lastscmnum>1:
           message = 'Copy of data is complete'
           log.message(message)
       else:
           message = 'No data was taken on %s' % obsdate
           log.message(message)


       #process the data RSS data
       if lastrssnum>1:
           preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose)

       #process the  SCAM data
       if lastscmnum>1:
           preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose)

       #process the HRS data
       if lasthrsnum>1:
           preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose)
           preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose)
         


       #check that all data was given a proper proposal id
       #only do it for semesters after the start of science operations
       if int(obsdate)>=20110901:
           # Check to see that the PROPID keyword exists and if not add it
           message = '\nSALTPIPE -- Checking for PROPID keyword'
           log.message(message)

           #check rss data
           rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log)
           #check scam data
           scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log)
           #check hrsB data
           hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log)
           #check hrsB data
           hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log)

           if not rssstatus  or not scmstatus or not hrsbstatus or not hrsrstatus: 
               msg='The PROPIDs for these files needs to be updated and re-start the pipeline'
               raise SaltError("Invalid PROPID in images:"+msg)

       #process the RSS data
       rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate,  propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
      
       #advance process the data
       if rssrawnum > 0:
           advanceprocess('rss', obsdate,  propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose)

       #process the SCAM data
       scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode,  median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)

       #process the HRS data
       hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose)

       #upload the data to the database
       img_list=glob.glob(workpath+'scam/product/*bxgp*.fits')
       img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits'))
       img_list.extend(glob.glob(workpath+'hrs/raw/*.fits'))
       if img_list:
           img=','.join('%s' %  (k) for k in img_list)
           saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
                  password=sdbpass, logfile=logfile, verbose=verbose)

       #add junk sources to the database
       raw_list=glob.glob(workpath+'scam/raw/S*.fits')
       raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits'))
       if raw_list:
          img=''
          for img in raw_list:
              hdu=pyfits.open(img)
              if hdu[0].header['PROPID'].strip()=='JUNK':
                saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
                    password=sdbpass, logfile=logfile, verbose=verbose)
              hdu.close()
       

       # construct observation and pipeline documentation
       if lastrssnum > 1 and rssrawnum>0:
           rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits'
       else:
           rssobslog = 'none'

       if lastscmnum > 1 and scmrawnum>0:
           scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits'
       else:
           scmobslog = 'None'

       if lasthrsnum > 1 and hrsrawnum>0:
           hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits'
       else:
           hrsobslog = 'None'



       if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0:
           msg='No data  processed for  %s' % obsdate
           email=False
           ftp=False
           log.message(msg)

       htmlpath = '.'
       nightlog = '../nightlogfiles/' + obsdate + '.log'
       readme = iraf.osfn('pipetools$html/readme.template')
       if not os.path.isfile(nightlog):
           nightlog = ''
           message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found'
           log.warning(message)

       if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
           salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath,
                      nightlog=nightlog,readme=readme,clobber=True,logfile=logfile,
                      verbose=verbose)

       #add a pause to allow syncing of the databases
       time.sleep(10)

       #Add in the environmental information
       if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
           propids=saltmysql.getpropcodes(sdb, obsdate)
           for pid in propids:
               saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass,
                           sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose) 
               try:
                  outfile='%s_%s_elsdata.fits' % (pid, obsdate)
                  outdir='%s/doc/' % (pid)
                  shutil.move(outfile, outdir)
               except:
                  os.remove(outfile)


       #ftp the data
       beachdir='/salt/ftparea/'
       if ftp:
           try:
               saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
                   password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
                   sdbname=sdbname,sdbuser=sdbuser,splitfiles=False, 
                   cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
           except Exception,e:
               message="Not able to copy data to FTP area:\n%s " % e
               raise SaltError(message)     
           #run with the splitting of files
           try: 
               saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
                   password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
                   sdbname=sdbname,sdbuser=sdbuser,splitfiles=True, 
                   cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
           except Exception,e:
               message="Not able to copy data to FTP area:\n%s " % e
               raise SaltError(message)