Esempio n. 1
0
def saltelserror(obsdate, elshost, elsname, elsuser, elspass, 
            sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):

   # set up

   proposers = []
   propids = []
   pids = []
   
   with logging(logfile,debug) as log:

       #open the database
       els=saltmysql.connectdb(elshost, elsname, elsuser, elspass)
       sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)  

       #create the values for the entire night
       nid=saltmysql.getnightinfoid(sdb, obsdate)
       stime, etime=saltmysql.select(sdb, 'EveningTwilightEnd,MorningTwilightStart', 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
       print stime, etime

       errors=gettcserrors(els, stime, etime)
       if not errors: return None

       for e in errors:
           i=e.index('T')
           print i, error_list[i], e[-1]
       
       print len(errors)
Esempio n. 2
0
def erroremail(obsdate, server='', username='', password='', sender='', recipient='', bcc='',
              sdbhost='sdb.salt', sdbname='sdb', sdbuser='', sdbpass='', logfile='saltlog.log', verbose=True):
    """Update the PipelineStatistics table with the current information
       about the pipeline

    """

    with logging(logfile,debug) as log:

       #connect the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)

       #get the nightinfo_id
       night_id=saltmysql.getnightinfoid(sdb, obsdate)

       #create the message

       try:
          record=saltmysql.select(sdb, 'PipelineStatus, ErrorMessage', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % night_id)
          status, error_message=record[0]
       except Exception, e:
          raise SaltError('Unable to download information: %s' % e)

       #set up the subject and message
       subject='Pipeline Error on %s' % obsdate
       
       message="Failure is simply the opportunity to begin again, this time more intelligently.--Henry Ford\n\n"  
       message+="%s" %  error_message
       log.message(message, with_stdout=verbose)

       #send the email
       saltio.email(server,username,password,sender,recipient,bcc, subject,message)
Esempio n. 3
0
def saltweather(weathertime, timespan, elshost, elsname, elsuser, elspass, sdbhost, sdbname, sdbuser, password):
    print weathertime

    # open the database
    els = saltmysql.connectdb(elshost, elsname, elsuser, elspass)
    sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    # determine the obsdate
    obsdate = weathertime - datetime.timedelta(seconds=43200)
    obsdate = "%i%s%s" % (obsdate.year, string.zfill(obsdate.month, 2), string.zfill(obsdate.day, 2))
    nid = saltmysql.getnightinfoid(sdb, obsdate)

    print nid
    # get the most recent weather data
    airpressure, dewout, extrelhum, temp2m, temp30m, windspeed, winddir, rain = getweatherdata(
        els, weathertime, timespan
    )
    dewin, relhum = getinsidedata(els, weathertime, timespan)
    tempin = getacdata(els, weathertime, timespan)

    # upload that to the sdb
    upcmd = "Weather_Time='%s', NightInfo_Id=%i" % (weathertime, nid)

    if tempin is not None:
        upcmd += ",TemperatureInside=%4.2f" % tempin
    if temp2m is not None:
        upcmd += ",Temperature2m=%4.2f" % temp2m
    if temp30m is not None:
        upcmd += ",Temperature30m=%4.2f" % temp30m
    if windspeed is not None:
        upcmd += ",WindSpeed=%5.2f" % windspeed
    if winddir is not None:
        upcmd += ",WindDirection=%5.2f" % winddir
    if dewin is not None:
        upcmd += ",DewPointInside=%3.2f" % dewin
    if dewout is not None:
        upcmd += ",DewPointOutside=%3.2f" % dewout
    if airpressure is not None:
        upcmd += ",AirPressure=%4.2f" % airpressure
    if relhum is not None:
        upcmd += ",RelativeHumidty=%4.2f" % relhum
    if extrelhum is not None:
        upcmd += ",ExternalRelativeHumidity=%4.2f" % extrelhum
    if rain is not None:
        upcmd += ",Rain=%i" % rain

    saltmysql.insert(sdb, upcmd, "Weather")
    print upcmd

    return
Esempio n. 4
0
def pipelinestatus(obsdate, status, message=None, rawsize=None, reducedsize=None, 
              runtime=None, emailsent=None, sdbhost='sdb.saao', sdbname='sdb',  sdbuser='', password='', 
              logfile='saltlog.log', verbose=True):
    """Update the PipelineStatistics table with the current information
       about the pipeline

    """

    with logging(logfile,debug) as log:

       #connect the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #get the nightinfo_id
       night_id=saltmysql.getnightinfoid(sdb, obsdate)

       #get the status_id for the given status
       status_id=getstatusid(sdb, status)
           
       #create the insert command
       obsdate=str(obsdate)
       inst_cmd="NightInfo_Id=%s, PipelineStatus_Id=%i" % (night_id, status_id)
       if status_id>10:
          inst_cmd+=',ErrorMessage="%s"' % message
       if rawsize is not None: 
          inst_cmd+=",RawSize=%f" % rawsize
       if reducedsize is not None: 
          inst_cmd+=",ReducedSize=%f" % rawsize
       if runtime is not None:
          inst_cmd+=",PipelineRunTime=%i" % runtime
       if emailsent is not None:
          inst_cmd+=",EmailSent=%i" % emailsent 
       print inst_cmd
     
       #insert or update the pipeline
       if checktable(sdb, night_id): 
          saltmysql.update(sdb, inst_cmd, 'PipelineStatistics', 'NightInfo_Id=%i' % night_id)
          msg="Updating information for Night_Id=%i\n" % night_id
       else:
          saltmysql.insert(sdb, inst_cmd, 'PipelineStatistics')
          msg="Inserting  information for Night_Id=%i\n" % night_id
          

       #log the call
       log.message(msg+inst_cmd, with_stdout=verbose)
Esempio n. 5
0
def updatenightinfo(obsdate, sdbhost='sdb.saao', sdbname='sdb', \
              sdbuser='', password='', logfile='saltlog.log', verbose=True):
    """Update the nightinfo table with current information about the
       night

    """

    with logging(logfile,debug) as log:

       #connect the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #get the nightinfo_id
       try:
           night_id=saltmysql.getnightinfoid(sdb, obsdate)
       except SaltError:
           night_id=None
           
       #get the information for the night
       time_list=get_nightdetails(obsdate)

       #create the insert command
       obsdate=str(obsdate)
       inst_cmd="Date='%s-%s-%s'," % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
       inst_cmd+="SunSet='%s', SunRise='%s', MoonSet='%s', MoonRise='%s', EveningTwilightEnd='%s', MorningTwilightStart='%s'" % \
               (time_list[0], time_list[1], time_list[2], time_list[3], time_list[4], time_list[5])
       inst_cmd+=",MoonPhase_Percent=%i" % (round(float(time_list[6])))

       if night_id: 
          saltmysql.update(sdb, inst_cmd, 'NightInfo', 'NightInfo_Id=%i' % night_id)
          msg="Updating information for Night_Id=%i\n" % night_id
       else:
          saltmysql.insert(sdb, inst_cmd, 'NightInfo')
          msg="Inserting  information for Night_Id=%i\n" % night_id
          

       #log the call
       log.message(msg+inst_cmd, with_stdout=verbose)
Esempio n. 6
0
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
             ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, 
             elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
             clobber, runstatus, logfile,verbose):

   # set up

   basedir=os.getcwd()
   propcode=pinames
   sender = emuser + '@salt.ac.za'
   recipient = sender
   emessage = ''
   emailfile = '../piemaillist/email.lis'

   # check the observation date is sensible
   if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
       emessage = 'Observation date does not look sensible - YYYYMMDD\n'
       raise SaltError(emessage)


   # stop if the obsdate temporary directory already exists

   obsdir='%s' % obsdate
   if os.path.exists(obsdir):
       emessage += 'The temporary working directory ' + os.getcwd() + '/'
       emessage += obsdate + ' already exists. '
       raise SaltError(emessage)

   # create a temporary working directory and move to it
   saltio.createdir(obsdir)
   saltio.changedir(obsdir)
   workpath = saltio.abspath('.')

   # test the logfile
   logfile = workpath+logfile
   logfile = saltio.logname(logfile)

   #note the starttime
   starttime = time.time()

   #start logging
   with logging(logfile,debug) as log:

       #connect to the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
       
       #get the nightinfo id   
       nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)

       #Get the list of proposal codes
       state_select='Proposal_Code'
       state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
       state_logic="current=1"
       records=saltmysql.select(sdb, state_select, state_tables, state_logic)
       propids=[k[0] for k in records]

       # Calculate the current date
       currentdate=salttime.currentobsdate()

       # are the arguments defined
       saltio.argdefined('obsdate',obsdate)

       # check email and ftp arguments are consistent
       if email and not ftp:
           message =  'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
           message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
           raise SaltError(message)

       # identify a potential list of keyword edits
       keyfile = '../newheadfiles/list_newhead_' + obsdate
       if not os.path.isfile(keyfile):
           message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
           log.message(message)


       # check directories for the raw RSS data
       rssrawpath = makerawdir(obsdate, 'rss')

       # check directories for the raw SALTICAM data
       scmrawpath = makerawdir(obsdate, 'scam')


       # check raw directories for the disk.file record and find last file number
       #check rss data
       lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
       #check scame data
       lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
       #check for HRS Data--not filedata yet, so cannot check
   
       if lastrssnum == 1 and lastscmnum == 1:
           message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
           emessage += '\n' + message + '\n'
           log.message(message)

       #copy the data to the working directory
       if lastrssnum > 1:
           message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(rssrawpath,'rss/raw')
       if lastscmnum > 1:
           message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(scmrawpath,'scam/raw')

       #copy and pre-process the HRS data
       try:
           hrsbrawpath = makerawdir(obsdate, 'hbdet')
           saltio.createdir('hrs')
           saltio.createdir('hrs/raw')
           message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)

           lasthrbnum=len(glob.glob('hrs/raw/*fits'))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrbnum=0

       try:
           hrsrrawpath = makerawdir(obsdate, 'hrdet')
           message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
           
           lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits')))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrsnum=lasthrbnum
Esempio n. 7
0
def saltcharge(
    obsdate,
    outfile,
    sdbhost="sdb.saao",
    sdbname="sdb",
    sdbuser="",
    password="",
    clobber=False,
    logfile="saltlog.log",
    verbose=True,
):
    """Calculate the charged time for proposals observed during a night

    """

    with logging(logfile, debug) as log:

        # check the outfiles
        if not saltio.checkfornone(outfile):
            outfile = None

        # check that the output file can be deleted
        if outfile:
            saltio.overwrite(outfile, clobber)

        # connect the database
        sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

        # get all the proposals observed during the night
        select_state = "distinct Proposal_Code"
        table_state = "FileData Join ProposalCode using (ProposalCode_Id)"
        logic_state = "FileName like '%" + obsdate + "%'"

        records = saltmysql.select(sdb, select_state, table_state, logic_state)
        pids = []
        pid_time = {}
        for r in records:
            pids.append(r[0])
            pid_time[r[0]] = 0
        if len(pids) == 0:
            message = "There are no proposal to charge time for %s" % obsdate
            log.message(message)
            return

        # get the nightinfo_id
        night_id = saltmysql.getnightinfoid(sdb, obsdate)
        print night_id

        # get a list of all the images taken during the night
        select_state = "FileName, Proposal_Code, Target_Name, ExposureTime, UTSTART, INSTRUME, OBSMODE, DETMODE, CCDTYPE, NExposures"
        table_state = "FileData Join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id)"
        logic_state = "FileName like '%" + obsdate + "%'"
        img_list = saltmysql.select(sdb, select_state, table_state, logic_state)

        # get all the blocks visited
        select_state = "Block_Id, Accepted, Proposal_Code"
        table_state = "Block join BlockVisit using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)"
        logic_state = "NightInfo_Id=%i" % night_id
        block_list = saltmysql.select(sdb, select_state, table_state, logic_state)
        print block_list

        # get the start and end of twilight
        nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
        nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
        print nightstart, nightend, (nightend - nightstart).seconds
        print

        # download the SOMMI log from the night
        try:
            sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
        except Exception, e:
            msg = "Unable to read in SOMMI log for %s" % obsdate
            raise SaltError(msg)

        # add to account for error in SOMMI log
        if int(obsdate) < 20111027:
            sommi_log = sommi_log.replace("Object name", "\nObject name")

        # parse the sommi log
        point_list = parseforpointings(sommi_log)

        # now find all blocks observed during a night
        for point in point_list:
            # proposal for that block
            pid = point[0].strip()

            # start time for that block
            starttime = point[1]

            # find the end time for that block
            endtime = findnexttime(point[1], point_list, nightend)

            # find the start time of the first object to be observed
            # for this block
            startimage = findfirstimage(pid, starttime, img_list)
            lastimage = findlastimage(pid, endtime, img_list)

            # check to see if the end time for the last file observed for
            # this block
            if startimage is not None:
                if startimage > endtime:
                    startimage = None

            # determine the acquisition time for the block
            if startimage is not None:
                acqdelta = (startimage - starttime).seconds
            else:
                acqdelta = -1

            # find the shutter open time
            shuttertime = calculate_shutteropen(img_list, starttime, endtime)

            # if the shutter and time of the last image is substantially different from the
            # end of the block, then take the last image as the end of the block
            # *TODO* Change to use expected block time
            st = starttime + datetime.timedelta(0, shuttertime + acqdelta)
            if (endtime - st).seconds > 900:
                if lastimage is None:
                    endtime = st
                elif (lastimage - st).seconds > 3600:
                    endtime = st
                else:
                    endtime = lastimage
            # account for the time for that block
            tblock = (endtime - starttime).seconds

            if acqdelta > -1:
                # find the associated block
                block_id, blockvisit_id, accepted = getblockid(sdb, night_id, pid, img_list, starttime, endtime)

                if accepted and pid.count("-3-"):
                    # charged time for that block
                    try:
                        pid_time[pid] += tblock
                    except KeyError:
                        pid_time[pid] = tblock

                    print block_id, blockvisit_id, pid, starttime, tblock, acqdelta, shuttertime, shuttertime / tblock, block_id, accepted
                # update the charge time
                slewdelta = 0
                scidelta = tblock - acqdelta
                update_cmd = "TotalSlewTime=%i, TotalAcquisitionTime=%i, TotalScienceTime=%i" % (
                    slewdelta,
                    acqdelta,
                    scidelta,
                )
                table_cmd = "BlockVisit"
                logic_cmd = "BlockVisit_Id=%i" % blockvisit_id
                saltmysql.update(sdb, update_cmd, table_cmd, logic_cmd)

        return

        # update the charged time information
        # TODO: Check to see if the block was approved
        ptime_tot = 0
        stime_tot = 0
        for k in pid_time:
            ptime = pid_time[k]
            if ptime > 0:
                obsratio = stime / ptime
            else:
                obsratio = 0
            print "%25s %5i %5i %3.2f" % (k, ptime, stime, obsratio)
            if k.count("-3-"):
                ptime_tot += ptime
                stime_tot += stime

        # print out the total night statistics
        tdelta = nightend - nightstart
        print tdelta.seconds, ptime_tot, stime_tot
Esempio n. 8
0
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
             ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname, 
             elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
             clobber, runstatus, logfile,verbose):

   # set up

   basedir=os.getcwd()
   propcode=pinames
   sender = emuser + '@salt.ac.za'
   recipient = sender
   emessage = ''
   emailfile = '../piemaillist/email.lis'

   # check the observation date is sensible
   if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
       emessage = 'Observation date does not look sensible - YYYYMMDD\n'
       raise SaltError(emessage)


   # stop if the obsdate temporary directory already exists

   obsdir='%s' % obsdate
   if os.path.exists(obsdir):
       emessage += 'The temporary working directory ' + os.getcwd() + '/'
       emessage += obsdate + ' already exists. '
       raise SaltError(emessage)

   # create a temporary working directory and move to it
   saltio.createdir(obsdir)
   saltio.changedir(obsdir)
   workpath = saltio.abspath('.')

   # test the logfile
   logfile = workpath+logfile
   logfile = saltio.logname(logfile)

   #note the starttime
   starttime = time.time()

   #start logging
   with logging(logfile,debug) as log:

       #connect to the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
       
       #get the nightinfo id   
       nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)

       #Get the list of proposal codes
       state_select='Proposal_Code'
       state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
       state_logic="current=1"
       records=saltmysql.select(sdb, state_select, state_tables, state_logic)
       propids=[k[0] for k in records]

       # Calculate the current date
       currentdate=salttime.currentobsdate()

       # are the arguments defined
       saltio.argdefined('obsdate',obsdate)

       # check email and ftp arguments are consistent
       if email and not ftp:
           message =  'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
           message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
           raise SaltError(message)

       # identify a potential list of keyword edits
       keyfile = '../newheadfiles/list_newhead_' + obsdate
       if not os.path.isfile(keyfile):
           message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
           log.message(message)


       # check directories for the raw RSS data
       rssrawpath = makerawdir(obsdate, 'rss')

       # check directories for the raw SALTICAM data
       scmrawpath = makerawdir(obsdate, 'scam')


       # check raw directories for the disk.file record and find last file number
       #check rss data
       lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
       #check scame data
       lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
       #check for HRS Data--not filedata yet, so cannot check
   
       if lastrssnum == 1 and lastscmnum == 1:
           message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
           emessage += '\n' + message + '\n'
           log.message(message)

       #copy the data to the working directory
       if lastrssnum > 1:
           message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(rssrawpath,'rss/raw')
       if lastscmnum > 1:
           message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           saltio.copydir(scmrawpath,'scam/raw')

       #copy and pre-process the HRS data
       try:
           hrsbrawpath = makerawdir(obsdate, 'hbdet')
           saltio.createdir('hrs')
           saltio.createdir('hrs/raw')
           message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)

           hrsrrawpath = makerawdir(obsdate, 'hrdet')
           message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
           log.message(message)
           salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
           
           lasthrsnum=len(glob.glob('hrs/raw/*fits'))
       except Exception,e:
           log.message('Could not copy HRS data because %s' % e)
           lasthrsnum=0

       if lastrssnum>1 or lastscmnum>1:
           message = 'Copy of data is complete'
           log.message(message)
       else:
           message = 'No data was taken on %s' % obsdate
           log.message(message)


       #process the data RSS data
       if lastrssnum>1:
           preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose)

       #process the  SCAM data
       if lastscmnum>1:
           preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose)

       #process the HRS data
       if lasthrsnum>1:
           preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose)
           preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose)
         


       #check that all data was given a proper proposal id
       #only do it for semesters after the start of science operations
       if int(obsdate)>=20110901:
           # Check to see that the PROPID keyword exists and if not add it
           message = '\nSALTPIPE -- Checking for PROPID keyword'
           log.message(message)

           #check rss data
           rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log)
           #check scam data
           scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log)
           #check hrsB data
           hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log)
           #check hrsB data
           hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log)

           if not rssstatus  or not scmstatus or not hrsbstatus or not hrsrstatus: 
               msg='The PROPIDs for these files needs to be updated and re-start the pipeline'
               raise SaltError("Invalid PROPID in images:"+msg)

       #process the RSS data
       rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate,  propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
      
       #advance process the data
       if rssrawnum > 0:
           advanceprocess('rss', obsdate,  propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose)

       #process the SCAM data
       scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode,  median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)

       #process the HRS data
       hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose)

       #upload the data to the database
       img_list=glob.glob(workpath+'scam/product/*bxgp*.fits')
       img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits'))
       img_list.extend(glob.glob(workpath+'hrs/raw/*.fits'))
       if img_list:
           img=','.join('%s' %  (k) for k in img_list)
           saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
                  password=sdbpass, logfile=logfile, verbose=verbose)

       #add junk sources to the database
       raw_list=glob.glob(workpath+'scam/raw/S*.fits')
       raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits'))
       if raw_list:
          img=''
          for img in raw_list:
              hdu=pyfits.open(img)
              if hdu[0].header['PROPID'].strip()=='JUNK':
                saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
                    password=sdbpass, logfile=logfile, verbose=verbose)
              hdu.close()
       

       # construct observation and pipeline documentation
       if lastrssnum > 1 and rssrawnum>0:
           rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits'
       else:
           rssobslog = 'none'

       if lastscmnum > 1 and scmrawnum>0:
           scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits'
       else:
           scmobslog = 'None'

       if lasthrsnum > 1 and hrsrawnum>0:
           hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits'
       else:
           hrsobslog = 'None'



       if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0:
           msg='No data  processed for  %s' % obsdate
           email=False
           ftp=False
           log.message(msg)

       htmlpath = '.'
       nightlog = '../nightlogfiles/' + obsdate + '.log'
       readme = iraf.osfn('pipetools$html/readme.template')
       if not os.path.isfile(nightlog):
           nightlog = ''
           message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found'
           log.warning(message)

       if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
           salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath,
                      nightlog=nightlog,readme=readme,clobber=True,logfile=logfile,
                      verbose=verbose)

       #add a pause to allow syncing of the databases
       time.sleep(10)

       #Add in the environmental information
       if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
           propids=saltmysql.getpropcodes(sdb, obsdate)
           for pid in propids:
               saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass,
                           sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose) 
               try:
                  outfile='%s_%s_elsdata.fits' % (pid, obsdate)
                  outdir='%s/doc/' % (pid)
                  shutil.move(outfile, outdir)
               except:
                  os.remove(outfile)


       #ftp the data
       beachdir='/salt/ftparea/'
       if ftp:
           try:
               saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
                   password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
                   sdbname=sdbname,sdbuser=sdbuser,splitfiles=False, 
                   cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
           except Exception,e:
               message="Not able to copy data to FTP area:\n%s " % e
               raise SaltError(message)     
           #run with the splitting of files
           try: 
               saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
                   password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
                   sdbname=sdbname,sdbuser=sdbuser,splitfiles=True, 
                   cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
           except Exception,e:
               message="Not able to copy data to FTP area:\n%s " % e
               raise SaltError(message)     
Esempio n. 9
0
'''
    where_term = "Proposal_Code like '%s' and Date='%s-%s-%s'" % (
        propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
    print select_term, from_term, where_term
    try:
        record = saltmysql.select(sdb, select_term, from_term,
                                  where_term)[0][0]
    except:
        record = None
    print record

    if record == 'FastEmail':
        return
    else:
        #insert information into the database
        nightinfoid = saltmysql.getnightinfoid(sdb, obsdate)
        insert_term = "NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (
            nightinfoid, propid)
        table_term = "PipelineProposalStatistics"
        saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

        #send email
        sender = '*****@*****.**'
        recipient = email
        bcc = '*****@*****.**'
        subject = 'SALT data available for %s' % propcode
        message = open(readmefile).read()
        message = message.replace('OBSDATE', obsdate)
        sendemail(server, 'sa', password, sender, recipient, bcc, subject,
                  message)
Esempio n. 10
0
def makenightstats(els, sdb, outfile, obsdate, clobber=False):
   """Retrieve data for a given observation during an observation date for 
      a proposal
   """
   fout=saltio.openascii(outfile, 'w')
 

   headerstr="""<html>
<head><title>SALT Night Report for %s</title></head>
<body bgcolor="white" text="black" link="blue" vlink="blue">

<center><h1>
SALT Night Report for %s<br>
</h1></center>
"""  % (obsdate, obsdate)
   fout.write(headerstr)


   #set up the Observing Statistics
   fout.write('<h2> A. Observing Information </h2>\n')
   nid=saltmysql.getnightinfoid(sdb, obsdate)
   print nid
   #get duty information
   selcmd='sa.Surname,  so.Surname,  ct.surname' 
   tabcmd='NightInfo join SaltOperator as so on (SO1_Id=SO_Id) join Investigator as sa on (SA_Id=sa.Investigator_Id) join Investigator as ct on (CTDuty_Id=ct.Investigator_Id)'
   record=saltmysql.select(sdb, selcmd, tabcmd, 'NightInfo_Id=%i' % nid)
   try:
      sa,so,ct=record[0]
      dutystr='SA: %s <br>\nSO: %s <br> \nCT: %s <br>\n<br>' % (sa, so,ct)
   except:
      dutystr='SA: %s <br>\n SO: %s <br> \n CT: %s <br>\n<br>' % ('', '', '')
   fout.write(dutystr)

   #get night time information
   selcmd='SunSet, SunRise, MoonSet, MoonRise, MoonPhase_Percent, EveningTwilightEnd, MorningTwilightStart' 
   record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
   statlist=['Sun Set', 'Sun Rise', 'Moon Set', 'Moon Rise', 'Moon Phase', 'Evening Twilight', 'Morning Twilight'] 

   statstr='<table border=1><tr><th colspan=2>Nighttime Statistics </th></tr>\n'
   for s,r in zip (statlist, record):
       statstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
   statstr+='</table><br>\n'
   fout.write(statstr)
   mintime=record[0]-datetime.timedelta(seconds=1*3600)
   maxtime=record[1]+datetime.timedelta(seconds=1*3600)
   
   obsstatlist=['Science Time', 'Engineering Time', 'Lost to Weather', 'Lost to Problems']
   selcmd='ScienceTime, EngineeringTime, TimeLostToWeather, TimeLostToProblems'
   record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
   obsstatstr='<table border=1><tr><th colspan=2>Observing Statistics </th></tr>\n'
   for s,r in zip (obsstatlist, record):
       obsstatstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
   obsstatstr+='</table><br>\n'
   fout.write(obsstatstr)

   #Set up the Envirnmental Statistics
   fout.write('<h2> B. Environmental Statistics </h2>\n')

   #create the tables
   print mintime, maxtime
   #guihdu=guidertable(els, mintime, maxtime)

   #temperature plot
   fout.write('<table>')
   weahdu=weathertable(els, mintime, maxtime)
   tempplot=temperatureplot(weahdu, mintime, maxtime, obsdate)
   fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(tempplot))
   windfile=windplot(weahdu, mintime, maxtime, obsdate)
   fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(windfile))

   #seeing plot
   seeplot=makeseeingplot(sdb, mintime, maxtime, obsdate)
   fout.write('<tr><td><img width=700 height=200 src=%s><br></td></tr>\n' % os.path.basename(seeplot))

   fout.write('</table>\n')


   #Set up the Pipeline Statistics
   fout.write('<h2> C. Pipeline Statistics </h2>\n')

   fout.write('<table>\n')
   record=saltmysql.select(sdb, 'PipelineStatus, RawSize, ReducedSize, PipelineRunTime', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % nid)[0]
   print record
   print record[1]/3600.0
   pipelinestatus='Completed'
   fout.write('<tr><td>Pipeline Status:</td><td>%s</td></tr>' % record[0])
   fout.write('<tr><td>Raw Data:</td><td>%3.2f Gb</td></tr>' % (record[1]/1e9))
   fout.write('<tr><td>Reduced Data:</td><td>%3.2f Gb</td></tr>' % (record[2]/1e9))
   fout.write('<tr><td>Run Time:</td><td>%3.2f min</td></tr>' % (record[3]/60.0))
   fout.write('</table>\n')

   #Set up the Proposal/Block Statistics
   fout.write('<h2> D. Proposal Statistics </h2>\n')
   selcmd='Proposal_Code, Block_Name, Accepted, ObsTime, BlockRejectedReason_Id, BlockVisit_Id' 
   tabcmd='Block join BlockVisit as bv using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)'
   logcmd='NightInfo_Id=%i' % nid
   record=saltmysql.select(sdb, selcmd, tabcmd, logcmd)
   print record
   fout.write('<table border=1>\n')
   fout.write('<tr><th>Proposal</th><th>Block</th><th>Obs Time</th><th>Accepted?</th><th>Rejected Reason</th></tr>\n')
   for r in record:
       if r[2]>=1: 
          accept='Yes'
          reason=''
       else:
          accept='No'
          print r
          reason=saltmysql.select(sdb, 'RejectedReason', 'BlockRejectedReason', 'BlockRejectedReason_Id=%i' % int(r[4]))[0][0]
       bstr='<tr><td><a href="https://www.salt.ac.za/wm/proposal/%s/">%s</a></td><td>%s</td><td>%3.2f</td><td>%s</td><td>%s</td></tr>\n' % (r[0], r[0], r[1], float(r[3])/3600.0, accept, reason)
       fout.write(bstr)
   fout.write('</table>\n')

   #Set up the Data Quality Statistics
   fout.write('<h2> E. Data Quality </h2>\n')


   fout.write('</body> \n</hmtl>')
   fout.close()
Esempio n. 11
0
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
   where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
   print select_term, from_term, where_term
   try:
     record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
   except:
     record=None
   print record

   if record=='FastEmail':
      return
   else:
      #insert information into the database
      nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
      insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
      table_term="PipelineProposalStatistics"
      saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

      #send email
      sender='*****@*****.**'
      recipient=email
      bcc='*****@*****.**'
      subject='SALT data available for %s' % propcode
      message=open(readmefile).read()
      message=message.replace('OBSDATE', obsdate)
      sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
      
   
Esempio n. 12
0
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password):
   """Handle fast data delivery for the proposal.  For a given filename
   """
   
   if propcode is None or propcode=='None': return

   #first check in the sdb if fast data delivery is needed
   sdb=saltmysql.connectdb(sdbhost,sdbname, sdbuser, password)
   select_term='Distinct Surname, email, username, ProposalCode_Id'
   from_term='''
Block join Pointing using (Block_Id) 
      join PipelineConfig using (Pointing_Id)
      join Proposal using (Proposal_Id)
      join ProposalCode using (ProposalCode_Id)
      join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id)
      join ProposalContact using (Proposal_Id)
      join Investigator on (Investigator_Id=Contact_Id)
      join PiptUser using (PiptUser_Id)
   '''
   where_term="Proposal_Code like '%s' and current=1 and DataAccessMethod='Fast'" \
              % (propcode)
   #print 'Select %s from %s where %s' % (select_term, from_term, where_term)
   try:
       record=saltmysql.select(sdb, select_term, from_term, where_term)
   except Exception as e:
       print(e)
       return None
   #print "Checking for fast data"
   #print record
 
   if record:
      surname, email, username, propid= record[0]
      #print surname, email, username, propid
   else:
      return

   #second if so, then copy the data to the contact PI directory
   #on saltpipe under the fast directory. 
   #rawfilename=getrawfilename(filename)
   y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
   if y==256:
      y=os.system('ssh sa@saltpipe mkdir /salt/ftparea/%s/fast%s' % (username, obsdate))
      y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
  
   if y!=0:
      print("Problem with copying file %s to /salt/ftparea/%s/fast%s/"  % (filename,  username, obsdate))

   #copy the reduced data
   y=os.system('scp mbxp%s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (os.path.basename(filename), username, obsdate))
   #check the type of data it is and copy over an ancillery data as well

   #if it is the first object file, check to see if an email has been
   #sent, and if not, send email
   #try to copy the spectroscopic data
   print(filename, filename.startswith('P'))
   if os.path.basename(filename).startswith('P'):
       sfilename='smbxp%s.txt' % (os.path.basename(filename).split('.fits')[0])
       print(sfilename)
       try:
           y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
       except Exception as e:
           print(e)
   if os.path.basename(filename).startswith('S'):
       try:
           sfilename='mbxp%s.cat' % (os.path.basename(filename).split('.fits')[0])
           print(sfilename)
           y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
       except Exception as e:
           print(e)
 

   #check to see if an email has been sent
   select_term='PipelineStatus'
   from_term='''
PipelineProposalStatistics 
join PipelineStatus using (PipelineStatus_Id)
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
   where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
   print(select_term, from_term, where_term)
   try:
     record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
   except:
     record=None
   print(record)

   if record=='FastEmail':
      return
   else:
      #insert information into the database
      nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
      insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
      table_term="PipelineProposalStatistics"
      saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

      #send email
      sender='*****@*****.**'
      recipient=email
      bcc='*****@*****.**'
      subject='SALT data available for %s' % propcode
      message=open(readmefile).read()
      message=message.replace('OBSDATE', obsdate)
      sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
      
   

   sdb.close()
   return
Esempio n. 13
0
def saltnightinfo(
    obsdate,
    sdbhost="sdb.saao",
    sdbname="sdb",
    sdbuser="",
    password="",
    clobber=False,
    logfile="saltlog.log",
    verbose=True,
):
    """Update the nightinfo table from the SOMMI log

    """

    with logging(logfile, debug) as log:

        # connect the database
        sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

        # get the nightinfo_id
        night_id = saltmysql.getnightinfoid(sdb, obsdate)
        print night_id

        # get the start and end of twilight
        nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
        nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
        print nightstart
        print nightend
        print (nightend - nightstart).seconds

        # download the SOMMI log from the night
        try:
            sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
        except:
            raise SaltError("Unable to read SOMMI log in the database for %s" % obsdate)

        # set up the time for the night

        try:
            ntime = (nightend - nightstart).seconds
        except:
            raise SaltError("Unable to read night length from database for %s" % obsdate)

        # parse the sommi log
        slog = sommi_log.split("\n")
        stime = 0
        for i in range(len(slog)):
            if slog[i].count("Science Time:"):
                stime = extractinformation(slog[i])
            if slog[i].count("Engineering Time:") and not slog[i].count("Non-observing Engineering Time"):
                etime = extractinformation(slog[i])
            if slog[i].count("Time lost to Weather:"):
                wtime = extractinformation(slog[i])
            if slog[i].count("Time lost to Tech. Problems:"):
                ttime = extractinformation(slog[i])
            if slog[i].count("Non-observing Engineering Time:"):
                ltime = extractinformation(slog[i])
        print etime
        tot_time = stime + etime + wtime + ttime
        print night_id, ntime, stime, etime, wtime, ttime, ltime, tot_time

        # insert the information into the database
        print tot_time, ntime
        if abs(tot_time - ntime) < 900:
            message = "Updating NightInfo Table with the following Times:\n"
            message += "Science Time=%i\n" % stime
            message += "Engineeringh=%i\n" % etime
            message += "Time lost to Weather=%i\n" % wtime
            message += "Time lost to Tech. Problems=%i\n" % ttime
            message += "Non-observing Engineering Time=%i\n" % ltime
            log.message(message)

            insert_state = (
                "ScienceTime=%i, EngineeringTime=%i, TimeLostToWeather=%i, TimeLostToProblems=%i, NonObservingEngineeringTime=%i"
                % (stime, etime, wtime, ttime, ltime)
            )
            table_state = "NightInfo"
            logic_state = "NightInfo_Id=%i" % night_id
            saltmysql.update(sdb, insert_state, table_state, logic_state)
        else:
            message = "The total time for the night is not equal to the length of the night\n"
            message += "Night Length=%i\n--------------------\n" % ntime
            message += "Science Time=%i\n" % stime
            message += "Engineeringh=%i\n" % etime
            message += "Time lost to Weather=%i\n" % wtime
            message += "Time lost to Tech. Problems=%i\n" % ttime
            message += "Non-observing Engineering Time=%i\n" % ltime
            message += "Total time for the Night=%i\n" % tot_time
            log.message(message)
Esempio n. 14
0
def saltslewstats(startdate, enddate, elshost, elsname, elsuser, elspass, 
            sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):

   # set up

   proposers = []
   propids = []
   pids = []
   
   with logging(logfile,debug) as log:

       #open the database
       els=saltmysql.connectdb(elshost, elsname, elsuser, elspass)
       sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)  

       #loop through each night and determine the statistics for the slew times
       #for each night
       obsdate=int(startdate)
       slew_list=[]
       while obsdate < int(enddate):
           night_id=saltmysql.getnightinfoid(sdb, obsdate)

           start,end=saltmysql.select(sdb,'EveningTwilightEnd,MorningTwilightStart','NightInfo', 'NightInfo_Id=%i' % night_id)[0]
           
           tslew, nslew=slewtime(els, start, end)
           if nslew>0: 
              print obsdate, night_id, start, end, nslew, tslew/nslew
              slew_list.append([start, nslew, tslew])
           obsdate=int(getnextdate(obsdate))

       slew_arr=np.array(slew_list)
 
       days=np.zeros(len(slew_arr))
       for i in range(len(slew_arr)):
           days[i]=(slew_arr[i,0]-slew_arr[0,0]).days
       coef=np.polyfit(days, slew_arr[:,2]/slew_arr[:,1], 2)
       
       ave_date=[]
       ave_values=[]
       ave_nslews=[]
       nstep=10
       for i in np.arange(15,len(slew_arr), 2*nstep):
           ave_date.append(slew_arr[i,0])
           i1=i-nstep
           i2=min(i+nstep, len(slew_arr))
           #ave_values.append(np.median(slew_arr[i1:i2,2]/slew_arr[i1:i2,1]))
           ave_nslews.append(np.median(slew_arr[i1:i2,1]))
           ave_values.append(np.median(slew_arr[i1:i2,2])) 
           print ave_date[-1], ave_values[-1], ave_nslews[-1]

       ave_values=np.array(ave_values)
       ave_nslews=np.array(ave_nslews)
     
       mean_slew=ave_nslews.mean()
       mean_slew=11
       for i in range(len(ave_date)):
             #value is an attempt to correct for the average number of slews
             value=(ave_values[i]+30*(mean_slew-ave_nslews[i]))/mean_slew
             print ave_date[i], '%i %i %i' % (ave_values[i]/ave_nslews[i], ave_nslews[i], value)
 
       plt.figure()
       plt.plot(slew_arr[:,0], slew_arr[:,2]/slew_arr[:,1], ls='', marker='o')
       #plt.plot(slew_arr[:,0], slew_arr[:,1], ls='', marker='o')
       #plt.plot(slew_arr[:,0], slew_arr[:,2], ls='', marker='o')
       
       plt.plot(ave_date, ave_values/ave_nslews)
       #plt.plot(days, np.polyval(coef, days))
       plt.ylabel('Slew Time(s)')
       plt.xlabel('Date')
       plt.show()