Ejemplo n.º 1
0
def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
    """Load logstr into the SDB"""
    print logstr
    #connect to the db
    sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    #get the nightinfoid
    #date='YYYY-MM-DD' <- note speechmarks
    #data= your table
    #Select NightInfo_Id from NightInfo where Date=$date;
    #(get nightinfo_Id)
    #Insert into ObsLogTable values ($nightinfo_Id, $data);
    logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
    results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
    night_id = results[0][0]

    #check to see if it exists
    logic = 'NightInfo_Id=%i' % night_id
    results = saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
    if results:
        update_cmd = "ObsLogTable='%s'" % (logstr)
        saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
    else:
        insert_cmd = "NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
        saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')

    print obsdate, night_id
Ejemplo n.º 2
0
def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
    """Load logstr into the SDB"""
    print logstr
    #connect to the db
    sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    #get the nightinfoid
    #date='YYYY-MM-DD' <- note speechmarks
    #data= your table
    #Select NightInfo_Id from NightInfo where Date=$date;
    #(get nightinfo_Id)
    #Insert into ObsLogTable values ($nightinfo_Id, $data);
    logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
    results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
    night_id=results[0][0]

    #check to see if it exists
    logic='NightInfo_Id=%i' % night_id
    results=saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
    if results: 
        update_cmd="ObsLogTable='%s'" % (logstr)
        saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
    else:
        insert_cmd="NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
        saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')
    
    print obsdate, night_id
Ejemplo n.º 3
0
def updatedq(img, struct, sdb):
   """Add information about the image to the database
   """
 
   #get the filenumber
   #check to see if the FileData was created
   logic="FileName='%s'" % img
   records=saltmysql.select(sdb,'FileData_Id','FileData',logic)
   try:
       FileData_Id=records[0][0]
   except:
       message='WARNING:  File not yet in database'
       print message
       return


   #get the information from the image
   for i in range(1,len(struct)):
     if struct[i].name=='SCI':
       try:
           omean=struct[i].header['OVERSCAN']
       except:
           omean=None
       try:
           orms=struct[i].header['OVERRMS']
       except:
           orms=None
       #lets measureme the statistics in a 200x200 box in each image
       try:
          my,mx=struct[i].data.shape
          dx1=int(mx*0.5)
          dx2=min(mx,dx1+200)
          dy1=int(my*0.5)
          dy2=min(my,dy1+200)
          mean,med,sig=saltstat.iterstat(struct[i].data[dy1:dy2,dx1:dx2], 5, 5)
       except:
          mean, med, sig=(None, None, None)

       #update the database with this information
       #check to see if you need to update or insert
       record=saltmysql.select(sdb, 'FileData_Id', 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
       update=False
       if record: update=True
 
       ins_cmd=''
       if omean is not None: ins_cmd='OverscanMean=%s,' % omean
       if orms is not None:  ins_cmd+='OverscanRms=%s,' % orms
       if mean is not None:  ins_cmd+='BkgdMean=%f,' % mean
       if sig is not None:   ins_cmd+='BkgdRms=%f' % sig 
       if update:
          ins_cmd=ins_cmd.rstrip(',')    
          saltmysql.update(sdb, ins_cmd, 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
       else:
          ins_cmd+=',FileData_Id=%i, Extension=%i' % (FileData_Id, i)
          saltmysql.insert(sdb, ins_cmd, 'PipelineDataQuality_CCD')
Ejemplo n.º 4
0
def saltweather(weathertime, timespan, elshost, elsname, elsuser, elspass, sdbhost, sdbname, sdbuser, password):
    print weathertime

    # open the database
    els = saltmysql.connectdb(elshost, elsname, elsuser, elspass)
    sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    # determine the obsdate
    obsdate = weathertime - datetime.timedelta(seconds=43200)
    obsdate = "%i%s%s" % (obsdate.year, string.zfill(obsdate.month, 2), string.zfill(obsdate.day, 2))
    nid = saltmysql.getnightinfoid(sdb, obsdate)

    print nid
    # get the most recent weather data
    airpressure, dewout, extrelhum, temp2m, temp30m, windspeed, winddir, rain = getweatherdata(
        els, weathertime, timespan
    )
    dewin, relhum = getinsidedata(els, weathertime, timespan)
    tempin = getacdata(els, weathertime, timespan)

    # upload that to the sdb
    upcmd = "Weather_Time='%s', NightInfo_Id=%i" % (weathertime, nid)

    if tempin is not None:
        upcmd += ",TemperatureInside=%4.2f" % tempin
    if temp2m is not None:
        upcmd += ",Temperature2m=%4.2f" % temp2m
    if temp30m is not None:
        upcmd += ",Temperature30m=%4.2f" % temp30m
    if windspeed is not None:
        upcmd += ",WindSpeed=%5.2f" % windspeed
    if winddir is not None:
        upcmd += ",WindDirection=%5.2f" % winddir
    if dewin is not None:
        upcmd += ",DewPointInside=%3.2f" % dewin
    if dewout is not None:
        upcmd += ",DewPointOutside=%3.2f" % dewout
    if airpressure is not None:
        upcmd += ",AirPressure=%4.2f" % airpressure
    if relhum is not None:
        upcmd += ",RelativeHumidty=%4.2f" % relhum
    if extrelhum is not None:
        upcmd += ",ExternalRelativeHumidity=%4.2f" % extrelhum
    if rain is not None:
        upcmd += ",Rain=%i" % rain

    saltmysql.insert(sdb, upcmd, "Weather")
    print upcmd

    return
Ejemplo n.º 5
0
def pipelinestatus(obsdate, status, message=None, rawsize=None, reducedsize=None, 
              runtime=None, emailsent=None, sdbhost='sdb.saao', sdbname='sdb',  sdbuser='', password='', 
              logfile='saltlog.log', verbose=True):
    """Update the PipelineStatistics table with the current information
       about the pipeline

    """

    with logging(logfile,debug) as log:

       #connect the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #get the nightinfo_id
       night_id=saltmysql.getnightinfoid(sdb, obsdate)

       #get the status_id for the given status
       status_id=getstatusid(sdb, status)
           
       #create the insert command
       obsdate=str(obsdate)
       inst_cmd="NightInfo_Id=%s, PipelineStatus_Id=%i" % (night_id, status_id)
       if status_id>10:
          inst_cmd+=',ErrorMessage="%s"' % message
       if rawsize is not None: 
          inst_cmd+=",RawSize=%f" % rawsize
       if reducedsize is not None: 
          inst_cmd+=",ReducedSize=%f" % rawsize
       if runtime is not None:
          inst_cmd+=",PipelineRunTime=%i" % runtime
       if emailsent is not None:
          inst_cmd+=",EmailSent=%i" % emailsent 
       print inst_cmd
     
       #insert or update the pipeline
       if checktable(sdb, night_id): 
          saltmysql.update(sdb, inst_cmd, 'PipelineStatistics', 'NightInfo_Id=%i' % night_id)
          msg="Updating information for Night_Id=%i\n" % night_id
       else:
          saltmysql.insert(sdb, inst_cmd, 'PipelineStatistics')
          msg="Inserting  information for Night_Id=%i\n" % night_id
          

       #log the call
       log.message(msg+inst_cmd, with_stdout=verbose)
Ejemplo n.º 6
0
def updatenightinfo(obsdate, sdbhost='sdb.saao', sdbname='sdb', \
              sdbuser='', password='', logfile='saltlog.log', verbose=True):
    """Update the nightinfo table with current information about the
       night

    """

    with logging(logfile,debug) as log:

       #connect the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #get the nightinfo_id
       try:
           night_id=saltmysql.getnightinfoid(sdb, obsdate)
       except SaltError:
           night_id=None
           
       #get the information for the night
       time_list=get_nightdetails(obsdate)

       #create the insert command
       obsdate=str(obsdate)
       inst_cmd="Date='%s-%s-%s'," % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
       inst_cmd+="SunSet='%s', SunRise='%s', MoonSet='%s', MoonRise='%s', EveningTwilightEnd='%s', MorningTwilightStart='%s'" % \
               (time_list[0], time_list[1], time_list[2], time_list[3], time_list[4], time_list[5])
       inst_cmd+=",MoonPhase_Percent=%i" % (round(float(time_list[6])))

       if night_id: 
          saltmysql.update(sdb, inst_cmd, 'NightInfo', 'NightInfo_Id=%i' % night_id)
          msg="Updating information for Night_Id=%i\n" % night_id
       else:
          saltmysql.insert(sdb, inst_cmd, 'NightInfo')
          msg="Inserting  information for Night_Id=%i\n" % night_id
          

       #log the call
       log.message(msg+inst_cmd, with_stdout=verbose)
Ejemplo n.º 7
0
def findcal(obsdate, sdbhost, sdbname, sdbuser, password):
    """Find all of the unique configuration settings used on a night 
       and insert it into the database
  
       obsdate--Observation date in YYYYMMDD

       sdbhost--host name for sdb

       sdbname--name of the sdb

       sdbuser--user for the sdb

       password--sdb password
    """

    #connect to the db
    sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    #get the nightinfo id
    logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
    results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
    night_id=results[0][0]

    #select all the scam data from this obsdate
    cmd_select='d.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER'
    cmd_table=''' FileData  as d
  left join FitsHeaderImage as f using (FileData_Id) 
  left join FitsHeaderSalticam using (FileData_Id)
'''
    cmd_logic="d.FileName like 'S" + obsdate+"%' and f.CCDTYPE='OBJECT'"

    results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)

    #loop through all the results and return only the Set of identical results
    caldict=create_caldict(results)


    #insert the results into the database
    for k in caldict:
        #first check to see if it has already been entered
        record=saltmysql.select(sdb, 'FileData_Id', 'SalticamNightlyCalibration', 'FileData_Id=%i' % k)
        if len(record)<1:
           #check for block_id
           blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0]

           #get the calibration types requested
           if blockid:
               request=saltmysql.select(sdb, 'SalticamCalibrationType_Id', 'SalitcamCalibration', 'Block_Id=%i' % blockid)
               for cid in request:
                   cid=cid[0]
                   cmd_insert='NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % (night_id, k, cid)
                   #saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration')
               print(k, " ".join([str(k) for k in caldict[k]]))

    #list of rss calibration types
    #+-----------------------+----------------------------------+
    #| RssCalibrationType_Id | CalibrationType                  |
    #+-----------------------+----------------------------------+
    #|                     2 | Arc                              |
    #|                     1 | Bias                             |
    #|                    14 | Imaging flat - Lamp              |
    #|                    15 | Imaging flat - Twilight          |
    #|                     3 | Spectroscopic flat - Lamp        |
    #|                     4 | Spectroscopic flat - Twilight    |
    #|                    12 | Standard - Circular polarimetric |
    #|                    13 | Standard - Lick                  |
    #|                     9 | Standard - Linear polarimetric   |
    #|                     5 | Standard - Photometric           |
    #|                     8 | Standard - RV                    |
    #|                    11 | Standard - Smooth spectrum       |
    #|                     7 | Standard - Spectrophotometric    |
    #|                     6 | Standard - Spectroscopic         |
    #|                    10 | Standard - Unpolarised           |
    #+-----------------------+----------------------------------+

    #select all the RSS data from this obsdate
    rssheaderlist='f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
    cmd_select='d.FileName,d.FileData_Id, %s' % rssheaderlist
    # CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
    cmd_table=''' FileData as d
  left join FitsHeaderImage as f using (FileData_Id) 
  left join FitsHeaderRss using (FileData_Id)
  join ProposalCode using (ProposalCode_Id)
'''
    cmd_logic="d.FileName like 'P" + obsdate+"%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST'"

    results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)

    #loop through all the results and return only the Set of identical results
    caldict=create_caldict(results)

    #insert the rss results into the database
    for k in caldict:
       #first check to see if it has already been entered
       record=saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration', 'FileData_Id=%i' % k) 
       if len(record)<1:
           #period for checking for SPST.  If the uses requests this, it gets set to 
           # 7 days, but the default is taken within the last month for non-requests
           period=30

           #check for block_id
           blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0]

           #get the calibration types requested
           if blockid:
               request=saltmysql.select(sdb, 'RssCalibrationType_Id', 'RssCalibration', 'Block_Id=%i' % blockid)
               for cid in request:
                   cid=cid[0]
                   #check to see if the request is already in the database or if a similar request has
                   #been taken recently
                   if cid==1:  #bias
                       if not checkforbias(sdb, k):
                          cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
                          saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
                   elif cid in [3,4]: #flats
                       if not checkforflats(sdb, k, cid, rssheaderlist, instr='rss', keylist=caldict[k], period=90):
                          
                          cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
                          saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
                       
                   elif cid==7: #specstand
                       period=7
                       #print period, k, caldict[k]
                       if not checkforspst(sdb, k, caldict[k], rssheaderlist, period=period):
                          cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % (night_id, k)
                          saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
                   else:
                       cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
                       saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
                     

                   print(k, cid," ".join([str(w) for w in caldict[k]]))

    return
Ejemplo n.º 8
0
    try:
        record = saltmysql.select(sdb, select_term, from_term,
                                  where_term)[0][0]
    except:
        record = None
    print record

    if record == 'FastEmail':
        return
    else:
        #insert information into the database
        nightinfoid = saltmysql.getnightinfoid(sdb, obsdate)
        insert_term = "NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (
            nightinfoid, propid)
        table_term = "PipelineProposalStatistics"
        saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

        #send email
        sender = '*****@*****.**'
        recipient = email
        bcc = '*****@*****.**'
        subject = 'SALT data available for %s' % propcode
        message = open(readmefile).read()
        message = message.replace('OBSDATE', obsdate)
        sendemail(server, 'sa', password, sender, recipient, bcc, subject,
                  message)

    sdb.close()
    return

Ejemplo n.º 9
0
   where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
   print select_term, from_term, where_term
   try:
     record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
   except:
     record=None
   print record

   if record=='FastEmail':
      return
   else:
      #insert information into the database
      nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
      insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
      table_term="PipelineProposalStatistics"
      saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

      #send email
      sender='*****@*****.**'
      recipient=email
      bcc='*****@*****.**'
      subject='SALT data available for %s' % propcode
      message=open(readmefile).read()
      message=message.replace('OBSDATE', obsdate)
      sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
      
   

   sdb.close()
   return
Ejemplo n.º 10
0
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password):
   """Handle fast data delivery for the proposal.  For a given filename
   """
   
   if propcode is None or propcode=='None': return

   #first check in the sdb if fast data delivery is needed
   sdb=saltmysql.connectdb(sdbhost,sdbname, sdbuser, password)
   select_term='Distinct Surname, email, username, ProposalCode_Id'
   from_term='''
Block join Pointing using (Block_Id) 
      join PipelineConfig using (Pointing_Id)
      join Proposal using (Proposal_Id)
      join ProposalCode using (ProposalCode_Id)
      join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id)
      join ProposalContact using (Proposal_Id)
      join Investigator on (Investigator_Id=Contact_Id)
      join PiptUser using (PiptUser_Id)
   '''
   where_term="Proposal_Code like '%s' and current=1 and DataAccessMethod='Fast'" \
              % (propcode)
   #print 'Select %s from %s where %s' % (select_term, from_term, where_term)
   try:
       record=saltmysql.select(sdb, select_term, from_term, where_term)
   except Exception as e:
       print(e)
       return None
   #print "Checking for fast data"
   #print record
 
   if record:
      surname, email, username, propid= record[0]
      #print surname, email, username, propid
   else:
      return

   #second if so, then copy the data to the contact PI directory
   #on saltpipe under the fast directory. 
   #rawfilename=getrawfilename(filename)
   y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
   if y==256:
      y=os.system('ssh sa@saltpipe mkdir /salt/ftparea/%s/fast%s' % (username, obsdate))
      y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
  
   if y!=0:
      print("Problem with copying file %s to /salt/ftparea/%s/fast%s/"  % (filename,  username, obsdate))

   #copy the reduced data
   y=os.system('scp mbxp%s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (os.path.basename(filename), username, obsdate))
   #check the type of data it is and copy over an ancillery data as well

   #if it is the first object file, check to see if an email has been
   #sent, and if not, send email
   #try to copy the spectroscopic data
   print(filename, filename.startswith('P'))
   if os.path.basename(filename).startswith('P'):
       sfilename='smbxp%s.txt' % (os.path.basename(filename).split('.fits')[0])
       print(sfilename)
       try:
           y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
       except Exception as e:
           print(e)
   if os.path.basename(filename).startswith('S'):
       try:
           sfilename='mbxp%s.cat' % (os.path.basename(filename).split('.fits')[0])
           print(sfilename)
           y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
       except Exception as e:
           print(e)
 

   #check to see if an email has been sent
   select_term='PipelineStatus'
   from_term='''
PipelineProposalStatistics 
join PipelineStatus using (PipelineStatus_Id)
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
   where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
   print(select_term, from_term, where_term)
   try:
     record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
   except:
     record=None
   print(record)

   if record=='FastEmail':
      return
   else:
      #insert information into the database
      nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
      insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
      table_term="PipelineProposalStatistics"
      saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")

      #send email
      sender='*****@*****.**'
      recipient=email
      bcc='*****@*****.**'
      subject='SALT data available for %s' % propcode
      message=open(readmefile).read()
      message=message.replace('OBSDATE', obsdate)
      sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
      
   

   sdb.close()
   return
Ejemplo n.º 11
0
def findcal(obsdate, sdbhost, sdbname, sdbuser, password):
    """Find all of the unique configuration settings used on a night 
       and insert it into the database
  
       obsdate--Observation date in YYYYMMDD

       sdbhost--host name for sdb

       sdbname--name of the sdb

       sdbuser--user for the sdb

       password--sdb password
    """

    #connect to the db
    sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

    #get the nightinfo id
    logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
    results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
    night_id = results[0][0]

    #select all the scam data from this obsdate
    cmd_select = 'd.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER'
    cmd_table = ''' FileData  as d
  left join FitsHeaderImage as f using (FileData_Id) 
  left join FitsHeaderSalticam using (FileData_Id)
'''
    cmd_logic = "d.FileName like 'S" + obsdate + "%' and f.CCDTYPE='OBJECT'"

    results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)

    #loop through all the results and return only the Set of identical results
    caldict = create_caldict(results)

    #insert the results into the database
    for k in caldict:
        #first check to see if it has already been entered
        record = saltmysql.select(sdb, 'FileData_Id',
                                  'SalticamNightlyCalibration',
                                  'FileData_Id=%i' % k)
        if len(record) < 1:
            #check for block_id
            blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
                                       'FileData_Id=%i' % k)[0][0]

            #get the calibration types requested
            if blockid:
                request = saltmysql.select(sdb, 'SalticamCalibrationType_Id',
                                           'SalitcamCalibration',
                                           'Block_Id=%i' % blockid)
                for cid in request:
                    cid = cid[0]
                    cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % (
                        night_id, k, cid)
                    #saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration')
                print k, " ".join([str(k) for k in caldict[k]])

    #list of rss calibration types
    #+-----------------------+----------------------------------+
    #| RssCalibrationType_Id | CalibrationType                  |
    #+-----------------------+----------------------------------+
    #|                     2 | Arc                              |
    #|                     1 | Bias                             |
    #|                    14 | Imaging flat - Lamp              |
    #|                    15 | Imaging flat - Twilight          |
    #|                     3 | Spectroscopic flat - Lamp        |
    #|                     4 | Spectroscopic flat - Twilight    |
    #|                    12 | Standard - Circular polarimetric |
    #|                    13 | Standard - Lick                  |
    #|                     9 | Standard - Linear polarimetric   |
    #|                     5 | Standard - Photometric           |
    #|                     8 | Standard - RV                    |
    #|                    11 | Standard - Smooth spectrum       |
    #|                     7 | Standard - Spectrophotometric    |
    #|                     6 | Standard - Spectroscopic         |
    #|                    10 | Standard - Unpolarised           |
    #+-----------------------+----------------------------------+

    #select all the RSS data from this obsdate
    rssheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, AR_STA, MASKID'
    cmd_select = 'd.FileName,d.FileData_Id, %s' % rssheaderlist
    # CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
    cmd_table = ''' FileData as d
  left join FitsHeaderImage as f using (FileData_Id) 
  left join FitsHeaderRss using (FileData_Id)
  join ProposalCode using (ProposalCode_Id)
'''
    cmd_logic = "d.FileName like 'P" + obsdate + "%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST' and GRTILT > 0"

    results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)

    #loop through all the results and return only the Set of identical results
    caldict = create_caldict(results)

    #insert the rss results into the database
    for k in caldict:
        #first check to see if it has already been entered
        record = saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration',
                                  'FileData_Id=%i' % k)

        if len(record) < 1:
            #period for checking for SPST.  If the uses requests this, it gets set to
            # 7 days, but the default is taken within the last month for non-requests
            period = 30

            #check for block_id
            blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
                                       'FileData_Id=%i' % k)[0][0]

            #get the calibration types requested
            if blockid:
                request = saltmysql.select(sdb, 'RssCalibrationType_Id',
                                           'RssCalibration',
                                           'Block_Id=%i' % blockid)
                for cid in request:
                    cid = cid[0]
                    #check to see if the request is already in the database or if a similar request has
                    #been taken recently
                    if cid == 1:  #bias
                        if not checkforbias(sdb, k):
                            cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
                                night_id, k, cid)
                            saltmysql.insert(sdb, cmd_insert,
                                             'RssNightlyCalibration')
                    elif cid in [3, 4]:  #flats
                        if not checkforflats(sdb,
                                             k,
                                             cid,
                                             rssheaderlist,
                                             instr='rss',
                                             keylist=caldict[k],
                                             period=90):

                            cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
                                night_id, k, cid)
                            saltmysql.insert(sdb, cmd_insert,
                                             'RssNightlyCalibration')

                    elif cid == 7:  #specstand
                        period = 7
                        #print period, k, caldict[k]
                        if not checkforspst(
                                sdb, k, caldict[k], rssheaderlist,
                                period=period):
                            cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % (
                                night_id, k)
                            saltmysql.insert(sdb, cmd_insert,
                                             'RssNightlyCalibration')
                    else:
                        cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
                            night_id, k, cid)
                        saltmysql.insert(sdb, cmd_insert,
                                         'RssNightlyCalibration')

                    print k, cid, " ".join([str(w) for w in caldict[k]])

    #select all the HRS data from this obsdate

    #+-----------------------+-------------------------------+
    #| HrsCalibrationType_Id | CalibrationType               |
    #+-----------------------+-------------------------------+
    #|                     3 | Arc - Calsys                  |
    #|                     2 | Arc - Internal                |
    #|                     1 | Bias                          |
    #|                     4 | Spectroscopic flat - Lamp     |
    #|                     5 | Spectroscopic flat - Twilight |
    #|                    11 | Standard - Lick               |
    #|                     6 | Standard - Photometric        |
    #|                     9 | Standard - RV                 |
    #|                    10 | Standard - Smooth spectrum    |
    #|                     8 | Standard - Spectrophotometric |
    #|                     7 | Standard - Spectroscopic      |
    #+-----------------------+-------------------------------+

    hrsheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED'
    cmd_select = 'd.FileName,d.FileData_Id, %s' % hrsheaderlist
    cmd_table = ''' FileData as d
  left join FitsHeaderImage as f using (FileData_Id) 
  left join FitsHeaderHrs using (FileData_Id)
  join ProposalCode using (ProposalCode_Id)
'''
    cmd_logic = "d.FileName like 'H" + obsdate + "%' and CCDTYPE='Science' and Proposal_Code not like 'CAL_SPST'"

    results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)

    #loop through all the results and return only the Set of identical results
    caldict = create_caldict(results)

    for k in caldict:
        #first check to see if it has already been entered
        record = saltmysql.select(sdb, 'FileData_Id', 'HrsNightlyCalibration',
                                  'FileData_Id=%i' % k)
        if len(record) < 1:
            #period for checking for SPST.  If the uses requests this, it gets set to
            # 7 days, but the default is taken within the last month for non-requests
            period = 30

            #check for block_id
            blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
                                       'FileData_Id=%i' % k)[0][0]

            #get the calibration types requested
            if not checkforbias(sdb, k, instr='hrs'):
                cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, HrsCalibrationType_Id=%i' % (
                    night_id, k, 1)
                saltmysql.insert(sdb, cmd_insert, 'HrsNightlyCalibration')

    return