def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password): """Load logstr into the SDB""" print logstr #connect to the db sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #get the nightinfoid #date='YYYY-MM-DD' <- note speechmarks #data= your table #Select NightInfo_Id from NightInfo where Date=$date; #(get nightinfo_Id) #Insert into ObsLogTable values ($nightinfo_Id, $data); logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8]) results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic) night_id = results[0][0] #check to see if it exists logic = 'NightInfo_Id=%i' % night_id results = saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic) if results: update_cmd = "ObsLogTable='%s'" % (logstr) saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic) else: insert_cmd = "NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr) saltmysql.insert(sdb, insert_cmd, 'ObsLogTable') print obsdate, night_id
def checkforbias(sdb, k, instr='rss'): """Check to see if a bias of the same type already exists in the calibration table """ if instr=='rss': caltable='RssNightlyCalibration' logic='RssCalibrationType_Id=1' fitstable='FitsHeaderRss' elif instr=='scam': caltable='SalticamNightlyCalibration' logic='SalticamCalibrationType_Id=1' fitstable='FitsHeaderSalticam' #get all the bias requests logic='CalibrationTaken is Null and Ignored=0 and %s' % logic record=saltmysql.select(sdb, 'FileData_Id', caltable, logic) # if no results return false if len(record)<1: return False select='CCDSUM,GAINSET, ROSPEED' table='FileData join %s using (FileData_Id)' % fitstable logic='FileData_Id=%i' % k kdata=saltmysql.select(sdb, select, table, logic)[0] for i in record: idata=saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0] if compare_configs(kdata, idata): return True return False
def checkforbias(sdb, k, instr='rss'): """Check to see if a bias of the same type already exists in the calibration table """ if instr == 'rss': caltable = 'RssNightlyCalibration' logic = 'RssCalibrationType_Id=1' fitstable = 'FitsHeaderRss' elif instr == 'scam': caltable = 'SalticamNightlyCalibration' logic = 'SalticamCalibrationType_Id=1' fitstable = 'FitsHeaderSalticam' elif instr == 'hrs': caltable = 'HrsNightlyCalibration' logic = 'HrsCalibrationType_Id=1' fitstable = 'FitsHeaderHrs' #get all the bias requests logic = 'CalibrationTaken is Null and Ignored=0 and %s' % logic record = saltmysql.select(sdb, 'FileData_Id', caltable, logic) # if no results return false if len(record) < 1: return False #hrs only uses one mode if instr == 'hrs': return True select = 'CCDSUM,GAINSET, ROSPEED' table = 'FileData join %s using (FileData_Id)' % fitstable logic = 'FileData_Id=%i' % k kdata = saltmysql.select(sdb, select, table, logic)[0] for i in record: idata = saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0] if compare_configs(kdata, idata): return True return False
def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password): """Load logstr into the SDB""" print logstr #connect to the db sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #get the nightinfoid #date='YYYY-MM-DD' <- note speechmarks #data= your table #Select NightInfo_Id from NightInfo where Date=$date; #(get nightinfo_Id) #Insert into ObsLogTable values ($nightinfo_Id, $data); logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8]) results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic) night_id=results[0][0] #check to see if it exists logic='NightInfo_Id=%i' % night_id results=saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic) if results: update_cmd="ObsLogTable='%s'" % (logstr) saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic) else: insert_cmd="NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr) saltmysql.insert(sdb, insert_cmd, 'ObsLogTable') print obsdate, night_id
def checkforflats(sdb, fid, caltype, plist, instr='rss', keylist=None, period=90): """Check to see if a bias of the same type already exists in the calibration table fid: int FileData_Id caltype: int calibration type being checked for keylist: list list of RSS header keywords plist: string string of RSS headers needed instr: string either 'rss' or 'scam' """ if instr == 'rss': caltable = 'RssNightlyCalibration' logic = 'RssCalibrationType_Id=%i' % caltype fitstable = 'FitsHeaderRss' elif instr == 'scam': caltable = 'SalticamNightlyCalibration' logic = 'SalticamCalibrationType_Id=%i' % caltype fitstable = 'FitsHeaderSalticam' if keylist is not None: #set the period for to check for the data try: utstart = saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0] except Exception, e: print e return False utstart = utstart - datetime.timedelta(days=period) #select all the RSS data from this obsdate cmd_select = 'FileName,FileData_Id, %s' % plist cmd_table = ''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic = " UTSTART>'%s'" % (utstart) results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #compare results for r in results: if compare_configs(keylist[:-1], r[2:-1]): return True
def updatedq(img, struct, sdb): """Add information about the image to the database """ #get the filenumber #check to see if the FileData was created logic="FileName='%s'" % img records=saltmysql.select(sdb,'FileData_Id','FileData',logic) try: FileData_Id=records[0][0] except: message='WARNING: File not yet in database' print message return #get the information from the image for i in range(1,len(struct)): if struct[i].name=='SCI': try: omean=struct[i].header['OVERSCAN'] except: omean=None try: orms=struct[i].header['OVERRMS'] except: orms=None #lets measureme the statistics in a 200x200 box in each image try: my,mx=struct[i].data.shape dx1=int(mx*0.5) dx2=min(mx,dx1+200) dy1=int(my*0.5) dy2=min(my,dy1+200) mean,med,sig=saltstat.iterstat(struct[i].data[dy1:dy2,dx1:dx2], 5, 5) except: mean, med, sig=(None, None, None) #update the database with this information #check to see if you need to update or insert record=saltmysql.select(sdb, 'FileData_Id', 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i)) update=False if record: update=True ins_cmd='' if omean is not None: ins_cmd='OverscanMean=%s,' % omean if orms is not None: ins_cmd+='OverscanRms=%s,' % orms if mean is not None: ins_cmd+='BkgdMean=%f,' % mean if sig is not None: ins_cmd+='BkgdRms=%f' % sig if update: ins_cmd=ins_cmd.rstrip(',') saltmysql.update(sdb, ins_cmd, 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i)) else: ins_cmd+=',FileData_Id=%i, Extension=%i' % (FileData_Id, i) saltmysql.insert(sdb, ins_cmd, 'PipelineDataQuality_CCD')
def checkforflats(sdb, fid, caltype, plist, instr='rss', keylist=None, period=90): """Check to see if a bias of the same type already exists in the calibration table fid: int FileData_Id caltype: int calibration type being checked for keylist: list list of RSS header keywords plist: string string of RSS headers needed instr: string either 'rss' or 'scam' """ if instr=='rss': caltable='RssNightlyCalibration' logic='RssCalibrationType_Id=%i' % caltype fitstable='FitsHeaderRss' elif instr=='scam': caltable='SalticamNightlyCalibration' logic='SalticamCalibrationType_Id=%i' %caltype fitstable='FitsHeaderSalticam' if keylist is not None: #set the period for to check for the data try: utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0] except Exception, e: print e return False utstart=utstart-datetime.timedelta(days=period) #select all the RSS data from this obsdate cmd_select='FileName,FileData_Id, %s' % plist cmd_table=''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic="Proposal_Code like 'CAL_FLAT' and UTSTART>'%s'" % (utstart) results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #compare results for r in results: if compare_configs(keylist[-1], r[2:-1]): return True
def checkforhrscal(sdb, obsmode, propid, cal_type, period=7): """Check the sdb for HRS arcs for a certain mode. Return true if the data have been taken fid: int FileData_Id obsmode: str Observation mode propid: str Proposal code cal_type: int Calibration Type period: int number of days in the past to check if data were taken """ utstart = datetime.datetime.now() # set the period for to check for the data utstart = utstart - datetime.timedelta(days=period) # select all the HRS data from this obsdate cmd_select = "FileName,FileData_Id" cmd_table = """ FileData as d join ProposalCode using (ProposalCode_Id) """ cmd_logic = "Proposal_Code like '%s' and OBSMODE like '%s' and UTSTART>'%s'" % (propid, obsmode, utstart) results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) if len(results) > 0: return True # now check the nightly calibrations table table = "HrsNightlyCalibration join FileData using (FileData_Id)" logic = 'OBSMODE like "%s" and CalibrationTaken is Null and Ignored=0 and HrsCalibrationType_Id=%i' % ( obsmode, cal_type, ) record = saltmysql.select(sdb, "FileData_Id", table, logic) # if no results return false if len(record) > 0: return True return False
def saltelserror(obsdate, elshost, elsname, elsuser, elspass, sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose): # set up proposers = [] propids = [] pids = [] with logging(logfile,debug) as log: #open the database els=saltmysql.connectdb(elshost, elsname, elsuser, elspass) sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password) #create the values for the entire night nid=saltmysql.getnightinfoid(sdb, obsdate) stime, etime=saltmysql.select(sdb, 'EveningTwilightEnd,MorningTwilightStart', 'NightInfo', 'NightInfo_Id=%i' % nid)[0] print stime, etime errors=gettcserrors(els, stime, etime) if not errors: return None for e in errors: i=e.index('T') print i, error_list[i], e[-1] print len(errors)
def movebvitdata(pid, datapath, ftpdir, obsdate, sdb, clobber): """sub-routine to move the files to the mirror location for the beachhead returns status """ #log into database and determine username from proposer name state_select='p.Username' state_from='Proposal as pr join ProposalCode as c using (ProposalCode_Id) join ProposalContact as pc using (Proposal_Id) join Investigator as i on (pc.Contact_Id=i.Investigator_Id) join PiptUser as p using (PiptUser_Id)' state_logic="pr.current=1 and c.Proposal_Code='%s'" % pid record=saltmysql.select(sdb,state_select,state_from,state_logic) record=set(record) #check to see if it was successful and raise an error if not if len(record)<1: message='SALTFTP--Unable to find username for %s' % pid raise SaltError(message) #go through and add the data to each entry in username for entry in record: pi_username=entry[0] ftppath=ftpdir+'/'+pi_username+'/'+pid+'_'+obsdate #copy the zip file to the beachhead directory if clobber and os.path.exists(ftppath): shutil.rmtree(ftppath) #copy the file shutil.copytree(datapath, ftppath) message = 'SALTFTP -- %s --> %s' % (datapath, ftppath) return message
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password): """Handle fast data delivery for the proposal. For a given filename """ if propcode is None or propcode == 'None': return #first check in the sdb if fast data delivery is needed sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) select_term = 'Distinct Surname, email, username, ProposalCode_Id' from_term = ''' Block join Pointing using (Block_Id) join PipelineConfig using (Pointing_Id) join ProposalCode using (ProposalCode_Id) join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id) join ProposalContact using (ProposalCode_Id) join Investigator on (Investigator_Id=Contact_Id) join PiptUser using (PiptUser_Id) ''' where_term="Proposal_Code like '%s' and DataAccessMethod='Fast'" \ % (propcode) #print 'Select %s from %s where %s' % (select_term, from_term, where_term) try: record = saltmysql.select(sdb, select_term, from_term, where_term) except Exception, e: print e return None
def checkforspst(sdb, fid, keylist, plist, period=7): """Check the sdb for SPST observations. Returns true if the data have been taken in the last period. fid: int FileData_Id dlist: list header information for a given file keylist: list list of RSS header keywords plist: string string of RSS headers needed period: int number of days in the past to check if data were taken Note ---- The [:-1] selection in the comparison of the lists is to exclude maskid """ caltable='RssNightlyCalibration' try: utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0] except Exception, e: print e return False
def checksdbforfits(inname, sdb, log,verbose): """ Check to see if the image is in the database returns FileData_Id, status """ FileData_Id=-1 #check to see if the name is the FileName logic="FileName='%s'" % inname records=saltmysql.select(sdb,'FileData_Id','FileData',logic) if not records: FileData_Id=-1 message='SALTSDBLOADFITS: Adding %s to database' % inname elif len(records[0])==1 : FileData_Id=records[0][0] message='SALTSDBLOADFITS: Updating %s in database' % inname elif len(records[0])>1: message='SALTLOADSDBFITS -- ERROR: Ambigous record for %s' % inname raise SaltError( message) if verbose: log.message(message, with_header=False) return FileData_Id
def seeingtable(sdb, mintime, maxtime): #extract the seeing data from the sdb sel_cmd='DateTime, Mass, Dimm' tab_cmd='MassDimm' log_cmd="DateTime>'%s' and DateTime<'%s'" % (mintime, maxtime) see_rec=saltmysql.select(sdb, sel_cmd, tab_cmd, log_cmd) if len(see_rec)<2: return None stime_list=[] mass_arr=np.zeros(len(see_rec)) dimm_arr=np.zeros(len(see_rec)) for i in range(len(see_rec)): stime_list.append(see_rec[i][0]) mass_arr[i]=see_rec[i][1] dimm_arr[i]=see_rec[i][2] seecol=[] seecol.append(pyfits.Column(name='Timestamp', format='20A', array=stime_list)) seecol.append(pyfits.Column(name='MASS', format='F', array=mass_arr )) seecol.append(pyfits.Column(name='DIMM', format='F', array=dimm_arr )) seetab= saltio.fitscolumns(seecol) seehdu= pyfits.new_table(seetab) seehdu.name='Seeing' return seehdu
def erroremail(obsdate, server='', username='', password='', sender='', recipient='', bcc='', sdbhost='sdb.salt', sdbname='sdb', sdbuser='', sdbpass='', logfile='saltlog.log', verbose=True): """Update the PipelineStatistics table with the current information about the pipeline """ with logging(logfile,debug) as log: #connect the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass) #get the nightinfo_id night_id=saltmysql.getnightinfoid(sdb, obsdate) #create the message try: record=saltmysql.select(sdb, 'PipelineStatus, ErrorMessage', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % night_id) status, error_message=record[0] except Exception, e: raise SaltError('Unable to download information: %s' % e) #set up the subject and message subject='Pipeline Error on %s' % obsdate message="Failure is simply the opportunity to begin again, this time more intelligently.--Henry Ford\n\n" message+="%s" % error_message log.message(message, with_stdout=verbose) #send the email saltio.email(server,username,password,sender,recipient,bcc, subject,message)
def getweatherdata(els, weathertime, timespan): """Creates the weather table from the data in the els """ etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200 stime = etime - timespan # now extact weather information from the els sel_cmd = "timestamp, air_pressure, dewpoint, rel_humidity, wind_mag_30m, wind_dir_30m, wind_mag_10m, wind_dir_10m, temperatures, rain_detected" tab_cmd = "bms_external_conditions" log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime) wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd) if len(wea_rec) < 1: return 8 * [None] time_list = [] air_arr = np.zeros(len(wea_rec)) dew_arr = np.zeros(len(wea_rec)) hum_arr = np.zeros(len(wea_rec)) w30_arr = np.zeros(len(wea_rec)) w30d_arr = np.zeros(len(wea_rec)) w10_arr = np.zeros(len(wea_rec)) w10d_arr = np.zeros(len(wea_rec)) rain_list = [] t02_arr = np.zeros(len(wea_rec)) t05_arr = np.zeros(len(wea_rec)) t10_arr = np.zeros(len(wea_rec)) t15_arr = np.zeros(len(wea_rec)) t20_arr = np.zeros(len(wea_rec)) t25_arr = np.zeros(len(wea_rec)) t30_arr = np.zeros(len(wea_rec)) for i in range(len(wea_rec)): time_list.append(str(wea_rec[i][0])) air_arr[i] = wea_rec[i][1] dew_arr[i] = wea_rec[i][2] hum_arr[i] = wea_rec[i][3] w30_arr[i] = wea_rec[i][4] w30d_arr[i] = wea_rec[i][5] w10_arr[i] = wea_rec[i][6] w10d_arr[i] = wea_rec[i][7] rain_list.append(wea_rec[i][9]) t_arr = converttemperature(wea_rec[i][8]) t02_arr[i] = t_arr[0] t05_arr[i] = t_arr[1] t10_arr[i] = t_arr[2] t15_arr[i] = t_arr[3] t20_arr[i] = t_arr[4] t25_arr[i] = t_arr[5] t30_arr[i] = t_arr[6] # average the wind direction by taking the arctan of the average of the sin and cos wdir = np.degrees(np.arctan2(np.sin(np.radians(w30d_arr)).mean(), np.cos(np.radians(w30d_arr)).mean())) if wdir < 0: wdir += 360 # determine if there was any rain during the period rain = 0 if "T" in rain_list: rain = 1 return air_arr.mean(), dew_arr.mean(), hum_arr.mean(), t02_arr.mean(), t30_arr.mean(), w30_arr.mean(), wdir, rain
def checksdbforfits(inname, sdb, log, verbose): """ Check to see if the image is in the database returns FileData_Id, status """ FileData_Id = -1 #check to see if the name is the FileName logic = "FileName='%s'" % inname records = saltmysql.select(sdb, 'FileData_Id', 'FileData', logic) if not records: FileData_Id = -1 message = 'SALTSDBLOADFITS: Adding %s to database' % inname elif len(records[0]) == 1: FileData_Id = records[0][0] message = 'SALTSDBLOADFITS: Updating %s in database' % inname elif len(records[0]) > 1: message = 'SALTLOADSDBFITS -- ERROR: Ambigous record for %s' % inname raise SaltError(message) if verbose: log.message(message, with_header=False) return FileData_Id
def getblockid(sdb, night_id, pid, img_list, starttime, endtime): """Return the block id for this set of data""" blockid = -1 targlist = [] for img in img_list: itime = img[4] + datetime.timedelta(0, 7200.0) if itime > starttime and itime < endtime: if img[8] == "OBJECT": targlist.append(img[2]) for t in set(targlist): select_state = "Block_Id, BlockVisit_Id, Accepted" table_state = """Block join BlockVisit using (Block_Id) join Pointing using (Block_Id) join Observation using (Pointing_Id) join Target using (Target_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id) """ logic_state = "NightInfo_Id=%i and Target_Name='%s' and Proposal_Code='%s'" % (night_id, t, pid.strip()) block_list = saltmysql.select(sdb, select_state, table_state, logic_state) try: return block_list[0] except: return -1, -1, -1
def guidertable(els, mintime, maxtime): """Extract the guider data from the els""" #extract the guider data from the els sel_cmd='_timestamp_, guidance_available, ee50, mag50' tab_cmd='tpc_guidance_status__timestamp ' log_cmd="_timestamp_>'%s' and _timestamp_<'%s'" % (mintime, maxtime) gui_rec=saltmysql.select(els, sel_cmd, tab_cmd, log_cmd) if len(gui_rec)<2: return None gtime_list=[] ee50_arr=np.zeros(len(gui_rec)) mag50_arr=np.zeros(len(gui_rec)) avail_list=[] for i in range(len(gui_rec)): gtime_list.append(gui_rec[i][0]) ee50_arr[i]=gui_rec[i][2] mag50_arr[i]=gui_rec[i][3] avail_list.append(gui_rec[i][1]) avail_arr=(np.array(avail_list)=='T') #write the results to a fits table guicol=[] guicol.append(pyfits.Column(name='Timestamp', format='20A', array=gtime_list)) guicol.append(pyfits.Column(name='Available', format='L', array=avail_arr )) guicol.append(pyfits.Column(name='EE50', format='F', array=ee50_arr )) guicol.append(pyfits.Column(name='mag50', format='F', array=mag50_arr )) guitab= saltio.fitscolumns(guicol) guihdu= pyfits.new_table(guitab) guihdu.name='Guider' return guihdu
def checkforspst(sdb, fid, keylist, plist, period=7, utstart=None): """Check the sdb for SPST observations. Returns true if the data have been taken in the last period. fid: int FileData_Id keylist: list list of RSS header keywords plist: string string of RSS headers needed period: int number of days in the past to check if data were taken Note ---- The [:-1] selection in the comparison of the lists is to exclude maskid """ caltable = 'RssNightlyCalibration' if utstart is None: try: utstart = saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0] except Exception, e: print e return False
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password): """Handle fast data delivery for the proposal. For a given filename """ if propcode is None or propcode=='None': return #first check in the sdb if fast data delivery is needed sdb=saltmysql.connectdb(sdbhost,sdbname, sdbuser, password) select_term='Distinct Surname, email, username, ProposalCode_Id' from_term=''' Block join Pointing using (Block_Id) join PipelineConfig using (Pointing_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id) join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id) join ProposalContact using (Proposal_Id) join Investigator on (Investigator_Id=Contact_Id) join PiptUser using (PiptUser_Id) ''' where_term="Proposal_Code like '%s' and current=1 and DataAccessMethod='Fast'" \ % (propcode) print 'Select %s from %s where %s' % (select_term, from_term, where_term) try: record=saltmysql.select(sdb, select_term, from_term, where_term) except Exception, e: print e return None
def checkforhrscal(sdb, obsmode, propid, cal_type, period=7): """Check the sdb for HRS arcs for a certain mode. Return true if the data have been taken fid: int FileData_Id obsmode: str Observation mode propid: str Proposal code cal_type: int Calibration Type period: int number of days in the past to check if data were taken """ utstart = datetime.datetime.now() #set the period for to check for the data utstart = utstart - datetime.timedelta(days=period) #select all the HRS data from this obsdate cmd_select = 'FileName,FileData_Id' cmd_table = ''' FileData as d join ProposalCode using (ProposalCode_Id) ''' cmd_logic = "Proposal_Code like '%s' and OBSMODE like '%s' and UTSTART>'%s'" % ( propid, obsmode, utstart) results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) if len(results) > 0: return True #now check the nightly calibrations table table = 'HrsNightlyCalibration join FileData using (FileData_Id)' logic = 'OBSMODE like "%s" and CalibrationTaken is Null and Ignored=0 and HrsCalibrationType_Id=%i' % ( obsmode, cal_type) record = saltmysql.select(sdb, 'FileData_Id', table, logic) # if no results return false if len(record) > 0: return True return False
def tcsstatus(els, start, end): """Determine the time spent slewing between two observations return a list of start of slew time, end of slew time for the times between start and end """ select='_timestamp_, tcs_mode' table='tcs_status__timestamp' logic="_timestamp_> '%s' and _timestamp_< '%s'" % (start, end) record=saltmysql.select(els, select, table, logic) return record
def get_image_info(sdb, filename): """Get information about the image from the database""" select = "fd.FileName, hdr.PROPID, hdr.CCDTYPE, hdr.OBSMODE, hdr.DETMODE, hdr.OBJECT, hdr.TELRA, hdr.TELDEC, hdr.DATE_OBS, hdr.TIME_OBS " table = ( "FileData as fd join FitsHeaderImage as hdr using (FileData_Id)" ) # join FitsHeaderHrs as hrs using (FileData_Id)' logic = 'fd.FileName like "%s"' % filename results = saltmysql.select(sdb, select, table, logic) return results[0]
def gettcserrors(els, mintime, maxtime): """Retrieve data for a given observation during an observation date for a proposal """ #now extact weather information from the els sel_cmd='*' tab_cmd='tcs_system_errors__timestamp' log_cmd="_timestamp_>'%s' and _timestamp_<'%s'" % (mintime, maxtime) tcs_rec=saltmysql.select(els, sel_cmd, tab_cmd, log_cmd) return tcs_rec
def checkforbias(sdb, k, instr="rss"): """Check to see if a bias of the same type already exists in the calibration table """ if instr == "rss": caltable = "RssNightlyCalibration" logic = "RssCalibrationType_Id=1" fitstable = "FitsHeaderRss" elif instr == "scam": caltable = "SalticamNightlyCalibration" logic = "SalticamCalibrationType_Id=1" fitstable = "FitsHeaderSalticam" elif instr == "hrs": caltable = "HrsNightlyCalibration" logic = "HrsCalibrationType_Id=1" fitstable = "FitsHeaderHrs" # get all the bias requests logic = "CalibrationTaken is Null and Ignored=0 and %s" % logic record = saltmysql.select(sdb, "FileData_Id", caltable, logic) # if no results return false if len(record) < 1: return False # hrs only uses one mode if instr == "hrs": return True select = "CCDSUM,GAINSET, ROSPEED" table = "FileData join %s using (FileData_Id)" % fitstable logic = "FileData_Id=%i" % k print select, table, logic kdata = saltmysql.select(sdb, select, table, logic)[0] for i in record: idata = saltmysql.select(sdb, select, table, "FileData_Id=%i" % i)[0] if compare_configs(kdata, idata): return True return False
def dataquality(obsdate, sdb, instrume, clobber=True, logfile=None, verbose=True): """Run the data quality measurement for each instrument """ if instrume=='rss': instrumetable='FitsHeaderRss' elif instrume=='scam': instrumetable='FitsHeaderSalticam' #Look for any biases from the night sel_cmd='FileName' tab_cmd='FileData join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable logic_cmd=makelogic("CCDTYPE='BIAS'", obsdate, getnextdate(obsdate)) record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd) print record #Look for any flatfields from the night sel_cmd='FileName' tab_cmd='FileData join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable logic_cmd=makelogic("CCDTYPE='FLAT'", obsdate, getnextdate(obsdate)) record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd) print record #Look for any Cal_SPST stars sel_cmd='FileName' tab_cmd='FileData join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable logic_cmd=makelogic("CCDTYPE='OBJECT' and Proposal_Code='CAL_SPST'", obsdate, getnextdate(obsdate)) record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd) print record #reduce each of the spectrophotometric standards if len(record): for infile in record: profile='/salt/data/%s/%s/%s/product/mbxgp%s' % (obsdate[0:4], obsdate[4:8], instrume, infile[0]) reducespst(profile, obsdate, clobber=clobber, logfile=logfile, verbose=verbose)
def isproprietary(fid, sdb): """Determine if a data set is proprietary""" select='FileData_Id, FileName, Proposal_Code, ReleaseDate' table='FileData join ProposalCode using (ProposalCode_Id) join Proposal using (ProposalCode_Id)' logic="FileData_Id='%i' and current=1" % fid record=saltmysql.select(sdb, select, table, logic) #if no proposal exists, it isn't proprietary if not record: return False #if the release date is less than today, it isn't propietary releasedate=record[0][3] if releasedate<date.today(): return False #default is it is proprietary return True
def getacdata(els, weathertime, timespan): """Determien the AC temperature weathertime should be SAST """ # mktime converts weather time into seconds from UNIX epoch # 7200 converts to UT # 2082852000 converts from Unix Epoch to Labview Epoch of 1 Jan 1904 etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200 stime = etime - timespan # now extact weather information from the els sel_cmd = "AVG(timestamp), AVG(actual_a_c_temperature)" tab_cmd = "bms_status" log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime) wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd) if len(wea_rec) < 1: return None return wea_rec[0][1]
def getinsidedata(els, weathertime, timespan): """Creates the inside data from the els database Weathertime should be in SAST """ # mktime converts weather time into seconds from UNIX epoch # 7200 converts to UT # 2082852000 converts from Unix Epoch to Labview Epoch of 1 Jan 1904 etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200 stime = etime - timespan # now extact weather information from the els sel_cmd = "AVG(timestamp), AVG(dew_point), AVG(rel_humidity)" tab_cmd = "bms_internal_conditions" log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime) wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd) if len(wea_rec) < 1: return None, None return wea_rec[0][1], wea_rec[0][2]
def loaddata(fid, filename, sdb, das): """Load the data from the sdb to the das database for file with filedata_id given by fid""" print fid #load the data in filed data select="FileData_Id, UTStart, Target_Name, ExposureTime, FileName, INSTRUME, OBSMODE, DETMODE, Proposal_Code, Filter, TelRa, TelDec, FileName, FileSize" table="FileData join ProposalCode using (ProposalCode_Id)" if filename.startswith('S'): table += "join FitsHeaderSalticam using (FileData_Id)" elif filename.startswith('P'): table += "join FitsHeaderRss using (FileData_Id)" logic="FileData_ID=%i" % fid record = saltmysql.select(sdb, select, table, logic) print '%s' % record[0][1] record=record[0] insert="filedata_id=%i, UTSTART='%s', object='%s', exposuretime=%f, filename='%s', instrume='%s', obsmode='%s', detmode='%s', proposal_id='%s', filter='%s'" % \ (record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8], record[9]) insert+=",ra='%s'" % (dec2sex(record[10]/15.0)) insert+=",decl='%s'" % (dec2sex(record[11])) insert+=",filepath='%s', filesize=%f" % (makepath(filename), record[13]) print insert
def determinetime(sdb, pid, obsdate): #for a proposal, retrieve the maximum and minimum time sel_cmd='FileName, UTStart, ExposureTime, NExposures' tab_cmd='FileData join ProposalCode using (ProposalCode_Id)' log_cmd="FileName like '%"+str(obsdate)+"%' and Proposal_Code='"+pid+"'" record=saltmysql.select(sdb, sel_cmd, tab_cmd, log_cmd) if not record: raise SaltError('%s does not have any data on %s' % (pid, obsdate)) print pid, record maxtime=record[0][1] mintime=record[0][1] for r in record: maxtime=max(maxtime, r[1]+datetime.timedelta(seconds=r[2]*r[3])) mintime=min(mintime, r[1]) #correct the time from UTC to SAST mintime+=datetime.timedelta(seconds=2*3600.0) maxtime+=datetime.timedelta(seconds=2*3600.0) return mintime, maxtime
def saltdasload(obsdate, sdbhost, sdbname, sdbuser, password, dashost, dasname, dasuser, daspassword, logfile, verbose): """Upload data from the science database to the SALTDAS database """ with logging(logfile,debug) as log: #open the science database sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password) #open saltdas das=saltmysql.connectdb(dashost,dasname,dasuser,daspassword) select='FileData_Id, FileName, Proposal_Code' table='FileData join ProposalCode using (ProposalCode_Id)' logic="FileName like '%"+str(obsdate)+"%'" record=saltmysql.select(sdb, select, table, logic) for r in record[:]: if not isproprietary(r[0], sdb): print r loaddata(r[0], r[1], sdb, das) return
def findpropinfo(pid,sdb): """query the Sdb server and determine the email and surname for the contact information for the proposal return a dictionary with Surname,Email """ propinfo={} #setup the the query state_select='pr.Proposal_Id,c.Proposal_Code,i2.Surname,i2.Email' state_from=''' Investigator as i join PiptUser using (PiptUser_Id) join Investigator as i2 on (PiptUser.Investigator_Id=i2.Investigator_Id), Proposal as pr join ProposalCode as c using (ProposalCode_Id) join ProposalContact as pc using (Proposal_Id) ''' state_logic="i.Investigator_Id=pc.Contact_ID and pc.Proposal_ID=pr.Proposal_ID and pr.current=1 and c.Proposal_Code='%s'" %pid # left over by intersting way to call it: ORDER BY pr.Proposal_ID DESC" % pid record=saltmysql.select(sdb,state_select,state_from,state_logic) if len(record)<1: message = pid + ' is not in the Science database' raise SaltError(message) elif len(record)==1: surname=[record[0][2]] email=[record[0][3]] propinfo[pid]=(surname,email) else: prop_id=-1 surname=[] email=[] for entry in record: if entry[0] > prop_id: prop_id=entry[0] prop_code=entry[1] surname.append(entry[2]) email.append(entry[3]) propinfo[pid]=(surname,email) return propinfo
def checkforfast(pid, target, db): """Check the database to see if fast response is needed returns status""" record='' #log into database and determine response type of proposal #If the object is not in the proposal, and there is a request for a fast response #for other objects in the proposal, then it should responsd with a fast response. state_select='T.Target_name, pdam.DataAccessMethod' state_from=""" Proposal join ProposalCode using (ProposalCode_Id) join Block using (Proposal_Id) join Pointing using (Block_Id) join Observation using (Pointing_Id) join Target as T using (Target_Id) join PipelineConfig using (Pointing_Id) join PipelineDataAccessMethod as pdam using (PipelineDataAccessMethod_Id) """ state_logic="Current=1" state_logic +=" and Proposal_Code='%s'" % pid #state_logic +=" and pr.Proposal_Code='%s' and T.Target_Name like '%s' " % (pid, target) #print "select %s from %s whre %s" % (state_select,state_from,state_logic) record=saltmysql.select(db,state_select,state_from,state_logic) record=set(record) #if there are no records in the set if len(record)==0: return False #TODO: Add a check to make sure that for t,m in record: if m.upper()=='FAST' and t in target: return True return False
def findcal(obsdate, sdbhost, sdbname, sdbuser, password): """Find all of the unique configuration settings used on a night and insert it into the database obsdate--Observation date in YYYYMMDD sdbhost--host name for sdb sdbname--name of the sdb sdbuser--user for the sdb password--sdb password """ #connect to the db sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #get the nightinfo id logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8]) results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic) night_id = results[0][0] #select all the scam data from this obsdate cmd_select = 'd.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER' cmd_table = ''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderSalticam using (FileData_Id) ''' cmd_logic = "d.FileName like 'S" + obsdate + "%' and f.CCDTYPE='OBJECT'" results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #loop through all the results and return only the Set of identical results caldict = create_caldict(results) #insert the results into the database for k in caldict: #first check to see if it has already been entered record = saltmysql.select(sdb, 'FileData_Id', 'SalticamNightlyCalibration', 'FileData_Id=%i' % k) if len(record) < 1: #check for block_id blockid = saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0] #get the calibration types requested if blockid: request = saltmysql.select(sdb, 'SalticamCalibrationType_Id', 'SalitcamCalibration', 'Block_Id=%i' % blockid) for cid in request: cid = cid[0] cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % ( night_id, k, cid) #saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration') print k, " ".join([str(k) for k in caldict[k]]) #list of rss calibration types #+-----------------------+----------------------------------+ #| RssCalibrationType_Id | CalibrationType | #+-----------------------+----------------------------------+ #| 2 | Arc | #| 1 | Bias | #| 14 | Imaging flat - Lamp | #| 15 | Imaging flat - Twilight | #| 3 | Spectroscopic flat - Lamp | #| 4 | Spectroscopic flat - Twilight | #| 12 | Standard - Circular polarimetric | #| 13 | Standard - Lick | #| 9 | Standard - Linear polarimetric | #| 5 | Standard - Photometric | #| 8 | Standard - RV | #| 11 | Standard - Smooth spectrum | #| 7 | Standard - Spectrophotometric | #| 6 | Standard - Spectroscopic | #| 10 | Standard - Unpolarised | #+-----------------------+----------------------------------+ #select all the RSS data from this obsdate rssheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, AR_STA, MASKID' cmd_select = 'd.FileName,d.FileData_Id, %s' % rssheaderlist # CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID' cmd_table = ''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic = "d.FileName like 'P" + obsdate + "%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST' and GRTILT > 0" results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #loop through all the results and return only the Set of identical results caldict = create_caldict(results) #insert the rss results into the database for k in caldict: #first check to see if it has already been entered record = saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration', 'FileData_Id=%i' % k) if len(record) < 1: #period for checking for SPST. If the uses requests this, it gets set to # 7 days, but the default is taken within the last month for non-requests period = 30 #check for block_id blockid = saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0] #get the calibration types requested if blockid: request = saltmysql.select(sdb, 'RssCalibrationType_Id', 'RssCalibration', 'Block_Id=%i' % blockid) for cid in request: cid = cid[0] #check to see if the request is already in the database or if a similar request has #been taken recently if cid == 1: #bias if not checkforbias(sdb, k): cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % ( night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') elif cid in [3, 4]: #flats if not checkforflats(sdb, k, cid, rssheaderlist, instr='rss', keylist=caldict[k], period=90): cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % ( night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') elif cid == 7: #specstand period = 7 #print period, k, caldict[k] if not checkforspst( sdb, k, caldict[k], rssheaderlist, period=period): cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % ( night_id, k) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') else: cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % ( night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') print k, cid, " ".join([str(w) for w in caldict[k]]) #select all the HRS data from this obsdate #+-----------------------+-------------------------------+ #| HrsCalibrationType_Id | CalibrationType | #+-----------------------+-------------------------------+ #| 3 | Arc - Calsys | #| 2 | Arc - Internal | #| 1 | Bias | #| 4 | Spectroscopic flat - Lamp | #| 5 | Spectroscopic flat - Twilight | #| 11 | Standard - Lick | #| 6 | Standard - Photometric | #| 9 | Standard - RV | #| 10 | Standard - Smooth spectrum | #| 8 | Standard - Spectrophotometric | #| 7 | Standard - Spectroscopic | #+-----------------------+-------------------------------+ hrsheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED' cmd_select = 'd.FileName,d.FileData_Id, %s' % hrsheaderlist cmd_table = ''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderHrs using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic = "d.FileName like 'H" + obsdate + "%' and CCDTYPE='Science' and Proposal_Code not like 'CAL_SPST'" results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #loop through all the results and return only the Set of identical results caldict = create_caldict(results) for k in caldict: #first check to see if it has already been entered record = saltmysql.select(sdb, 'FileData_Id', 'HrsNightlyCalibration', 'FileData_Id=%i' % k) if len(record) < 1: #period for checking for SPST. If the uses requests this, it gets set to # 7 days, but the default is taken within the last month for non-requests period = 30 #check for block_id blockid = saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0] #get the calibration types requested if not checkforbias(sdb, k, instr='hrs'): cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, HrsCalibrationType_Id=%i' % ( night_id, k, 1) saltmysql.insert(sdb, cmd_insert, 'HrsNightlyCalibration') return
def checkforspst(sdb, fid, keylist, plist, period=7): """Check the sdb for SPST observations. Returns true if the data have been taken in the last period. fid: int FileData_Id keylist: list list of RSS header keywords plist: string string of RSS headers needed period: int number of days in the past to check if data were taken Note ---- The [:-1] selection in the comparison of the lists is to exclude maskid """ caltable='RssNightlyCalibration' try: utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0] except Exception as e: print(e) return False #set the period for to check for the data utstart=utstart-datetime.timedelta(days=period) #select all the RSS data from this obsdate cmd_select='FileName,FileData_Id, %s' % plist cmd_table=''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic="Proposal_Code like 'CAL_SPST' and UTSTART>'%s'" % (utstart) results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #compare results for r in results: if compare_configs(keylist[:-1], r[2:-1]): return True #now check the nightly calibrations table logic='CalibrationTaken is Null and Ignored=0 and RssCalibrationType_Id=7' record=saltmysql.select(sdb, 'FileData_Id', caltable, logic) # if no results return false if len(record)<1: return False #selection the data from the list and compare results cmd_select='FileName,FileData_Id, %s' % plist cmd_table=''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' for i in record: cmd_logic='FileData_Id=%i' % i r=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)[0] if compare_configs(keylist[:-1], r[2:-1]): return True return False
def findcal(obsdate, sdbhost, sdbname, sdbuser, password): """Find all of the unique configuration settings used on a night and insert it into the database obsdate--Observation date in YYYYMMDD sdbhost--host name for sdb sdbname--name of the sdb sdbuser--user for the sdb password--sdb password """ #connect to the db sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #get the nightinfo id logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8]) results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic) night_id=results[0][0] #select all the scam data from this obsdate cmd_select='d.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER' cmd_table=''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderSalticam using (FileData_Id) ''' cmd_logic="d.FileName like 'S" + obsdate+"%' and f.CCDTYPE='OBJECT'" results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #loop through all the results and return only the Set of identical results caldict=create_caldict(results) #insert the results into the database for k in caldict: #first check to see if it has already been entered record=saltmysql.select(sdb, 'FileData_Id', 'SalticamNightlyCalibration', 'FileData_Id=%i' % k) if len(record)<1: #check for block_id blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0] #get the calibration types requested if blockid: request=saltmysql.select(sdb, 'SalticamCalibrationType_Id', 'SalitcamCalibration', 'Block_Id=%i' % blockid) for cid in request: cid=cid[0] cmd_insert='NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % (night_id, k, cid) #saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration') print(k, " ".join([str(k) for k in caldict[k]])) #list of rss calibration types #+-----------------------+----------------------------------+ #| RssCalibrationType_Id | CalibrationType | #+-----------------------+----------------------------------+ #| 2 | Arc | #| 1 | Bias | #| 14 | Imaging flat - Lamp | #| 15 | Imaging flat - Twilight | #| 3 | Spectroscopic flat - Lamp | #| 4 | Spectroscopic flat - Twilight | #| 12 | Standard - Circular polarimetric | #| 13 | Standard - Lick | #| 9 | Standard - Linear polarimetric | #| 5 | Standard - Photometric | #| 8 | Standard - RV | #| 11 | Standard - Smooth spectrum | #| 7 | Standard - Spectrophotometric | #| 6 | Standard - Spectroscopic | #| 10 | Standard - Unpolarised | #+-----------------------+----------------------------------+ #select all the RSS data from this obsdate rssheaderlist='f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID' cmd_select='d.FileName,d.FileData_Id, %s' % rssheaderlist # CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID' cmd_table=''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic="d.FileName like 'P" + obsdate+"%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST'" results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #loop through all the results and return only the Set of identical results caldict=create_caldict(results) #insert the rss results into the database for k in caldict: #first check to see if it has already been entered record=saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration', 'FileData_Id=%i' % k) if len(record)<1: #period for checking for SPST. If the uses requests this, it gets set to # 7 days, but the default is taken within the last month for non-requests period=30 #check for block_id blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0] #get the calibration types requested if blockid: request=saltmysql.select(sdb, 'RssCalibrationType_Id', 'RssCalibration', 'Block_Id=%i' % blockid) for cid in request: cid=cid[0] #check to see if the request is already in the database or if a similar request has #been taken recently if cid==1: #bias if not checkforbias(sdb, k): cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') elif cid in [3,4]: #flats if not checkforflats(sdb, k, cid, rssheaderlist, instr='rss', keylist=caldict[k], period=90): cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') elif cid==7: #specstand period=7 #print period, k, caldict[k] if not checkforspst(sdb, k, caldict[k], rssheaderlist, period=period): cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % (night_id, k) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') else: cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid) saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration') print(k, cid," ".join([str(w) for w in caldict[k]])) return
cmd_table = ''' FileData as d left join FitsHeaderImage as f using (FileData_Id) left join FitsHeaderRss using (FileData_Id) join ProposalCode using (ProposalCode_Id) ''' cmd_logic = " UTSTART>'%s'" % (utstart) results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic) #compare results for r in results: if compare_configs(keylist[:-1], r[2:-1]): return True #get all the flat requests logic = 'CalibrationTaken is Null and Ignored=0 and %s' % logic record = saltmysql.select(sdb, 'FileData_Id', caltable, logic) # if no results return false if len(record) < 1: return False select = plist table = 'FileData as d join FitsHeaderImage as f using (FileData_Id) join %s using (FileData_Id)' % fitstable logic = 'FileData_Id=%i' % fid #select kdata = saltmysql.select(sdb, select, table, logic)[0] for i in record: idata = saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0] if compare_configs(kdata, idata): return True return False
except Exception, e: print e #check to see if an email has been sent select_term = 'PipelineStatus' from_term = ''' PipelineProposalStatistics join PipelineStatus using (PipelineStatus_Id) join NightInfo using (NightInfo_Id) join ProposalCode using (ProposalCode_Id) ''' where_term = "Proposal_Code like '%s' and Date='%s-%s-%s'" % ( propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8]) print select_term, from_term, where_term try: record = saltmysql.select(sdb, select_term, from_term, where_term)[0][0] except: record = None print record if record == 'FastEmail': return else: #insert information into the database nightinfoid = saltmysql.getnightinfoid(sdb, obsdate) insert_term = "NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % ( nightinfoid, propid) table_term = "PipelineProposalStatistics" saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics") #send email