def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
"""Load logstr into the SDB"""
print logstr
#connect to the db
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfoid
#date='YYYY-MM-DD' <- note speechmarks
#data= your table
#Select NightInfo_Id from NightInfo where Date=$date;
#(get nightinfo_Id)
#Insert into ObsLogTable values ($nightinfo_Id, $data);
logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id = results[0][0]
#check to see if it exists
logic = 'NightInfo_Id=%i' % night_id
results = saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
if results:
update_cmd = "ObsLogTable='%s'" % (logstr)
saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
else:
insert_cmd = "NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')
print obsdate, night_id
def checkforbias(sdb, k, instr='rss'):
"""Check to see if a bias of the same type already exists in the calibration table
"""
if instr=='rss':
caltable='RssNightlyCalibration'
logic='RssCalibrationType_Id=1'
fitstable='FitsHeaderRss'
elif instr=='scam':
caltable='SalticamNightlyCalibration'
logic='SalticamCalibrationType_Id=1'
fitstable='FitsHeaderSalticam'
#get all the bias requests
logic='CalibrationTaken is Null and Ignored=0 and %s' % logic
record=saltmysql.select(sdb, 'FileData_Id', caltable, logic)
# if no results return false
if len(record)<1: return False
select='CCDSUM,GAINSET, ROSPEED'
table='FileData join %s using (FileData_Id)' % fitstable
logic='FileData_Id=%i' % k
kdata=saltmysql.select(sdb, select, table, logic)[0]
for i in record:
idata=saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0]
if compare_configs(kdata, idata): return True
return False
def checkforbias(sdb, k, instr='rss'):
"""Check to see if a bias of the same type already exists in the calibration table
"""
if instr == 'rss':
caltable = 'RssNightlyCalibration'
logic = 'RssCalibrationType_Id=1'
fitstable = 'FitsHeaderRss'
elif instr == 'scam':
caltable = 'SalticamNightlyCalibration'
logic = 'SalticamCalibrationType_Id=1'
fitstable = 'FitsHeaderSalticam'
elif instr == 'hrs':
caltable = 'HrsNightlyCalibration'
logic = 'HrsCalibrationType_Id=1'
fitstable = 'FitsHeaderHrs'
#get all the bias requests
logic = 'CalibrationTaken is Null and Ignored=0 and %s' % logic
record = saltmysql.select(sdb, 'FileData_Id', caltable, logic)
# if no results return false
if len(record) < 1: return False
#hrs only uses one mode
if instr == 'hrs': return True
select = 'CCDSUM,GAINSET, ROSPEED'
table = 'FileData join %s using (FileData_Id)' % fitstable
logic = 'FileData_Id=%i' % k
kdata = saltmysql.select(sdb, select, table, logic)[0]
for i in record:
idata = saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0]
if compare_configs(kdata, idata): return True
return False
def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
"""Load logstr into the SDB"""
print logstr
#connect to the db
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfoid
#date='YYYY-MM-DD' <- note speechmarks
#data= your table
#Select NightInfo_Id from NightInfo where Date=$date;
#(get nightinfo_Id)
#Insert into ObsLogTable values ($nightinfo_Id, $data);
logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id=results[0][0]
#check to see if it exists
logic='NightInfo_Id=%i' % night_id
results=saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
if results:
update_cmd="ObsLogTable='%s'" % (logstr)
saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
else:
insert_cmd="NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')
print obsdate, night_id
def checkforflats(sdb,
fid,
caltype,
plist,
instr='rss',
keylist=None,
period=90):
"""Check to see if a bias of the same type already exists in the calibration table
fid: int
FileData_Id
caltype: int
calibration type being checked for
keylist: list
list of RSS header keywords
plist: string
string of RSS headers needed
instr: string
either 'rss' or 'scam'
"""
if instr == 'rss':
caltable = 'RssNightlyCalibration'
logic = 'RssCalibrationType_Id=%i' % caltype
fitstable = 'FitsHeaderRss'
elif instr == 'scam':
caltable = 'SalticamNightlyCalibration'
logic = 'SalticamCalibrationType_Id=%i' % caltype
fitstable = 'FitsHeaderSalticam'
if keylist is not None:
#set the period for to check for the data
try:
utstart = saltmysql.select(sdb, 'UTStart', 'FileData',
'FileData_Id=%i' % fid)[0][0]
except Exception, e:
print e
return False
utstart = utstart - datetime.timedelta(days=period)
#select all the RSS data from this obsdate
cmd_select = 'FileName,FileData_Id, %s' % plist
cmd_table = ''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic = " UTSTART>'%s'" % (utstart)
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#compare results
for r in results:
if compare_configs(keylist[:-1], r[2:-1]):
return True
def updatedq(img, struct, sdb):
"""Add information about the image to the database
"""
#get the filenumber
#check to see if the FileData was created
logic="FileName='%s'" % img
records=saltmysql.select(sdb,'FileData_Id','FileData',logic)
try:
FileData_Id=records[0][0]
except:
message='WARNING: File not yet in database'
print message
return
#get the information from the image
for i in range(1,len(struct)):
if struct[i].name=='SCI':
try:
omean=struct[i].header['OVERSCAN']
except:
omean=None
try:
orms=struct[i].header['OVERRMS']
except:
orms=None
#lets measureme the statistics in a 200x200 box in each image
try:
my,mx=struct[i].data.shape
dx1=int(mx*0.5)
dx2=min(mx,dx1+200)
dy1=int(my*0.5)
dy2=min(my,dy1+200)
mean,med,sig=saltstat.iterstat(struct[i].data[dy1:dy2,dx1:dx2], 5, 5)
except:
mean, med, sig=(None, None, None)
#update the database with this information
#check to see if you need to update or insert
record=saltmysql.select(sdb, 'FileData_Id', 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
update=False
if record: update=True
ins_cmd=''
if omean is not None: ins_cmd='OverscanMean=%s,' % omean
if orms is not None: ins_cmd+='OverscanRms=%s,' % orms
if mean is not None: ins_cmd+='BkgdMean=%f,' % mean
if sig is not None: ins_cmd+='BkgdRms=%f' % sig
if update:
ins_cmd=ins_cmd.rstrip(',')
saltmysql.update(sdb, ins_cmd, 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
else:
ins_cmd+=',FileData_Id=%i, Extension=%i' % (FileData_Id, i)
saltmysql.insert(sdb, ins_cmd, 'PipelineDataQuality_CCD')
def checkforflats(sdb, fid, caltype, plist, instr='rss', keylist=None, period=90):
"""Check to see if a bias of the same type already exists in the calibration table
fid: int
FileData_Id
caltype: int
calibration type being checked for
keylist: list
list of RSS header keywords
plist: string
string of RSS headers needed
instr: string
either 'rss' or 'scam'
"""
if instr=='rss':
caltable='RssNightlyCalibration'
logic='RssCalibrationType_Id=%i' % caltype
fitstable='FitsHeaderRss'
elif instr=='scam':
caltable='SalticamNightlyCalibration'
logic='SalticamCalibrationType_Id=%i' %caltype
fitstable='FitsHeaderSalticam'
if keylist is not None:
#set the period for to check for the data
try:
utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0]
except Exception, e:
print e
return False
utstart=utstart-datetime.timedelta(days=period)
#select all the RSS data from this obsdate
cmd_select='FileName,FileData_Id, %s' % plist
cmd_table=''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic="Proposal_Code like 'CAL_FLAT' and UTSTART>'%s'" % (utstart)
results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#compare results
for r in results:
if compare_configs(keylist[-1], r[2:-1]):
return True
def checkforhrscal(sdb, obsmode, propid, cal_type, period=7):
"""Check the sdb for HRS arcs for a certain mode. Return true if the data have
been taken
fid: int
FileData_Id
obsmode: str
Observation mode
propid: str
Proposal code
cal_type: int
Calibration Type
period: int
number of days in the past to check if data were taken
"""
utstart = datetime.datetime.now()
# set the period for to check for the data
utstart = utstart - datetime.timedelta(days=period)
# select all the HRS data from this obsdate
cmd_select = "FileName,FileData_Id"
cmd_table = """ FileData as d
join ProposalCode using (ProposalCode_Id)
"""
cmd_logic = "Proposal_Code like '%s' and OBSMODE like '%s' and UTSTART>'%s'" % (propid, obsmode, utstart)
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
if len(results) > 0:
return True
# now check the nightly calibrations table
table = "HrsNightlyCalibration join FileData using (FileData_Id)"
logic = 'OBSMODE like "%s" and CalibrationTaken is Null and Ignored=0 and HrsCalibrationType_Id=%i' % (
obsmode,
cal_type,
)
record = saltmysql.select(sdb, "FileData_Id", table, logic)
# if no results return false
if len(record) > 0:
return True
return False
def saltelserror(obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):
# set up
proposers = []
propids = []
pids = []
with logging(logfile,debug) as log:
#open the database
els=saltmysql.connectdb(elshost, elsname, elsuser, elspass)
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)
#create the values for the entire night
nid=saltmysql.getnightinfoid(sdb, obsdate)
stime, etime=saltmysql.select(sdb, 'EveningTwilightEnd,MorningTwilightStart', 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
print stime, etime
errors=gettcserrors(els, stime, etime)
if not errors: return None
for e in errors:
i=e.index('T')
print i, error_list[i], e[-1]
print len(errors)
def movebvitdata(pid, datapath, ftpdir, obsdate, sdb, clobber):
"""sub-routine to move the files to the mirror location for the beachhead
returns status
"""
#log into database and determine username from proposer name
state_select='p.Username'
state_from='Proposal as pr join ProposalCode as c using (ProposalCode_Id) join ProposalContact as pc using (Proposal_Id) join Investigator as i on (pc.Contact_Id=i.Investigator_Id) join PiptUser as p using (PiptUser_Id)'
state_logic="pr.current=1 and c.Proposal_Code='%s'" % pid
record=saltmysql.select(sdb,state_select,state_from,state_logic)
record=set(record)
#check to see if it was successful and raise an error if not
if len(record)<1:
message='SALTFTP--Unable to find username for %s' % pid
raise SaltError(message)
#go through and add the data to each entry in username
for entry in record:
pi_username=entry[0]
ftppath=ftpdir+'/'+pi_username+'/'+pid+'_'+obsdate
#copy the zip file to the beachhead directory
if clobber and os.path.exists(ftppath): shutil.rmtree(ftppath)
#copy the file
shutil.copytree(datapath, ftppath)
message = 'SALTFTP -- %s --> %s' % (datapath, ftppath)
return message
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname,
sdbuser, password):
"""Handle fast data delivery for the proposal. For a given filename
"""
if propcode is None or propcode == 'None': return
#first check in the sdb if fast data delivery is needed
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
select_term = 'Distinct Surname, email, username, ProposalCode_Id'
from_term = '''
Block join Pointing using (Block_Id)
join PipelineConfig using (Pointing_Id)
join ProposalCode using (ProposalCode_Id)
join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id)
join ProposalContact using (ProposalCode_Id)
join Investigator on (Investigator_Id=Contact_Id)
join PiptUser using (PiptUser_Id)
'''
where_term="Proposal_Code like '%s' and DataAccessMethod='Fast'" \
% (propcode)
#print 'Select %s from %s where %s' % (select_term, from_term, where_term)
try:
record = saltmysql.select(sdb, select_term, from_term, where_term)
except Exception, e:
print e
return None
def checkforspst(sdb, fid, keylist, plist, period=7):
"""Check the sdb for SPST observations. Returns true if the data have been taken in the
last period.
fid: int
FileData_Id
dlist: list
header information for a given file
keylist: list
list of RSS header keywords
plist: string
string of RSS headers needed
period: int
number of days in the past to check if data were taken
Note
----
The [:-1] selection in the comparison of the lists is to exclude maskid
"""
caltable='RssNightlyCalibration'
try:
utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0]
except Exception, e:
print e
return False
def checksdbforfits(inname, sdb, log,verbose):
"""
Check to see if the image is in the database
returns FileData_Id, status
"""
FileData_Id=-1
#check to see if the name is the FileName
logic="FileName='%s'" % inname
records=saltmysql.select(sdb,'FileData_Id','FileData',logic)
if not records:
FileData_Id=-1
message='SALTSDBLOADFITS: Adding %s to database' % inname
elif len(records[0])==1 :
FileData_Id=records[0][0]
message='SALTSDBLOADFITS: Updating %s in database' % inname
elif len(records[0])>1:
message='SALTLOADSDBFITS -- ERROR: Ambigous record for %s' % inname
raise SaltError( message)
if verbose:
log.message(message, with_header=False)
return FileData_Id
def seeingtable(sdb, mintime, maxtime):
#extract the seeing data from the sdb
sel_cmd='DateTime, Mass, Dimm'
tab_cmd='MassDimm'
log_cmd="DateTime>'%s' and DateTime<'%s'" % (mintime, maxtime)
see_rec=saltmysql.select(sdb, sel_cmd, tab_cmd, log_cmd)
if len(see_rec)<2: return None
stime_list=[]
mass_arr=np.zeros(len(see_rec))
dimm_arr=np.zeros(len(see_rec))
for i in range(len(see_rec)):
stime_list.append(see_rec[i][0])
mass_arr[i]=see_rec[i][1]
dimm_arr[i]=see_rec[i][2]
seecol=[]
seecol.append(pyfits.Column(name='Timestamp', format='20A', array=stime_list))
seecol.append(pyfits.Column(name='MASS', format='F', array=mass_arr ))
seecol.append(pyfits.Column(name='DIMM', format='F', array=dimm_arr ))
seetab= saltio.fitscolumns(seecol)
seehdu= pyfits.new_table(seetab)
seehdu.name='Seeing'
return seehdu
def erroremail(obsdate, server='', username='', password='', sender='', recipient='', bcc='',
sdbhost='sdb.salt', sdbname='sdb', sdbuser='', sdbpass='', logfile='saltlog.log', verbose=True):
"""Update the PipelineStatistics table with the current information
about the pipeline
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo_id
night_id=saltmysql.getnightinfoid(sdb, obsdate)
#create the message
try:
record=saltmysql.select(sdb, 'PipelineStatus, ErrorMessage', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % night_id)
status, error_message=record[0]
except Exception, e:
raise SaltError('Unable to download information: %s' % e)
#set up the subject and message
subject='Pipeline Error on %s' % obsdate
message="Failure is simply the opportunity to begin again, this time more intelligently.--Henry Ford\n\n"
message+="%s" % error_message
log.message(message, with_stdout=verbose)
#send the email
saltio.email(server,username,password,sender,recipient,bcc, subject,message)
def getweatherdata(els, weathertime, timespan):
"""Creates the weather table from the data in the els
"""
etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200
stime = etime - timespan
# now extact weather information from the els
sel_cmd = "timestamp, air_pressure, dewpoint, rel_humidity, wind_mag_30m, wind_dir_30m, wind_mag_10m, wind_dir_10m, temperatures, rain_detected"
tab_cmd = "bms_external_conditions"
log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime)
wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd)
if len(wea_rec) < 1:
return 8 * [None]
time_list = []
air_arr = np.zeros(len(wea_rec))
dew_arr = np.zeros(len(wea_rec))
hum_arr = np.zeros(len(wea_rec))
w30_arr = np.zeros(len(wea_rec))
w30d_arr = np.zeros(len(wea_rec))
w10_arr = np.zeros(len(wea_rec))
w10d_arr = np.zeros(len(wea_rec))
rain_list = []
t02_arr = np.zeros(len(wea_rec))
t05_arr = np.zeros(len(wea_rec))
t10_arr = np.zeros(len(wea_rec))
t15_arr = np.zeros(len(wea_rec))
t20_arr = np.zeros(len(wea_rec))
t25_arr = np.zeros(len(wea_rec))
t30_arr = np.zeros(len(wea_rec))
for i in range(len(wea_rec)):
time_list.append(str(wea_rec[i][0]))
air_arr[i] = wea_rec[i][1]
dew_arr[i] = wea_rec[i][2]
hum_arr[i] = wea_rec[i][3]
w30_arr[i] = wea_rec[i][4]
w30d_arr[i] = wea_rec[i][5]
w10_arr[i] = wea_rec[i][6]
w10d_arr[i] = wea_rec[i][7]
rain_list.append(wea_rec[i][9])
t_arr = converttemperature(wea_rec[i][8])
t02_arr[i] = t_arr[0]
t05_arr[i] = t_arr[1]
t10_arr[i] = t_arr[2]
t15_arr[i] = t_arr[3]
t20_arr[i] = t_arr[4]
t25_arr[i] = t_arr[5]
t30_arr[i] = t_arr[6]
# average the wind direction by taking the arctan of the average of the sin and cos
wdir = np.degrees(np.arctan2(np.sin(np.radians(w30d_arr)).mean(), np.cos(np.radians(w30d_arr)).mean()))
if wdir < 0:
wdir += 360
# determine if there was any rain during the period
rain = 0
if "T" in rain_list:
rain = 1
return air_arr.mean(), dew_arr.mean(), hum_arr.mean(), t02_arr.mean(), t30_arr.mean(), w30_arr.mean(), wdir, rain
def checksdbforfits(inname, sdb, log, verbose):
"""
Check to see if the image is in the database
returns FileData_Id, status
"""
FileData_Id = -1
#check to see if the name is the FileName
logic = "FileName='%s'" % inname
records = saltmysql.select(sdb, 'FileData_Id', 'FileData', logic)
if not records:
FileData_Id = -1
message = 'SALTSDBLOADFITS: Adding %s to database' % inname
elif len(records[0]) == 1:
FileData_Id = records[0][0]
message = 'SALTSDBLOADFITS: Updating %s in database' % inname
elif len(records[0]) > 1:
message = 'SALTLOADSDBFITS -- ERROR: Ambigous record for %s' % inname
raise SaltError(message)
if verbose:
log.message(message, with_header=False)
return FileData_Id
def getblockid(sdb, night_id, pid, img_list, starttime, endtime):
"""Return the block id for this set of data"""
blockid = -1
targlist = []
for img in img_list:
itime = img[4] + datetime.timedelta(0, 7200.0)
if itime > starttime and itime < endtime:
if img[8] == "OBJECT":
targlist.append(img[2])
for t in set(targlist):
select_state = "Block_Id, BlockVisit_Id, Accepted"
table_state = """Block
join BlockVisit using (Block_Id)
join Pointing using (Block_Id)
join Observation using (Pointing_Id)
join Target using (Target_Id)
join Proposal using (Proposal_Id)
join ProposalCode using (ProposalCode_Id)
"""
logic_state = "NightInfo_Id=%i and Target_Name='%s' and Proposal_Code='%s'" % (night_id, t, pid.strip())
block_list = saltmysql.select(sdb, select_state, table_state, logic_state)
try:
return block_list[0]
except:
return -1, -1, -1
def guidertable(els, mintime, maxtime):
"""Extract the guider data from the els"""
#extract the guider data from the els
sel_cmd='_timestamp_, guidance_available, ee50, mag50'
tab_cmd='tpc_guidance_status__timestamp '
log_cmd="_timestamp_>'%s' and _timestamp_<'%s'" % (mintime, maxtime)
gui_rec=saltmysql.select(els, sel_cmd, tab_cmd, log_cmd)
if len(gui_rec)<2: return None
gtime_list=[]
ee50_arr=np.zeros(len(gui_rec))
mag50_arr=np.zeros(len(gui_rec))
avail_list=[]
for i in range(len(gui_rec)):
gtime_list.append(gui_rec[i][0])
ee50_arr[i]=gui_rec[i][2]
mag50_arr[i]=gui_rec[i][3]
avail_list.append(gui_rec[i][1])
avail_arr=(np.array(avail_list)=='T')
#write the results to a fits table
guicol=[]
guicol.append(pyfits.Column(name='Timestamp', format='20A', array=gtime_list))
guicol.append(pyfits.Column(name='Available', format='L', array=avail_arr ))
guicol.append(pyfits.Column(name='EE50', format='F', array=ee50_arr ))
guicol.append(pyfits.Column(name='mag50', format='F', array=mag50_arr ))
guitab= saltio.fitscolumns(guicol)
guihdu= pyfits.new_table(guitab)
guihdu.name='Guider'
return guihdu
def checkforspst(sdb, fid, keylist, plist, period=7, utstart=None):
"""Check the sdb for SPST observations. Returns true if the data have been taken in the
last period.
fid: int
FileData_Id
keylist: list
list of RSS header keywords
plist: string
string of RSS headers needed
period: int
number of days in the past to check if data were taken
Note
----
The [:-1] selection in the comparison of the lists is to exclude maskid
"""
caltable = 'RssNightlyCalibration'
if utstart is None:
try:
utstart = saltmysql.select(sdb, 'UTStart', 'FileData',
'FileData_Id=%i' % fid)[0][0]
except Exception, e:
print e
return False
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password):
"""Handle fast data delivery for the proposal. For a given filename
"""
if propcode is None or propcode=='None': return
#first check in the sdb if fast data delivery is needed
sdb=saltmysql.connectdb(sdbhost,sdbname, sdbuser, password)
select_term='Distinct Surname, email, username, ProposalCode_Id'
from_term='''
Block join Pointing using (Block_Id)
join PipelineConfig using (Pointing_Id)
join Proposal using (Proposal_Id)
join ProposalCode using (ProposalCode_Id)
join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id)
join ProposalContact using (Proposal_Id)
join Investigator on (Investigator_Id=Contact_Id)
join PiptUser using (PiptUser_Id)
'''
where_term="Proposal_Code like '%s' and current=1 and DataAccessMethod='Fast'" \
% (propcode)
print 'Select %s from %s where %s' % (select_term, from_term, where_term)
try:
record=saltmysql.select(sdb, select_term, from_term, where_term)
except Exception, e:
print e
return None
def checkforhrscal(sdb, obsmode, propid, cal_type, period=7):
"""Check the sdb for HRS arcs for a certain mode. Return true if the data have
been taken
fid: int
FileData_Id
obsmode: str
Observation mode
propid: str
Proposal code
cal_type: int
Calibration Type
period: int
number of days in the past to check if data were taken
"""
utstart = datetime.datetime.now()
#set the period for to check for the data
utstart = utstart - datetime.timedelta(days=period)
#select all the HRS data from this obsdate
cmd_select = 'FileName,FileData_Id'
cmd_table = ''' FileData as d
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic = "Proposal_Code like '%s' and OBSMODE like '%s' and UTSTART>'%s'" % (
propid, obsmode, utstart)
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
if len(results) > 0: return True
#now check the nightly calibrations table
table = 'HrsNightlyCalibration join FileData using (FileData_Id)'
logic = 'OBSMODE like "%s" and CalibrationTaken is Null and Ignored=0 and HrsCalibrationType_Id=%i' % (
obsmode, cal_type)
record = saltmysql.select(sdb, 'FileData_Id', table, logic)
# if no results return false
if len(record) > 0: return True
return False
def tcsstatus(els, start, end):
"""Determine the time spent slewing between two observations
return a list of start of slew time, end of slew time for the times between start and end
"""
select='_timestamp_, tcs_mode'
table='tcs_status__timestamp'
logic="_timestamp_> '%s' and _timestamp_< '%s'" % (start, end)
record=saltmysql.select(els, select, table, logic)
return record
def get_image_info(sdb, filename):
"""Get information about the image from the database"""
select = "fd.FileName, hdr.PROPID, hdr.CCDTYPE, hdr.OBSMODE, hdr.DETMODE, hdr.OBJECT, hdr.TELRA, hdr.TELDEC, hdr.DATE_OBS, hdr.TIME_OBS "
table = (
"FileData as fd join FitsHeaderImage as hdr using (FileData_Id)"
) # join FitsHeaderHrs as hrs using (FileData_Id)'
logic = 'fd.FileName like "%s"' % filename
results = saltmysql.select(sdb, select, table, logic)
return results[0]
def gettcserrors(els, mintime, maxtime):
"""Retrieve data for a given observation during an observation date for
a proposal
"""
#now extact weather information from the els
sel_cmd='*'
tab_cmd='tcs_system_errors__timestamp'
log_cmd="_timestamp_>'%s' and _timestamp_<'%s'" % (mintime, maxtime)
tcs_rec=saltmysql.select(els, sel_cmd, tab_cmd, log_cmd)
return tcs_rec
def checkforbias(sdb, k, instr="rss"):
"""Check to see if a bias of the same type already exists in the calibration table
"""
if instr == "rss":
caltable = "RssNightlyCalibration"
logic = "RssCalibrationType_Id=1"
fitstable = "FitsHeaderRss"
elif instr == "scam":
caltable = "SalticamNightlyCalibration"
logic = "SalticamCalibrationType_Id=1"
fitstable = "FitsHeaderSalticam"
elif instr == "hrs":
caltable = "HrsNightlyCalibration"
logic = "HrsCalibrationType_Id=1"
fitstable = "FitsHeaderHrs"
# get all the bias requests
logic = "CalibrationTaken is Null and Ignored=0 and %s" % logic
record = saltmysql.select(sdb, "FileData_Id", caltable, logic)
# if no results return false
if len(record) < 1:
return False
# hrs only uses one mode
if instr == "hrs":
return True
select = "CCDSUM,GAINSET, ROSPEED"
table = "FileData join %s using (FileData_Id)" % fitstable
logic = "FileData_Id=%i" % k
print select, table, logic
kdata = saltmysql.select(sdb, select, table, logic)[0]
for i in record:
idata = saltmysql.select(sdb, select, table, "FileData_Id=%i" % i)[0]
if compare_configs(kdata, idata):
return True
return False
def dataquality(obsdate, sdb, instrume, clobber=True, logfile=None, verbose=True):
"""Run the data quality measurement for each instrument
"""
if instrume=='rss':
instrumetable='FitsHeaderRss'
elif instrume=='scam':
instrumetable='FitsHeaderSalticam'
#Look for any biases from the night
sel_cmd='FileName'
tab_cmd='FileData join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable
logic_cmd=makelogic("CCDTYPE='BIAS'", obsdate, getnextdate(obsdate))
record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd)
print record
#Look for any flatfields from the night
sel_cmd='FileName'
tab_cmd='FileData join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable
logic_cmd=makelogic("CCDTYPE='FLAT'", obsdate, getnextdate(obsdate))
record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd)
print record
#Look for any Cal_SPST stars
sel_cmd='FileName'
tab_cmd='FileData join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id) join %s using (FileData_Id)' % instrumetable
logic_cmd=makelogic("CCDTYPE='OBJECT' and Proposal_Code='CAL_SPST'", obsdate, getnextdate(obsdate))
record=saltmysql.select(sdb, sel_cmd, tab_cmd, logic_cmd)
print record
#reduce each of the spectrophotometric standards
if len(record):
for infile in record:
profile='/salt/data/%s/%s/%s/product/mbxgp%s' % (obsdate[0:4], obsdate[4:8], instrume, infile[0])
reducespst(profile, obsdate, clobber=clobber, logfile=logfile, verbose=verbose)
def isproprietary(fid, sdb):
"""Determine if a data set is proprietary"""
select='FileData_Id, FileName, Proposal_Code, ReleaseDate'
table='FileData join ProposalCode using (ProposalCode_Id) join Proposal using (ProposalCode_Id)'
logic="FileData_Id='%i' and current=1" % fid
record=saltmysql.select(sdb, select, table, logic)
#if no proposal exists, it isn't proprietary
if not record: return False
#if the release date is less than today, it isn't propietary
releasedate=record[0][3]
if releasedate<date.today(): return False
#default is it is proprietary
return True
def getacdata(els, weathertime, timespan):
"""Determien the AC temperature
weathertime should be SAST
"""
# mktime converts weather time into seconds from UNIX epoch
# 7200 converts to UT
# 2082852000 converts from Unix Epoch to Labview Epoch of 1 Jan 1904
etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200
stime = etime - timespan
# now extact weather information from the els
sel_cmd = "AVG(timestamp), AVG(actual_a_c_temperature)"
tab_cmd = "bms_status"
log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime)
wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd)
if len(wea_rec) < 1:
return None
return wea_rec[0][1]
def getinsidedata(els, weathertime, timespan):
"""Creates the inside data from the els database
Weathertime should be in SAST
"""
# mktime converts weather time into seconds from UNIX epoch
# 7200 converts to UT
# 2082852000 converts from Unix Epoch to Labview Epoch of 1 Jan 1904
etime = time.mktime(weathertime.timetuple()) + 2082852000 - 7200
stime = etime - timespan
# now extact weather information from the els
sel_cmd = "AVG(timestamp), AVG(dew_point), AVG(rel_humidity)"
tab_cmd = "bms_internal_conditions"
log_cmd = "timestamp>'%s' and timestamp<'%s'" % (stime, etime)
wea_rec = saltmysql.select(els, sel_cmd, tab_cmd, log_cmd)
if len(wea_rec) < 1:
return None, None
return wea_rec[0][1], wea_rec[0][2]
def loaddata(fid, filename, sdb, das):
"""Load the data from the sdb to the das database for file with filedata_id given by fid"""
print fid
#load the data in filed data
select="FileData_Id, UTStart, Target_Name, ExposureTime, FileName, INSTRUME, OBSMODE, DETMODE, Proposal_Code, Filter, TelRa, TelDec, FileName, FileSize"
table="FileData join ProposalCode using (ProposalCode_Id)"
if filename.startswith('S'):
table += "join FitsHeaderSalticam using (FileData_Id)"
elif filename.startswith('P'):
table += "join FitsHeaderRss using (FileData_Id)"
logic="FileData_ID=%i" % fid
record = saltmysql.select(sdb, select, table, logic)
print '%s' % record[0][1]
record=record[0]
insert="filedata_id=%i, UTSTART='%s', object='%s', exposuretime=%f, filename='%s', instrume='%s', obsmode='%s', detmode='%s', proposal_id='%s', filter='%s'" % \
(record[0], record[1], record[2], record[3], record[4], record[5], record[6], record[7], record[8], record[9])
insert+=",ra='%s'" % (dec2sex(record[10]/15.0))
insert+=",decl='%s'" % (dec2sex(record[11]))
insert+=",filepath='%s', filesize=%f" % (makepath(filename), record[13])
print insert
def determinetime(sdb, pid, obsdate):
#for a proposal, retrieve the maximum and minimum time
sel_cmd='FileName, UTStart, ExposureTime, NExposures'
tab_cmd='FileData join ProposalCode using (ProposalCode_Id)'
log_cmd="FileName like '%"+str(obsdate)+"%' and Proposal_Code='"+pid+"'"
record=saltmysql.select(sdb, sel_cmd, tab_cmd, log_cmd)
if not record:
raise SaltError('%s does not have any data on %s' % (pid, obsdate))
print pid, record
maxtime=record[0][1]
mintime=record[0][1]
for r in record:
maxtime=max(maxtime, r[1]+datetime.timedelta(seconds=r[2]*r[3]))
mintime=min(mintime, r[1])
#correct the time from UTC to SAST
mintime+=datetime.timedelta(seconds=2*3600.0)
maxtime+=datetime.timedelta(seconds=2*3600.0)
return mintime, maxtime
def saltdasload(obsdate, sdbhost, sdbname, sdbuser, password, dashost, dasname, dasuser, daspassword, logfile, verbose):
"""Upload data from the science database to the SALTDAS database
"""
with logging(logfile,debug) as log:
#open the science database
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password)
#open saltdas
das=saltmysql.connectdb(dashost,dasname,dasuser,daspassword)
select='FileData_Id, FileName, Proposal_Code'
table='FileData join ProposalCode using (ProposalCode_Id)'
logic="FileName like '%"+str(obsdate)+"%'"
record=saltmysql.select(sdb, select, table, logic)
for r in record[:]:
if not isproprietary(r[0], sdb):
print r
loaddata(r[0], r[1], sdb, das)
return
def findpropinfo(pid,sdb):
"""query the Sdb server and determine the email and surname
for the contact information for the proposal
return a dictionary with Surname,Email
"""
propinfo={}
#setup the the query
state_select='pr.Proposal_Id,c.Proposal_Code,i2.Surname,i2.Email'
state_from='''
Investigator as i join PiptUser using (PiptUser_Id) join Investigator as i2 on (PiptUser.Investigator_Id=i2.Investigator_Id),
Proposal as pr
join ProposalCode as c using (ProposalCode_Id)
join ProposalContact as pc using (Proposal_Id) '''
state_logic="i.Investigator_Id=pc.Contact_ID and pc.Proposal_ID=pr.Proposal_ID and pr.current=1 and c.Proposal_Code='%s'" %pid
# left over by intersting way to call it: ORDER BY pr.Proposal_ID DESC" % pid
record=saltmysql.select(sdb,state_select,state_from,state_logic)
if len(record)<1:
message = pid + ' is not in the Science database'
raise SaltError(message)
elif len(record)==1:
surname=[record[0][2]]
email=[record[0][3]]
propinfo[pid]=(surname,email)
else:
prop_id=-1
surname=[]
email=[]
for entry in record:
if entry[0] > prop_id:
prop_id=entry[0]
prop_code=entry[1]
surname.append(entry[2])
email.append(entry[3])
propinfo[pid]=(surname,email)
return propinfo
def checkforfast(pid, target, db):
"""Check the database to see if fast response is needed
returns status"""
record=''
#log into database and determine response type of proposal
#If the object is not in the proposal, and there is a request for a fast response
#for other objects in the proposal, then it should responsd with a fast response.
state_select='T.Target_name, pdam.DataAccessMethod'
state_from="""
Proposal
join ProposalCode using (ProposalCode_Id)
join Block using (Proposal_Id)
join Pointing using (Block_Id)
join Observation using (Pointing_Id)
join Target as T using (Target_Id)
join PipelineConfig using (Pointing_Id)
join PipelineDataAccessMethod as pdam using (PipelineDataAccessMethod_Id)
"""
state_logic="Current=1"
state_logic +=" and Proposal_Code='%s'" % pid
#state_logic +=" and pr.Proposal_Code='%s' and T.Target_Name like '%s' " % (pid, target)
#print "select %s from %s whre %s" % (state_select,state_from,state_logic)
record=saltmysql.select(db,state_select,state_from,state_logic)
record=set(record)
#if there are no records in the set
if len(record)==0: return False
#TODO: Add a check to make sure that
for t,m in record:
if m.upper()=='FAST' and t in target: return True
return False
def findcal(obsdate, sdbhost, sdbname, sdbuser, password):
"""Find all of the unique configuration settings used on a night
and insert it into the database
obsdate--Observation date in YYYYMMDD
sdbhost--host name for sdb
sdbname--name of the sdb
sdbuser--user for the sdb
password--sdb password
"""
#connect to the db
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo id
logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id = results[0][0]
#select all the scam data from this obsdate
cmd_select = 'd.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER'
cmd_table = ''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderSalticam using (FileData_Id)
'''
cmd_logic = "d.FileName like 'S" + obsdate + "%' and f.CCDTYPE='OBJECT'"
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#loop through all the results and return only the Set of identical results
caldict = create_caldict(results)
#insert the results into the database
for k in caldict:
#first check to see if it has already been entered
record = saltmysql.select(sdb, 'FileData_Id',
'SalticamNightlyCalibration',
'FileData_Id=%i' % k)
if len(record) < 1:
#check for block_id
blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
'FileData_Id=%i' % k)[0][0]
#get the calibration types requested
if blockid:
request = saltmysql.select(sdb, 'SalticamCalibrationType_Id',
'SalitcamCalibration',
'Block_Id=%i' % blockid)
for cid in request:
cid = cid[0]
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % (
night_id, k, cid)
#saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration')
print k, " ".join([str(k) for k in caldict[k]])
#list of rss calibration types
#+-----------------------+----------------------------------+
#| RssCalibrationType_Id | CalibrationType |
#+-----------------------+----------------------------------+
#| 2 | Arc |
#| 1 | Bias |
#| 14 | Imaging flat - Lamp |
#| 15 | Imaging flat - Twilight |
#| 3 | Spectroscopic flat - Lamp |
#| 4 | Spectroscopic flat - Twilight |
#| 12 | Standard - Circular polarimetric |
#| 13 | Standard - Lick |
#| 9 | Standard - Linear polarimetric |
#| 5 | Standard - Photometric |
#| 8 | Standard - RV |
#| 11 | Standard - Smooth spectrum |
#| 7 | Standard - Spectrophotometric |
#| 6 | Standard - Spectroscopic |
#| 10 | Standard - Unpolarised |
#+-----------------------+----------------------------------+
#select all the RSS data from this obsdate
rssheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GRTILT, AR_STA, MASKID'
cmd_select = 'd.FileName,d.FileData_Id, %s' % rssheaderlist
# CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
cmd_table = ''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic = "d.FileName like 'P" + obsdate + "%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST' and GRTILT > 0"
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#loop through all the results and return only the Set of identical results
caldict = create_caldict(results)
#insert the rss results into the database
for k in caldict:
#first check to see if it has already been entered
record = saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration',
'FileData_Id=%i' % k)
if len(record) < 1:
#period for checking for SPST. If the uses requests this, it gets set to
# 7 days, but the default is taken within the last month for non-requests
period = 30
#check for block_id
blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
'FileData_Id=%i' % k)[0][0]
#get the calibration types requested
if blockid:
request = saltmysql.select(sdb, 'RssCalibrationType_Id',
'RssCalibration',
'Block_Id=%i' % blockid)
for cid in request:
cid = cid[0]
#check to see if the request is already in the database or if a similar request has
#been taken recently
if cid == 1: #bias
if not checkforbias(sdb, k):
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
night_id, k, cid)
saltmysql.insert(sdb, cmd_insert,
'RssNightlyCalibration')
elif cid in [3, 4]: #flats
if not checkforflats(sdb,
k,
cid,
rssheaderlist,
instr='rss',
keylist=caldict[k],
period=90):
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
night_id, k, cid)
saltmysql.insert(sdb, cmd_insert,
'RssNightlyCalibration')
elif cid == 7: #specstand
period = 7
#print period, k, caldict[k]
if not checkforspst(
sdb, k, caldict[k], rssheaderlist,
period=period):
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % (
night_id, k)
saltmysql.insert(sdb, cmd_insert,
'RssNightlyCalibration')
else:
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (
night_id, k, cid)
saltmysql.insert(sdb, cmd_insert,
'RssNightlyCalibration')
print k, cid, " ".join([str(w) for w in caldict[k]])
#select all the HRS data from this obsdate
#+-----------------------+-------------------------------+
#| HrsCalibrationType_Id | CalibrationType |
#+-----------------------+-------------------------------+
#| 3 | Arc - Calsys |
#| 2 | Arc - Internal |
#| 1 | Bias |
#| 4 | Spectroscopic flat - Lamp |
#| 5 | Spectroscopic flat - Twilight |
#| 11 | Standard - Lick |
#| 6 | Standard - Photometric |
#| 9 | Standard - RV |
#| 10 | Standard - Smooth spectrum |
#| 8 | Standard - Spectrophotometric |
#| 7 | Standard - Spectroscopic |
#+-----------------------+-------------------------------+
hrsheaderlist = 'f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED'
cmd_select = 'd.FileName,d.FileData_Id, %s' % hrsheaderlist
cmd_table = ''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderHrs using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic = "d.FileName like 'H" + obsdate + "%' and CCDTYPE='Science' and Proposal_Code not like 'CAL_SPST'"
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#loop through all the results and return only the Set of identical results
caldict = create_caldict(results)
for k in caldict:
#first check to see if it has already been entered
record = saltmysql.select(sdb, 'FileData_Id', 'HrsNightlyCalibration',
'FileData_Id=%i' % k)
if len(record) < 1:
#period for checking for SPST. If the uses requests this, it gets set to
# 7 days, but the default is taken within the last month for non-requests
period = 30
#check for block_id
blockid = saltmysql.select(sdb, 'Block_Id', 'FileData',
'FileData_Id=%i' % k)[0][0]
#get the calibration types requested
if not checkforbias(sdb, k, instr='hrs'):
cmd_insert = 'NightInfo_Id=%i, FileData_Id=%i, HrsCalibrationType_Id=%i' % (
night_id, k, 1)
saltmysql.insert(sdb, cmd_insert, 'HrsNightlyCalibration')
return
def checkforspst(sdb, fid, keylist, plist, period=7):
"""Check the sdb for SPST observations. Returns true if the data have been taken in the
last period.
fid: int
FileData_Id
keylist: list
list of RSS header keywords
plist: string
string of RSS headers needed
period: int
number of days in the past to check if data were taken
Note
----
The [:-1] selection in the comparison of the lists is to exclude maskid
"""
caltable='RssNightlyCalibration'
try:
utstart=saltmysql.select(sdb, 'UTStart', 'FileData', 'FileData_Id=%i' % fid)[0][0]
except Exception as e:
print(e)
return False
#set the period for to check for the data
utstart=utstart-datetime.timedelta(days=period)
#select all the RSS data from this obsdate
cmd_select='FileName,FileData_Id, %s' % plist
cmd_table=''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic="Proposal_Code like 'CAL_SPST' and UTSTART>'%s'" % (utstart)
results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#compare results
for r in results:
if compare_configs(keylist[:-1], r[2:-1]):
return True
#now check the nightly calibrations table
logic='CalibrationTaken is Null and Ignored=0 and RssCalibrationType_Id=7'
record=saltmysql.select(sdb, 'FileData_Id', caltable, logic)
# if no results return false
if len(record)<1: return False
#selection the data from the list and compare results
cmd_select='FileName,FileData_Id, %s' % plist
cmd_table=''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
for i in record:
cmd_logic='FileData_Id=%i' % i
r=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)[0]
if compare_configs(keylist[:-1], r[2:-1]):
return True
return False
def findcal(obsdate, sdbhost, sdbname, sdbuser, password):
"""Find all of the unique configuration settings used on a night
and insert it into the database
obsdate--Observation date in YYYYMMDD
sdbhost--host name for sdb
sdbname--name of the sdb
sdbuser--user for the sdb
password--sdb password
"""
#connect to the db
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo id
logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id=results[0][0]
#select all the scam data from this obsdate
cmd_select='d.FileName, d.FileData_Id, f.CCDTYPE, d.DETMODE, CCDSUM, GAINSET, ROSPEED, FILTER'
cmd_table=''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderSalticam using (FileData_Id)
'''
cmd_logic="d.FileName like 'S" + obsdate+"%' and f.CCDTYPE='OBJECT'"
results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#loop through all the results and return only the Set of identical results
caldict=create_caldict(results)
#insert the results into the database
for k in caldict:
#first check to see if it has already been entered
record=saltmysql.select(sdb, 'FileData_Id', 'SalticamNightlyCalibration', 'FileData_Id=%i' % k)
if len(record)<1:
#check for block_id
blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0]
#get the calibration types requested
if blockid:
request=saltmysql.select(sdb, 'SalticamCalibrationType_Id', 'SalitcamCalibration', 'Block_Id=%i' % blockid)
for cid in request:
cid=cid[0]
cmd_insert='NightInfo_Id=%i, FileData_Id=%i, SatlicamCalibrationType_Id=%i' % (night_id, k, cid)
#saltmysql.insert(sdb, cmd_insert, 'SalitcamNightlyCalibration')
print(k, " ".join([str(k) for k in caldict[k]]))
#list of rss calibration types
#+-----------------------+----------------------------------+
#| RssCalibrationType_Id | CalibrationType |
#+-----------------------+----------------------------------+
#| 2 | Arc |
#| 1 | Bias |
#| 14 | Imaging flat - Lamp |
#| 15 | Imaging flat - Twilight |
#| 3 | Spectroscopic flat - Lamp |
#| 4 | Spectroscopic flat - Twilight |
#| 12 | Standard - Circular polarimetric |
#| 13 | Standard - Lick |
#| 9 | Standard - Linear polarimetric |
#| 5 | Standard - Photometric |
#| 8 | Standard - RV |
#| 11 | Standard - Smooth spectrum |
#| 7 | Standard - Spectrophotometric |
#| 6 | Standard - Spectroscopic |
#| 10 | Standard - Unpolarised |
#+-----------------------+----------------------------------+
#select all the RSS data from this obsdate
rssheaderlist='f.CCDTYPE, d.DETMODE, d.OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
cmd_select='d.FileName,d.FileData_Id, %s' % rssheaderlist
# CCDTYPE, DETMODE, OBSMODE, CCDSUM, GAINSET, ROSPEED, FILTER, GRATING, GR_STA, AR_STA, MASKID'
cmd_table=''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic="d.FileName like 'P" + obsdate+"%' and CCDTYPE='OBJECT' and Proposal_Code not like 'CAL_SPST'"
results=saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#loop through all the results and return only the Set of identical results
caldict=create_caldict(results)
#insert the rss results into the database
for k in caldict:
#first check to see if it has already been entered
record=saltmysql.select(sdb, 'FileData_Id', 'RssNightlyCalibration', 'FileData_Id=%i' % k)
if len(record)<1:
#period for checking for SPST. If the uses requests this, it gets set to
# 7 days, but the default is taken within the last month for non-requests
period=30
#check for block_id
blockid=saltmysql.select(sdb, 'Block_Id', 'FileData', 'FileData_Id=%i' % k)[0][0]
#get the calibration types requested
if blockid:
request=saltmysql.select(sdb, 'RssCalibrationType_Id', 'RssCalibration', 'Block_Id=%i' % blockid)
for cid in request:
cid=cid[0]
#check to see if the request is already in the database or if a similar request has
#been taken recently
if cid==1: #bias
if not checkforbias(sdb, k):
cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
elif cid in [3,4]: #flats
if not checkforflats(sdb, k, cid, rssheaderlist, instr='rss', keylist=caldict[k], period=90):
cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
elif cid==7: #specstand
period=7
#print period, k, caldict[k]
if not checkforspst(sdb, k, caldict[k], rssheaderlist, period=period):
cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=7' % (night_id, k)
saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
else:
cmd_insert='NightInfo_Id=%i, FileData_Id=%i, RssCalibrationType_Id=%i' % (night_id, k, cid)
saltmysql.insert(sdb, cmd_insert, 'RssNightlyCalibration')
print(k, cid," ".join([str(w) for w in caldict[k]]))
return
cmd_table = ''' FileData as d
left join FitsHeaderImage as f using (FileData_Id)
left join FitsHeaderRss using (FileData_Id)
join ProposalCode using (ProposalCode_Id)
'''
cmd_logic = " UTSTART>'%s'" % (utstart)
results = saltmysql.select(sdb, cmd_select, cmd_table, cmd_logic)
#compare results
for r in results:
if compare_configs(keylist[:-1], r[2:-1]):
return True
#get all the flat requests
logic = 'CalibrationTaken is Null and Ignored=0 and %s' % logic
record = saltmysql.select(sdb, 'FileData_Id', caltable, logic)
# if no results return false
if len(record) < 1: return False
select = plist
table = 'FileData as d join FitsHeaderImage as f using (FileData_Id) join %s using (FileData_Id)' % fitstable
logic = 'FileData_Id=%i' % fid
#select
kdata = saltmysql.select(sdb, select, table, logic)[0]
for i in record:
idata = saltmysql.select(sdb, select, table, 'FileData_Id=%i' % i)[0]
if compare_configs(kdata, idata): return True
return False
except Exception, e:
print e
#check to see if an email has been sent
select_term = 'PipelineStatus'
from_term = '''
PipelineProposalStatistics
join PipelineStatus using (PipelineStatus_Id)
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
where_term = "Proposal_Code like '%s' and Date='%s-%s-%s'" % (
propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
print select_term, from_term, where_term
try:
record = saltmysql.select(sdb, select_term, from_term,
where_term)[0][0]
except:
record = None
print record
if record == 'FastEmail':
return
else:
#insert information into the database
nightinfoid = saltmysql.getnightinfoid(sdb, obsdate)
insert_term = "NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (
nightinfoid, propid)
table_term = "PipelineProposalStatistics"
saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")
#send email
