def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
"""Load logstr into the SDB"""
print logstr
#connect to the db
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfoid
#date='YYYY-MM-DD' <- note speechmarks
#data= your table
#Select NightInfo_Id from NightInfo where Date=$date;
#(get nightinfo_Id)
#Insert into ObsLogTable values ($nightinfo_Id, $data);
logic = "Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results = saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id = results[0][0]
#check to see if it exists
logic = 'NightInfo_Id=%i' % night_id
results = saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
if results:
update_cmd = "ObsLogTable='%s'" % (logstr)
saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
else:
insert_cmd = "NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')
print obsdate, night_id
def sdbloadobslog(logstr, obsdate, sdbhost, sdbname, sdbuser, password):
"""Load logstr into the SDB"""
print logstr
#connect to the db
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfoid
#date='YYYY-MM-DD' <- note speechmarks
#data= your table
#Select NightInfo_Id from NightInfo where Date=$date;
#(get nightinfo_Id)
#Insert into ObsLogTable values ($nightinfo_Id, $data);
logic="Date='%4s-%2s-%2s'" % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
results=saltmysql.select(sdb, 'NightInfo_Id', 'NightInfo', logic)
night_id=results[0][0]
#check to see if it exists
logic='NightInfo_Id=%i' % night_id
results=saltmysql.select(sdb, 'NightInfo_Id', 'ObsLogTable', logic)
if results:
update_cmd="ObsLogTable='%s'" % (logstr)
saltmysql.update(sdb, update_cmd, 'ObsLogTable', logic)
else:
insert_cmd="NightInfo_Id=%i, ObsLogTable='%s'" % (night_id, logstr)
saltmysql.insert(sdb, insert_cmd, 'ObsLogTable')
print obsdate, night_id
def updatedq(img, struct, sdb):
"""Add information about the image to the database
"""
#get the filenumber
#check to see if the FileData was created
logic="FileName='%s'" % img
records=saltmysql.select(sdb,'FileData_Id','FileData',logic)
try:
FileData_Id=records[0][0]
except:
message='WARNING: File not yet in database'
print message
return
#get the information from the image
for i in range(1,len(struct)):
if struct[i].name=='SCI':
try:
omean=struct[i].header['OVERSCAN']
except:
omean=None
try:
orms=struct[i].header['OVERRMS']
except:
orms=None
#lets measureme the statistics in a 200x200 box in each image
try:
my,mx=struct[i].data.shape
dx1=int(mx*0.5)
dx2=min(mx,dx1+200)
dy1=int(my*0.5)
dy2=min(my,dy1+200)
mean,med,sig=saltstat.iterstat(struct[i].data[dy1:dy2,dx1:dx2], 5, 5)
except:
mean, med, sig=(None, None, None)
#update the database with this information
#check to see if you need to update or insert
record=saltmysql.select(sdb, 'FileData_Id', 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
update=False
if record: update=True
ins_cmd=''
if omean is not None: ins_cmd='OverscanMean=%s,' % omean
if orms is not None: ins_cmd+='OverscanRms=%s,' % orms
if mean is not None: ins_cmd+='BkgdMean=%f,' % mean
if sig is not None: ins_cmd+='BkgdRms=%f' % sig
if update:
ins_cmd=ins_cmd.rstrip(',')
saltmysql.update(sdb, ins_cmd, 'PipelineDataQuality_CCD', 'FileData_Id=%i and Extension=%i' % (FileData_Id, i))
else:
ins_cmd+=',FileData_Id=%i, Extension=%i' % (FileData_Id, i)
saltmysql.insert(sdb, ins_cmd, 'PipelineDataQuality_CCD')
def pipelinestatus(obsdate, status, message=None, rawsize=None, reducedsize=None,
runtime=None, emailsent=None, sdbhost='sdb.saao', sdbname='sdb', sdbuser='', password='',
logfile='saltlog.log', verbose=True):
"""Update the PipelineStatistics table with the current information
about the pipeline
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo_id
night_id=saltmysql.getnightinfoid(sdb, obsdate)
#get the status_id for the given status
status_id=getstatusid(sdb, status)
#create the insert command
obsdate=str(obsdate)
inst_cmd="NightInfo_Id=%s, PipelineStatus_Id=%i" % (night_id, status_id)
if status_id>10:
inst_cmd+=',ErrorMessage="%s"' % message
if rawsize is not None:
inst_cmd+=",RawSize=%f" % rawsize
if reducedsize is not None:
inst_cmd+=",ReducedSize=%f" % rawsize
if runtime is not None:
inst_cmd+=",PipelineRunTime=%i" % runtime
if emailsent is not None:
inst_cmd+=",EmailSent=%i" % emailsent
print inst_cmd
#insert or update the pipeline
if checktable(sdb, night_id):
saltmysql.update(sdb, inst_cmd, 'PipelineStatistics', 'NightInfo_Id=%i' % night_id)
msg="Updating information for Night_Id=%i\n" % night_id
else:
saltmysql.insert(sdb, inst_cmd, 'PipelineStatistics')
msg="Inserting information for Night_Id=%i\n" % night_id
#log the call
log.message(msg+inst_cmd, with_stdout=verbose)
def updatenightinfo(obsdate, sdbhost='sdb.saao', sdbname='sdb', \
sdbuser='', password='', logfile='saltlog.log', verbose=True):
"""Update the nightinfo table with current information about the
night
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo_id
try:
night_id=saltmysql.getnightinfoid(sdb, obsdate)
except SaltError:
night_id=None
#get the information for the night
time_list=get_nightdetails(obsdate)
#create the insert command
obsdate=str(obsdate)
inst_cmd="Date='%s-%s-%s'," % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
inst_cmd+="SunSet='%s', SunRise='%s', MoonSet='%s', MoonRise='%s', EveningTwilightEnd='%s', MorningTwilightStart='%s'" % \
(time_list[0], time_list[1], time_list[2], time_list[3], time_list[4], time_list[5])
inst_cmd+=",MoonPhase_Percent=%i" % (round(float(time_list[6])))
if night_id:
saltmysql.update(sdb, inst_cmd, 'NightInfo', 'NightInfo_Id=%i' % night_id)
msg="Updating information for Night_Id=%i\n" % night_id
else:
saltmysql.insert(sdb, inst_cmd, 'NightInfo')
msg="Inserting information for Night_Id=%i\n" % night_id
#log the call
log.message(msg+inst_cmd, with_stdout=verbose)
def saltcharge(
obsdate,
outfile,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Calculate the charged time for proposals observed during a night
"""
with logging(logfile, debug) as log:
# check the outfiles
if not saltio.checkfornone(outfile):
outfile = None
# check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
# connect the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# get all the proposals observed during the night
select_state = "distinct Proposal_Code"
table_state = "FileData Join ProposalCode using (ProposalCode_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
records = saltmysql.select(sdb, select_state, table_state, logic_state)
pids = []
pid_time = {}
for r in records:
pids.append(r[0])
pid_time[r[0]] = 0
if len(pids) == 0:
message = "There are no proposal to charge time for %s" % obsdate
log.message(message)
return
# get the nightinfo_id
night_id = saltmysql.getnightinfoid(sdb, obsdate)
print night_id
# get a list of all the images taken during the night
select_state = "FileName, Proposal_Code, Target_Name, ExposureTime, UTSTART, INSTRUME, OBSMODE, DETMODE, CCDTYPE, NExposures"
table_state = "FileData Join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
img_list = saltmysql.select(sdb, select_state, table_state, logic_state)
# get all the blocks visited
select_state = "Block_Id, Accepted, Proposal_Code"
table_state = "Block join BlockVisit using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)"
logic_state = "NightInfo_Id=%i" % night_id
block_list = saltmysql.select(sdb, select_state, table_state, logic_state)
print block_list
# get the start and end of twilight
nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
print nightstart, nightend, (nightend - nightstart).seconds
print
# download the SOMMI log from the night
try:
sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
except Exception, e:
msg = "Unable to read in SOMMI log for %s" % obsdate
raise SaltError(msg)
# add to account for error in SOMMI log
if int(obsdate) < 20111027:
sommi_log = sommi_log.replace("Object name", "\nObject name")
# parse the sommi log
point_list = parseforpointings(sommi_log)
# now find all blocks observed during a night
for point in point_list:
# proposal for that block
pid = point[0].strip()
# start time for that block
starttime = point[1]
# find the end time for that block
endtime = findnexttime(point[1], point_list, nightend)
# find the start time of the first object to be observed
# for this block
startimage = findfirstimage(pid, starttime, img_list)
lastimage = findlastimage(pid, endtime, img_list)
# check to see if the end time for the last file observed for
# this block
if startimage is not None:
if startimage > endtime:
startimage = None
# determine the acquisition time for the block
if startimage is not None:
acqdelta = (startimage - starttime).seconds
else:
acqdelta = -1
# find the shutter open time
shuttertime = calculate_shutteropen(img_list, starttime, endtime)
# if the shutter and time of the last image is substantially different from the
# end of the block, then take the last image as the end of the block
# *TODO* Change to use expected block time
st = starttime + datetime.timedelta(0, shuttertime + acqdelta)
if (endtime - st).seconds > 900:
if lastimage is None:
endtime = st
elif (lastimage - st).seconds > 3600:
endtime = st
else:
endtime = lastimage
# account for the time for that block
tblock = (endtime - starttime).seconds
if acqdelta > -1:
# find the associated block
block_id, blockvisit_id, accepted = getblockid(sdb, night_id, pid, img_list, starttime, endtime)
if accepted and pid.count("-3-"):
# charged time for that block
try:
pid_time[pid] += tblock
except KeyError:
pid_time[pid] = tblock
print block_id, blockvisit_id, pid, starttime, tblock, acqdelta, shuttertime, shuttertime / tblock, block_id, accepted
# update the charge time
slewdelta = 0
scidelta = tblock - acqdelta
update_cmd = "TotalSlewTime=%i, TotalAcquisitionTime=%i, TotalScienceTime=%i" % (
slewdelta,
acqdelta,
scidelta,
)
table_cmd = "BlockVisit"
logic_cmd = "BlockVisit_Id=%i" % blockvisit_id
saltmysql.update(sdb, update_cmd, table_cmd, logic_cmd)
return
# update the charged time information
# TODO: Check to see if the block was approved
ptime_tot = 0
stime_tot = 0
for k in pid_time:
ptime = pid_time[k]
if ptime > 0:
obsratio = stime / ptime
else:
obsratio = 0
print "%25s %5i %5i %3.2f" % (k, ptime, stime, obsratio)
if k.count("-3-"):
ptime_tot += ptime
stime_tot += stime
# print out the total night statistics
tdelta = nightend - nightstart
print tdelta.seconds, ptime_tot, stime_tot
def saltnightinfo(
obsdate,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Update the nightinfo table from the SOMMI log
"""
with logging(logfile, debug) as log:
# connect the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# get the nightinfo_id
night_id = saltmysql.getnightinfoid(sdb, obsdate)
print night_id
# get the start and end of twilight
nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
print nightstart
print nightend
print (nightend - nightstart).seconds
# download the SOMMI log from the night
try:
sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
except:
raise SaltError("Unable to read SOMMI log in the database for %s" % obsdate)
# set up the time for the night
try:
ntime = (nightend - nightstart).seconds
except:
raise SaltError("Unable to read night length from database for %s" % obsdate)
# parse the sommi log
slog = sommi_log.split("\n")
stime = 0
for i in range(len(slog)):
if slog[i].count("Science Time:"):
stime = extractinformation(slog[i])
if slog[i].count("Engineering Time:") and not slog[i].count("Non-observing Engineering Time"):
etime = extractinformation(slog[i])
if slog[i].count("Time lost to Weather:"):
wtime = extractinformation(slog[i])
if slog[i].count("Time lost to Tech. Problems:"):
ttime = extractinformation(slog[i])
if slog[i].count("Non-observing Engineering Time:"):
ltime = extractinformation(slog[i])
print etime
tot_time = stime + etime + wtime + ttime
print night_id, ntime, stime, etime, wtime, ttime, ltime, tot_time
# insert the information into the database
print tot_time, ntime
if abs(tot_time - ntime) < 900:
message = "Updating NightInfo Table with the following Times:\n"
message += "Science Time=%i\n" % stime
message += "Engineeringh=%i\n" % etime
message += "Time lost to Weather=%i\n" % wtime
message += "Time lost to Tech. Problems=%i\n" % ttime
message += "Non-observing Engineering Time=%i\n" % ltime
log.message(message)
insert_state = (
"ScienceTime=%i, EngineeringTime=%i, TimeLostToWeather=%i, TimeLostToProblems=%i, NonObservingEngineeringTime=%i"
% (stime, etime, wtime, ttime, ltime)
)
table_state = "NightInfo"
logic_state = "NightInfo_Id=%i" % night_id
saltmysql.update(sdb, insert_state, table_state, logic_state)
else:
message = "The total time for the night is not equal to the length of the night\n"
message += "Night Length=%i\n--------------------\n" % ntime
message += "Science Time=%i\n" % stime
message += "Engineeringh=%i\n" % etime
message += "Time lost to Weather=%i\n" % wtime
message += "Time lost to Tech. Problems=%i\n" % ttime
message += "Non-observing Engineering Time=%i\n" % ltime
message += "Total time for the Night=%i\n" % tot_time
log.message(message)