def saltelserror(obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):
# set up
proposers = []
propids = []
pids = []
with logging(logfile,debug) as log:
#open the database
els=saltmysql.connectdb(elshost, elsname, elsuser, elspass)
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)
#create the values for the entire night
nid=saltmysql.getnightinfoid(sdb, obsdate)
stime, etime=saltmysql.select(sdb, 'EveningTwilightEnd,MorningTwilightStart', 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
print stime, etime
errors=gettcserrors(els, stime, etime)
if not errors: return None
for e in errors:
i=e.index('T')
print i, error_list[i], e[-1]
print len(errors)
def erroremail(obsdate, server='', username='', password='', sender='', recipient='', bcc='',
sdbhost='sdb.salt', sdbname='sdb', sdbuser='', sdbpass='', logfile='saltlog.log', verbose=True):
"""Update the PipelineStatistics table with the current information
about the pipeline
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo_id
night_id=saltmysql.getnightinfoid(sdb, obsdate)
#create the message
try:
record=saltmysql.select(sdb, 'PipelineStatus, ErrorMessage', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % night_id)
status, error_message=record[0]
except Exception, e:
raise SaltError('Unable to download information: %s' % e)
#set up the subject and message
subject='Pipeline Error on %s' % obsdate
message="Failure is simply the opportunity to begin again, this time more intelligently.--Henry Ford\n\n"
message+="%s" % error_message
log.message(message, with_stdout=verbose)
#send the email
saltio.email(server,username,password,sender,recipient,bcc, subject,message)
def saltweather(weathertime, timespan, elshost, elsname, elsuser, elspass, sdbhost, sdbname, sdbuser, password):
print weathertime
# open the database
els = saltmysql.connectdb(elshost, elsname, elsuser, elspass)
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# determine the obsdate
obsdate = weathertime - datetime.timedelta(seconds=43200)
obsdate = "%i%s%s" % (obsdate.year, string.zfill(obsdate.month, 2), string.zfill(obsdate.day, 2))
nid = saltmysql.getnightinfoid(sdb, obsdate)
print nid
# get the most recent weather data
airpressure, dewout, extrelhum, temp2m, temp30m, windspeed, winddir, rain = getweatherdata(
els, weathertime, timespan
)
dewin, relhum = getinsidedata(els, weathertime, timespan)
tempin = getacdata(els, weathertime, timespan)
# upload that to the sdb
upcmd = "Weather_Time='%s', NightInfo_Id=%i" % (weathertime, nid)
if tempin is not None:
upcmd += ",TemperatureInside=%4.2f" % tempin
if temp2m is not None:
upcmd += ",Temperature2m=%4.2f" % temp2m
if temp30m is not None:
upcmd += ",Temperature30m=%4.2f" % temp30m
if windspeed is not None:
upcmd += ",WindSpeed=%5.2f" % windspeed
if winddir is not None:
upcmd += ",WindDirection=%5.2f" % winddir
if dewin is not None:
upcmd += ",DewPointInside=%3.2f" % dewin
if dewout is not None:
upcmd += ",DewPointOutside=%3.2f" % dewout
if airpressure is not None:
upcmd += ",AirPressure=%4.2f" % airpressure
if relhum is not None:
upcmd += ",RelativeHumidty=%4.2f" % relhum
if extrelhum is not None:
upcmd += ",ExternalRelativeHumidity=%4.2f" % extrelhum
if rain is not None:
upcmd += ",Rain=%i" % rain
saltmysql.insert(sdb, upcmd, "Weather")
print upcmd
return
def pipelinestatus(obsdate, status, message=None, rawsize=None, reducedsize=None,
runtime=None, emailsent=None, sdbhost='sdb.saao', sdbname='sdb', sdbuser='', password='',
logfile='saltlog.log', verbose=True):
"""Update the PipelineStatistics table with the current information
about the pipeline
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo_id
night_id=saltmysql.getnightinfoid(sdb, obsdate)
#get the status_id for the given status
status_id=getstatusid(sdb, status)
#create the insert command
obsdate=str(obsdate)
inst_cmd="NightInfo_Id=%s, PipelineStatus_Id=%i" % (night_id, status_id)
if status_id>10:
inst_cmd+=',ErrorMessage="%s"' % message
if rawsize is not None:
inst_cmd+=",RawSize=%f" % rawsize
if reducedsize is not None:
inst_cmd+=",ReducedSize=%f" % rawsize
if runtime is not None:
inst_cmd+=",PipelineRunTime=%i" % runtime
if emailsent is not None:
inst_cmd+=",EmailSent=%i" % emailsent
print inst_cmd
#insert or update the pipeline
if checktable(sdb, night_id):
saltmysql.update(sdb, inst_cmd, 'PipelineStatistics', 'NightInfo_Id=%i' % night_id)
msg="Updating information for Night_Id=%i\n" % night_id
else:
saltmysql.insert(sdb, inst_cmd, 'PipelineStatistics')
msg="Inserting information for Night_Id=%i\n" % night_id
#log the call
log.message(msg+inst_cmd, with_stdout=verbose)
def updatenightinfo(obsdate, sdbhost='sdb.saao', sdbname='sdb', \
sdbuser='', password='', logfile='saltlog.log', verbose=True):
"""Update the nightinfo table with current information about the
night
"""
with logging(logfile,debug) as log:
#connect the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#get the nightinfo_id
try:
night_id=saltmysql.getnightinfoid(sdb, obsdate)
except SaltError:
night_id=None
#get the information for the night
time_list=get_nightdetails(obsdate)
#create the insert command
obsdate=str(obsdate)
inst_cmd="Date='%s-%s-%s'," % (obsdate[0:4], obsdate[4:6], obsdate[6:8])
inst_cmd+="SunSet='%s', SunRise='%s', MoonSet='%s', MoonRise='%s', EveningTwilightEnd='%s', MorningTwilightStart='%s'" % \
(time_list[0], time_list[1], time_list[2], time_list[3], time_list[4], time_list[5])
inst_cmd+=",MoonPhase_Percent=%i" % (round(float(time_list[6])))
if night_id:
saltmysql.update(sdb, inst_cmd, 'NightInfo', 'NightInfo_Id=%i' % night_id)
msg="Updating information for Night_Id=%i\n" % night_id
else:
saltmysql.insert(sdb, inst_cmd, 'NightInfo')
msg="Inserting information for Night_Id=%i\n" % night_id
#log the call
log.message(msg+inst_cmd, with_stdout=verbose)
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrbnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrbnum=0
try:
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=max(lasthrbnum, len(glob.glob('hrs/raw/*fits')))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=lasthrbnum
def saltcharge(
obsdate,
outfile,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Calculate the charged time for proposals observed during a night
"""
with logging(logfile, debug) as log:
# check the outfiles
if not saltio.checkfornone(outfile):
outfile = None
# check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
# connect the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# get all the proposals observed during the night
select_state = "distinct Proposal_Code"
table_state = "FileData Join ProposalCode using (ProposalCode_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
records = saltmysql.select(sdb, select_state, table_state, logic_state)
pids = []
pid_time = {}
for r in records:
pids.append(r[0])
pid_time[r[0]] = 0
if len(pids) == 0:
message = "There are no proposal to charge time for %s" % obsdate
log.message(message)
return
# get the nightinfo_id
night_id = saltmysql.getnightinfoid(sdb, obsdate)
print night_id
# get a list of all the images taken during the night
select_state = "FileName, Proposal_Code, Target_Name, ExposureTime, UTSTART, INSTRUME, OBSMODE, DETMODE, CCDTYPE, NExposures"
table_state = "FileData Join ProposalCode using (ProposalCode_Id) join FitsHeaderImage using (FileData_Id)"
logic_state = "FileName like '%" + obsdate + "%'"
img_list = saltmysql.select(sdb, select_state, table_state, logic_state)
# get all the blocks visited
select_state = "Block_Id, Accepted, Proposal_Code"
table_state = "Block join BlockVisit using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)"
logic_state = "NightInfo_Id=%i" % night_id
block_list = saltmysql.select(sdb, select_state, table_state, logic_state)
print block_list
# get the start and end of twilight
nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
print nightstart, nightend, (nightend - nightstart).seconds
print
# download the SOMMI log from the night
try:
sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
except Exception, e:
msg = "Unable to read in SOMMI log for %s" % obsdate
raise SaltError(msg)
# add to account for error in SOMMI log
if int(obsdate) < 20111027:
sommi_log = sommi_log.replace("Object name", "\nObject name")
# parse the sommi log
point_list = parseforpointings(sommi_log)
# now find all blocks observed during a night
for point in point_list:
# proposal for that block
pid = point[0].strip()
# start time for that block
starttime = point[1]
# find the end time for that block
endtime = findnexttime(point[1], point_list, nightend)
# find the start time of the first object to be observed
# for this block
startimage = findfirstimage(pid, starttime, img_list)
lastimage = findlastimage(pid, endtime, img_list)
# check to see if the end time for the last file observed for
# this block
if startimage is not None:
if startimage > endtime:
startimage = None
# determine the acquisition time for the block
if startimage is not None:
acqdelta = (startimage - starttime).seconds
else:
acqdelta = -1
# find the shutter open time
shuttertime = calculate_shutteropen(img_list, starttime, endtime)
# if the shutter and time of the last image is substantially different from the
# end of the block, then take the last image as the end of the block
# *TODO* Change to use expected block time
st = starttime + datetime.timedelta(0, shuttertime + acqdelta)
if (endtime - st).seconds > 900:
if lastimage is None:
endtime = st
elif (lastimage - st).seconds > 3600:
endtime = st
else:
endtime = lastimage
# account for the time for that block
tblock = (endtime - starttime).seconds
if acqdelta > -1:
# find the associated block
block_id, blockvisit_id, accepted = getblockid(sdb, night_id, pid, img_list, starttime, endtime)
if accepted and pid.count("-3-"):
# charged time for that block
try:
pid_time[pid] += tblock
except KeyError:
pid_time[pid] = tblock
print block_id, blockvisit_id, pid, starttime, tblock, acqdelta, shuttertime, shuttertime / tblock, block_id, accepted
# update the charge time
slewdelta = 0
scidelta = tblock - acqdelta
update_cmd = "TotalSlewTime=%i, TotalAcquisitionTime=%i, TotalScienceTime=%i" % (
slewdelta,
acqdelta,
scidelta,
)
table_cmd = "BlockVisit"
logic_cmd = "BlockVisit_Id=%i" % blockvisit_id
saltmysql.update(sdb, update_cmd, table_cmd, logic_cmd)
return
# update the charged time information
# TODO: Check to see if the block was approved
ptime_tot = 0
stime_tot = 0
for k in pid_time:
ptime = pid_time[k]
if ptime > 0:
obsratio = stime / ptime
else:
obsratio = 0
print "%25s %5i %5i %3.2f" % (k, ptime, stime, obsratio)
if k.count("-3-"):
ptime_tot += ptime
stime_tot += stime
# print out the total night statistics
tdelta = nightend - nightstart
print tdelta.seconds, ptime_tot, stime_tot
def saltpipe(obsdate,pinames,archive,ftp,email,emserver,emuser,empasswd,bcc, qcpcuser,qcpcpasswd,
ftpserver,ftpuser,ftppasswd,sdbhost, sdbname, sdbuser, sdbpass, elshost, elsname,
elsuser, elspass, median,function,order,rej_lo,rej_hi,niter,interp,
clobber, runstatus, logfile,verbose):
# set up
basedir=os.getcwd()
propcode=pinames
sender = emuser + '@salt.ac.za'
recipient = sender
emessage = ''
emailfile = '../piemaillist/email.lis'
# check the observation date is sensible
if ('/' in obsdate or '20' not in obsdate or len(obsdate) != 8):
emessage = 'Observation date does not look sensible - YYYYMMDD\n'
raise SaltError(emessage)
# stop if the obsdate temporary directory already exists
obsdir='%s' % obsdate
if os.path.exists(obsdir):
emessage += 'The temporary working directory ' + os.getcwd() + '/'
emessage += obsdate + ' already exists. '
raise SaltError(emessage)
# create a temporary working directory and move to it
saltio.createdir(obsdir)
saltio.changedir(obsdir)
workpath = saltio.abspath('.')
# test the logfile
logfile = workpath+logfile
logfile = saltio.logname(logfile)
#note the starttime
starttime = time.time()
#start logging
with logging(logfile,debug) as log:
#connect to the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, sdbpass)
#get the nightinfo id
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
#Get the list of proposal codes
state_select='Proposal_Code'
state_tables='Proposal join ProposalCode using (ProposalCode_Id)'
state_logic="current=1"
records=saltmysql.select(sdb, state_select, state_tables, state_logic)
propids=[k[0] for k in records]
# Calculate the current date
currentdate=salttime.currentobsdate()
# are the arguments defined
saltio.argdefined('obsdate',obsdate)
# check email and ftp arguments are consistent
if email and not ftp:
message = 'ERROR: SALTPIPE -- cannot send email to PI(s) unless data is transferred '
message += 'to the FTP server; use ftp=\'yes\' email=\'yes\''
raise SaltError(message)
# identify a potential list of keyword edits
keyfile = '../newheadfiles/list_newhead_' + obsdate
if not os.path.isfile(keyfile):
message = '\nSALTPIPE -- keyword edits ' + keyfile + ' not found locally'
log.message(message)
# check directories for the raw RSS data
rssrawpath = makerawdir(obsdate, 'rss')
# check directories for the raw SALTICAM data
scmrawpath = makerawdir(obsdate, 'scam')
# check raw directories for the disk.file record and find last file number
#check rss data
lastrssnum = checkfordata(rssrawpath, 'P', obsdate, log)
#check scame data
lastscmnum = checkfordata(scmrawpath, 'S', obsdate, log)
#check for HRS Data--not filedata yet, so cannot check
if lastrssnum == 1 and lastscmnum == 1:
message = 'SALTPIPE -- no SALTICAM or RSS data obtained on ' + obsdate
emessage += '\n' + message + '\n'
log.message(message)
#copy the data to the working directory
if lastrssnum > 1:
message = 'Copy ' + rssrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(rssrawpath,'rss/raw')
if lastscmnum > 1:
message = 'Copy ' + scmrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
saltio.copydir(scmrawpath,'scam/raw')
#copy and pre-process the HRS data
try:
hrsbrawpath = makerawdir(obsdate, 'hbdet')
saltio.createdir('hrs')
saltio.createdir('hrs/raw')
message = 'Copy ' + hrsbrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsbrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
hrsrrawpath = makerawdir(obsdate, 'hrdet')
message = 'Copy ' + hrsrrawpath + ' --> ' + workpath + 'raw/'
log.message(message)
salthrspreprocess(hrsrrawpath, 'hrs/raw/', clobber=True, log=log, verbose=verbose)
lasthrsnum=len(glob.glob('hrs/raw/*fits'))
except Exception,e:
log.message('Could not copy HRS data because %s' % e)
lasthrsnum=0
if lastrssnum>1 or lastscmnum>1:
message = 'Copy of data is complete'
log.message(message)
else:
message = 'No data was taken on %s' % obsdate
log.message(message)
#process the data RSS data
if lastrssnum>1:
preprocessdata('rss', 'P', obsdate, keyfile, log, logfile, verbose)
#process the SCAM data
if lastscmnum>1:
preprocessdata('scam', 'S', obsdate, keyfile, log, logfile, verbose)
#process the HRS data
if lasthrsnum>1:
preprocessdata('hrs', 'H', obsdate, keyfile, log, logfile, verbose)
preprocessdata('hrs', 'R', obsdate, keyfile, log, logfile, verbose)
#check that all data was given a proper proposal id
#only do it for semesters after the start of science operations
if int(obsdate)>=20110901:
# Check to see that the PROPID keyword exists and if not add it
message = '\nSALTPIPE -- Checking for PROPID keyword'
log.message(message)
#check rss data
rssstatus=runcheckforpropid(glob.glob('rss/raw/P*.fits'), propids, log)
#check scam data
scmstatus=runcheckforpropid(glob.glob('scam/raw/S*.fits'), propids, log)
#check hrsB data
hrsbstatus=runcheckforpropid(glob.glob('hrs/raw/H*.fits'), propids, log)
#check hrsB data
hrsrstatus=runcheckforpropid(glob.glob('hrs/raw/R*.fits'), propids, log)
if not rssstatus or not scmstatus or not hrsbstatus or not hrsrstatus:
msg='The PROPIDs for these files needs to be updated and re-start the pipeline'
raise SaltError("Invalid PROPID in images:"+msg)
#process the RSS data
rssrawsize, rssrawnum, rssprodsize, rssprodnum=processdata('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#advance process the data
if rssrawnum > 0:
advanceprocess('rss', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,sdbhost, sdbname, sdbuser, sdbpass, logfile, verbose)
#process the SCAM data
scmrawsize, scmrawnum, scmprodsize, scmprodnum=processdata('scam', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp,logfile, verbose)
#process the HRS data
hrsrawsize, hrsrawnum, hrsprodsize, hrsprodnum=hrsprocess('hrs', obsdate, propcode, median, function, order, rej_lo, rej_hi, niter, interp, logfile, verbose)
#upload the data to the database
img_list=glob.glob(workpath+'scam/product/*bxgp*.fits')
img_list.extend(glob.glob(workpath+'rss/product/*bxgp*.fits'))
img_list.extend(glob.glob(workpath+'hrs/raw/*.fits'))
if img_list:
img=','.join('%s' % (k) for k in img_list)
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
#add junk sources to the database
raw_list=glob.glob(workpath+'scam/raw/S*.fits')
raw_list.extend(glob.glob(workpath+'rss/raw/P*.fits'))
if raw_list:
img=''
for img in raw_list:
hdu=pyfits.open(img)
if hdu[0].header['PROPID'].strip()=='JUNK':
saltsdbloadfits(images=img, sdbname=sdbname, sdbhost=sdbhost, sdbuser=sdbuser, \
password=sdbpass, logfile=logfile, verbose=verbose)
hdu.close()
# construct observation and pipeline documentation
if lastrssnum > 1 and rssrawnum>0:
rssobslog = 'rss/product/P' + obsdate + 'OBSLOG.fits'
else:
rssobslog = 'none'
if lastscmnum > 1 and scmrawnum>0:
scmobslog = 'scam/product/S' + obsdate + 'OBSLOG.fits'
else:
scmobslog = 'None'
if lasthrsnum > 1 and hrsrawnum>0:
hrsobslog = 'hrs/product/H' + obsdate + 'OBSLOG.fits'
else:
hrsobslog = 'None'
if rssrawnum==0 and scmrawnum==0 and hrsrawnum==0:
msg='No data processed for %s' % obsdate
email=False
ftp=False
log.message(msg)
htmlpath = '.'
nightlog = '../nightlogfiles/' + obsdate + '.log'
readme = iraf.osfn('pipetools$html/readme.template')
if not os.path.isfile(nightlog):
nightlog = ''
message = 'No night log file ~/nightlogfiles/' + obsdate + '.log found'
log.warning(message)
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
salthtml(propcode=propcode,scamobslog=scmobslog,rssobslog=rssobslog, hrsobslog=hrsobslog, htmlpath=htmlpath,
nightlog=nightlog,readme=readme,clobber=True,logfile=logfile,
verbose=verbose)
#add a pause to allow syncing of the databases
time.sleep(10)
#Add in the environmental information
if (rssrawnum > 0 or scmrawnum > 0 or hrsrawnum>0):
propids=saltmysql.getpropcodes(sdb, obsdate)
for pid in propids:
saltelsdata(pid, obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, sdbpass, clobber, logfile,verbose)
try:
outfile='%s_%s_elsdata.fits' % (pid, obsdate)
outdir='%s/doc/' % (pid)
shutil.move(outfile, outdir)
except:
os.remove(outfile)
#ftp the data
beachdir='/salt/ftparea/'
if ftp:
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=False,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)
#run with the splitting of files
try:
saltftp(propcode=propcode,obsdate=obsdate, datapath=workpath,
password=ftppasswd,beachdir=beachdir,sdbhost=sdbhost,
sdbname=sdbname,sdbuser=sdbuser,splitfiles=True,
cleanup=True,clobber=True,logfile=logfile, verbose=verbose)
except Exception,e:
message="Not able to copy data to FTP area:\n%s " % e
raise SaltError(message)
'''
where_term = "Proposal_Code like '%s' and Date='%s-%s-%s'" % (
propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
print select_term, from_term, where_term
try:
record = saltmysql.select(sdb, select_term, from_term,
where_term)[0][0]
except:
record = None
print record
if record == 'FastEmail':
return
else:
#insert information into the database
nightinfoid = saltmysql.getnightinfoid(sdb, obsdate)
insert_term = "NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (
nightinfoid, propid)
table_term = "PipelineProposalStatistics"
saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")
#send email
sender = '*****@*****.**'
recipient = email
bcc = '*****@*****.**'
subject = 'SALT data available for %s' % propcode
message = open(readmefile).read()
message = message.replace('OBSDATE', obsdate)
sendemail(server, 'sa', password, sender, recipient, bcc, subject,
message)
def makenightstats(els, sdb, outfile, obsdate, clobber=False):
"""Retrieve data for a given observation during an observation date for
a proposal
"""
fout=saltio.openascii(outfile, 'w')
headerstr="""<html>
<head><title>SALT Night Report for %s</title></head>
<body bgcolor="white" text="black" link="blue" vlink="blue">
<center><h1>
SALT Night Report for %s<br>
</h1></center>
""" % (obsdate, obsdate)
fout.write(headerstr)
#set up the Observing Statistics
fout.write('<h2> A. Observing Information </h2>\n')
nid=saltmysql.getnightinfoid(sdb, obsdate)
print nid
#get duty information
selcmd='sa.Surname, so.Surname, ct.surname'
tabcmd='NightInfo join SaltOperator as so on (SO1_Id=SO_Id) join Investigator as sa on (SA_Id=sa.Investigator_Id) join Investigator as ct on (CTDuty_Id=ct.Investigator_Id)'
record=saltmysql.select(sdb, selcmd, tabcmd, 'NightInfo_Id=%i' % nid)
try:
sa,so,ct=record[0]
dutystr='SA: %s <br>\nSO: %s <br> \nCT: %s <br>\n<br>' % (sa, so,ct)
except:
dutystr='SA: %s <br>\n SO: %s <br> \n CT: %s <br>\n<br>' % ('', '', '')
fout.write(dutystr)
#get night time information
selcmd='SunSet, SunRise, MoonSet, MoonRise, MoonPhase_Percent, EveningTwilightEnd, MorningTwilightStart'
record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
statlist=['Sun Set', 'Sun Rise', 'Moon Set', 'Moon Rise', 'Moon Phase', 'Evening Twilight', 'Morning Twilight']
statstr='<table border=1><tr><th colspan=2>Nighttime Statistics </th></tr>\n'
for s,r in zip (statlist, record):
statstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
statstr+='</table><br>\n'
fout.write(statstr)
mintime=record[0]-datetime.timedelta(seconds=1*3600)
maxtime=record[1]+datetime.timedelta(seconds=1*3600)
obsstatlist=['Science Time', 'Engineering Time', 'Lost to Weather', 'Lost to Problems']
selcmd='ScienceTime, EngineeringTime, TimeLostToWeather, TimeLostToProblems'
record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
obsstatstr='<table border=1><tr><th colspan=2>Observing Statistics </th></tr>\n'
for s,r in zip (obsstatlist, record):
obsstatstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
obsstatstr+='</table><br>\n'
fout.write(obsstatstr)
#Set up the Envirnmental Statistics
fout.write('<h2> B. Environmental Statistics </h2>\n')
#create the tables
print mintime, maxtime
#guihdu=guidertable(els, mintime, maxtime)
#temperature plot
fout.write('<table>')
weahdu=weathertable(els, mintime, maxtime)
tempplot=temperatureplot(weahdu, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(tempplot))
windfile=windplot(weahdu, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(windfile))
#seeing plot
seeplot=makeseeingplot(sdb, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s><br></td></tr>\n' % os.path.basename(seeplot))
fout.write('</table>\n')
#Set up the Pipeline Statistics
fout.write('<h2> C. Pipeline Statistics </h2>\n')
fout.write('<table>\n')
record=saltmysql.select(sdb, 'PipelineStatus, RawSize, ReducedSize, PipelineRunTime', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % nid)[0]
print record
print record[1]/3600.0
pipelinestatus='Completed'
fout.write('<tr><td>Pipeline Status:</td><td>%s</td></tr>' % record[0])
fout.write('<tr><td>Raw Data:</td><td>%3.2f Gb</td></tr>' % (record[1]/1e9))
fout.write('<tr><td>Reduced Data:</td><td>%3.2f Gb</td></tr>' % (record[2]/1e9))
fout.write('<tr><td>Run Time:</td><td>%3.2f min</td></tr>' % (record[3]/60.0))
fout.write('</table>\n')
#Set up the Proposal/Block Statistics
fout.write('<h2> D. Proposal Statistics </h2>\n')
selcmd='Proposal_Code, Block_Name, Accepted, ObsTime, BlockRejectedReason_Id, BlockVisit_Id'
tabcmd='Block join BlockVisit as bv using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)'
logcmd='NightInfo_Id=%i' % nid
record=saltmysql.select(sdb, selcmd, tabcmd, logcmd)
print record
fout.write('<table border=1>\n')
fout.write('<tr><th>Proposal</th><th>Block</th><th>Obs Time</th><th>Accepted?</th><th>Rejected Reason</th></tr>\n')
for r in record:
if r[2]>=1:
accept='Yes'
reason=''
else:
accept='No'
print r
reason=saltmysql.select(sdb, 'RejectedReason', 'BlockRejectedReason', 'BlockRejectedReason_Id=%i' % int(r[4]))[0][0]
bstr='<tr><td><a href="https://www.salt.ac.za/wm/proposal/%s/">%s</a></td><td>%s</td><td>%3.2f</td><td>%s</td><td>%s</td></tr>\n' % (r[0], r[0], r[1], float(r[3])/3600.0, accept, reason)
fout.write(bstr)
fout.write('</table>\n')
#Set up the Data Quality Statistics
fout.write('<h2> E. Data Quality </h2>\n')
fout.write('</body> \n</hmtl>')
fout.close()
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
print select_term, from_term, where_term
try:
record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
except:
record=None
print record
if record=='FastEmail':
return
else:
#insert information into the database
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
table_term="PipelineProposalStatistics"
saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")
#send email
sender='*****@*****.**'
recipient=email
bcc='*****@*****.**'
subject='SALT data available for %s' % propcode
message=open(readmefile).read()
message=message.replace('OBSDATE', obsdate)
sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
def runfast(filename, propcode, obsdate, server, readmefile, sdbhost, sdbname, sdbuser, password):
"""Handle fast data delivery for the proposal. For a given filename
"""
if propcode is None or propcode=='None': return
#first check in the sdb if fast data delivery is needed
sdb=saltmysql.connectdb(sdbhost,sdbname, sdbuser, password)
select_term='Distinct Surname, email, username, ProposalCode_Id'
from_term='''
Block join Pointing using (Block_Id)
join PipelineConfig using (Pointing_Id)
join Proposal using (Proposal_Id)
join ProposalCode using (ProposalCode_Id)
join PipelineDataAccessMethod using (PipelineDataAccessMethod_Id)
join ProposalContact using (Proposal_Id)
join Investigator on (Investigator_Id=Contact_Id)
join PiptUser using (PiptUser_Id)
'''
where_term="Proposal_Code like '%s' and current=1 and DataAccessMethod='Fast'" \
% (propcode)
#print 'Select %s from %s where %s' % (select_term, from_term, where_term)
try:
record=saltmysql.select(sdb, select_term, from_term, where_term)
except Exception as e:
print(e)
return None
#print "Checking for fast data"
#print record
if record:
surname, email, username, propid= record[0]
#print surname, email, username, propid
else:
return
#second if so, then copy the data to the contact PI directory
#on saltpipe under the fast directory.
#rawfilename=getrawfilename(filename)
y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
if y==256:
y=os.system('ssh sa@saltpipe mkdir /salt/ftparea/%s/fast%s' % (username, obsdate))
y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (filename, username, obsdate))
if y!=0:
print("Problem with copying file %s to /salt/ftparea/%s/fast%s/" % (filename, username, obsdate))
#copy the reduced data
y=os.system('scp mbxp%s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (os.path.basename(filename), username, obsdate))
#check the type of data it is and copy over an ancillery data as well
#if it is the first object file, check to see if an email has been
#sent, and if not, send email
#try to copy the spectroscopic data
print(filename, filename.startswith('P'))
if os.path.basename(filename).startswith('P'):
sfilename='smbxp%s.txt' % (os.path.basename(filename).split('.fits')[0])
print(sfilename)
try:
y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
except Exception as e:
print(e)
if os.path.basename(filename).startswith('S'):
try:
sfilename='mbxp%s.cat' % (os.path.basename(filename).split('.fits')[0])
print(sfilename)
y=os.system('scp %s sa@saltpipe:/salt/ftparea/%s/fast%s/' % (sfilename, username, obsdate))
except Exception as e:
print(e)
#check to see if an email has been sent
select_term='PipelineStatus'
from_term='''
PipelineProposalStatistics
join PipelineStatus using (PipelineStatus_Id)
join NightInfo using (NightInfo_Id)
join ProposalCode using (ProposalCode_Id)
'''
where_term="Proposal_Code like '%s' and Date='%s-%s-%s'" % (propcode, obsdate[0:4], obsdate[4:6], obsdate[6:8])
print(select_term, from_term, where_term)
try:
record=saltmysql.select(sdb, select_term, from_term, where_term)[0][0]
except:
record=None
print(record)
if record=='FastEmail':
return
else:
#insert information into the database
nightinfoid=saltmysql.getnightinfoid(sdb, obsdate)
insert_term="NightInfo_Id=%i, ProposalCode_Id=%i, PipelineStatus_Id=8" % (nightinfoid, propid)
table_term="PipelineProposalStatistics"
saltmysql.insert(sdb, insert_term, "PipelineProposalStatistics")
#send email
sender='*****@*****.**'
recipient=email
bcc='*****@*****.**'
subject='SALT data available for %s' % propcode
message=open(readmefile).read()
message=message.replace('OBSDATE', obsdate)
sendemail(server,'sa',password,sender,recipient,bcc, subject,message)
sdb.close()
return
def saltnightinfo(
obsdate,
sdbhost="sdb.saao",
sdbname="sdb",
sdbuser="",
password="",
clobber=False,
logfile="saltlog.log",
verbose=True,
):
"""Update the nightinfo table from the SOMMI log
"""
with logging(logfile, debug) as log:
# connect the database
sdb = saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
# get the nightinfo_id
night_id = saltmysql.getnightinfoid(sdb, obsdate)
print night_id
# get the start and end of twilight
nightstart = saltmysql.select(sdb, "EveningTwilightEnd", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
nightend = saltmysql.select(sdb, "MorningTwilightStart", "NightInfo", "NightInfo_Id=%i" % night_id)[0][0]
print nightstart
print nightend
print (nightend - nightstart).seconds
# download the SOMMI log from the night
try:
sommi_log = saltmysql.select(sdb, "SONightLog", "NightLogs", "NightInfo_Id=%i" % night_id)[0][0]
except:
raise SaltError("Unable to read SOMMI log in the database for %s" % obsdate)
# set up the time for the night
try:
ntime = (nightend - nightstart).seconds
except:
raise SaltError("Unable to read night length from database for %s" % obsdate)
# parse the sommi log
slog = sommi_log.split("\n")
stime = 0
for i in range(len(slog)):
if slog[i].count("Science Time:"):
stime = extractinformation(slog[i])
if slog[i].count("Engineering Time:") and not slog[i].count("Non-observing Engineering Time"):
etime = extractinformation(slog[i])
if slog[i].count("Time lost to Weather:"):
wtime = extractinformation(slog[i])
if slog[i].count("Time lost to Tech. Problems:"):
ttime = extractinformation(slog[i])
if slog[i].count("Non-observing Engineering Time:"):
ltime = extractinformation(slog[i])
print etime
tot_time = stime + etime + wtime + ttime
print night_id, ntime, stime, etime, wtime, ttime, ltime, tot_time
# insert the information into the database
print tot_time, ntime
if abs(tot_time - ntime) < 900:
message = "Updating NightInfo Table with the following Times:\n"
message += "Science Time=%i\n" % stime
message += "Engineeringh=%i\n" % etime
message += "Time lost to Weather=%i\n" % wtime
message += "Time lost to Tech. Problems=%i\n" % ttime
message += "Non-observing Engineering Time=%i\n" % ltime
log.message(message)
insert_state = (
"ScienceTime=%i, EngineeringTime=%i, TimeLostToWeather=%i, TimeLostToProblems=%i, NonObservingEngineeringTime=%i"
% (stime, etime, wtime, ttime, ltime)
)
table_state = "NightInfo"
logic_state = "NightInfo_Id=%i" % night_id
saltmysql.update(sdb, insert_state, table_state, logic_state)
else:
message = "The total time for the night is not equal to the length of the night\n"
message += "Night Length=%i\n--------------------\n" % ntime
message += "Science Time=%i\n" % stime
message += "Engineeringh=%i\n" % etime
message += "Time lost to Weather=%i\n" % wtime
message += "Time lost to Tech. Problems=%i\n" % ttime
message += "Non-observing Engineering Time=%i\n" % ltime
message += "Total time for the Night=%i\n" % tot_time
log.message(message)
def saltslewstats(startdate, enddate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):
# set up
proposers = []
propids = []
pids = []
with logging(logfile,debug) as log:
#open the database
els=saltmysql.connectdb(elshost, elsname, elsuser, elspass)
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)
#loop through each night and determine the statistics for the slew times
#for each night
obsdate=int(startdate)
slew_list=[]
while obsdate < int(enddate):
night_id=saltmysql.getnightinfoid(sdb, obsdate)
start,end=saltmysql.select(sdb,'EveningTwilightEnd,MorningTwilightStart','NightInfo', 'NightInfo_Id=%i' % night_id)[0]
tslew, nslew=slewtime(els, start, end)
if nslew>0:
print obsdate, night_id, start, end, nslew, tslew/nslew
slew_list.append([start, nslew, tslew])
obsdate=int(getnextdate(obsdate))
slew_arr=np.array(slew_list)
days=np.zeros(len(slew_arr))
for i in range(len(slew_arr)):
days[i]=(slew_arr[i,0]-slew_arr[0,0]).days
coef=np.polyfit(days, slew_arr[:,2]/slew_arr[:,1], 2)
ave_date=[]
ave_values=[]
ave_nslews=[]
nstep=10
for i in np.arange(15,len(slew_arr), 2*nstep):
ave_date.append(slew_arr[i,0])
i1=i-nstep
i2=min(i+nstep, len(slew_arr))
#ave_values.append(np.median(slew_arr[i1:i2,2]/slew_arr[i1:i2,1]))
ave_nslews.append(np.median(slew_arr[i1:i2,1]))
ave_values.append(np.median(slew_arr[i1:i2,2]))
print ave_date[-1], ave_values[-1], ave_nslews[-1]
ave_values=np.array(ave_values)
ave_nslews=np.array(ave_nslews)
mean_slew=ave_nslews.mean()
mean_slew=11
for i in range(len(ave_date)):
#value is an attempt to correct for the average number of slews
value=(ave_values[i]+30*(mean_slew-ave_nslews[i]))/mean_slew
print ave_date[i], '%i %i %i' % (ave_values[i]/ave_nslews[i], ave_nslews[i], value)
plt.figure()
plt.plot(slew_arr[:,0], slew_arr[:,2]/slew_arr[:,1], ls='', marker='o')
#plt.plot(slew_arr[:,0], slew_arr[:,1], ls='', marker='o')
#plt.plot(slew_arr[:,0], slew_arr[:,2], ls='', marker='o')
plt.plot(ave_date, ave_values/ave_nslews)
#plt.plot(days, np.polyval(coef, days))
plt.ylabel('Slew Time(s)')
plt.xlabel('Date')
plt.show()