def readsolascii(solfile, soldict):
"""Read in an ascii file that has the wavelength solution in formation in it.
The ascii file is expected to be in the same format as the output from
saltidentify
return soldict
"""
# open up the ascii file
fin = saltsafeio.openascii(solfile, 'r')
data = fin.read()
# Each new solution in the file should have WS at the start of it. We will split
# the file on ws
sol_list = data.split('#WS')
for sol in sol_list[1:]:
name = findwskeyword('name', sol)
timeobs = enterdatetime(findwskeyword('time-obs', sol))
instrume = findwskeyword('instrument', sol)
grating = findwskeyword('grating', sol)
graang = float(findwskeyword('gratilt', sol))
arang = float(findwskeyword('artilt', sol))
filtername = findwskeyword('filter', sol)
function = findwskeyword('Function', sol)
order = int(findwskeyword('Order', sol))
try:
slitid = findwskeyword('slitid', sol)
name = name + '_' + slitid
except:
slitid = None
try:
domain = findwskeyword('domain', sol)
domain = [float(x) for x in domain.split(',')]
except:
domain = None
wsrow = []
wscoef = []
j = finddata(sol)
for ws in sol[j:].splitlines():
if ws.strip():
ws = ws.split()
wsrow.append(float(ws[0]))
wscoef.append(np.array(ws[1:], dtype=float))
soldict[name] = [
timeobs,
instrume,
grating,
graang,
arang,
filtername,
slitid,
function,
order,
np.array(wsrow),
wscoef,
domain]
return soldict
def readsolascii(solfile, soldict):
"""Read in an ascii file that has the wavelength solution in formation in it.
The ascii file is expected to be in the same format as the output from
saltidentify
return soldict
"""
# open up the ascii file
fin = saltsafeio.openascii(solfile, 'r')
data = fin.read()
# Each new solution in the file should have WS at the start of it. We will split
# the file on ws
sol_list = data.split('#WS')
for sol in sol_list[1:]:
name = findwskeyword('name', sol)
timeobs = enterdatetime(findwskeyword('time-obs', sol))
instrume = findwskeyword('instrument', sol)
grating = findwskeyword('grating', sol)
graang = float(findwskeyword('gratilt', sol))
arang = float(findwskeyword('artilt', sol))
filtername = findwskeyword('filter', sol)
function = findwskeyword('Function', sol)
order = int(findwskeyword('Order', sol))
try:
slitid = findwskeyword('slitid', sol)
name = name + '_' + slitid
except:
slitid = None
try:
domain = findwskeyword('domain', sol)
domain = [float(x) for x in domain.split(',')]
except:
domain = None
wsrow = []
wscoef = []
j = finddata(sol)
for ws in sol[j:].splitlines():
if ws.strip():
ws = ws.split()
wsrow.append(float(ws[0]))
wscoef.append(np.array(ws[1:], dtype=float))
soldict[name] = [
timeobs,
instrume,
grating,
graang,
arang,
filtername,
slitid,
function,
order,
np.array(wsrow),
wscoef,
domain]
return soldict
def writespectrum(spectra, outfile, error=False, ftype=None):
"""Given a spectrum, write it out to a file"""
fout=saltio.openascii(outfile, 'w')
for i in range(spectra.nwave):
fout.write('%8.6f ' % spectra.wavelength[i])
fout.write('%8.6e ' % spectra.flux[i])
if error: fout.write('%8.6e ' % spectra.sigma[i])
fout.write('\n')
fout.close()
def writespectrum(spectra, outfile, error=False, ftype=None):
"""Given a spectrum, write it out to a file"""
fout = saltio.openascii(outfile, 'w')
for i in range(spectra.nwave):
fout.write('%8.6f ' % spectra.wavelength[i])
fout.write('%8.6e ' % spectra.flux[i])
if error:
fout.write('%8.6e ' % spectra.var[i])
fout.write('\n')
fout.close()
def mkheader(file, keyword, value, comment):
"""create keyword with mkheader IRAF tool i.e. without opening the whole file"""
try:
tmpfile = saltsafeio.tmpfile('.', False)
tmp = saltsafeio.openascii(tmpfile, 'w')
tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword, value, comment))
saltsafeio.closeascii(tmp)
iraf.noao.artdata.mkheader(file, tmpfile, append='y', verbose='n')
saltsafeio.delete(tmpfile, False)
except:
raise SaltIOError('Cannot edit keyword ' + keyword + ' in ' + file)
def mkheader(file,keyword,value,comment):
"""create keyword with mkheader IRAF tool i.e. without opening the whole file"""
try:
tmpfile=saltsafeio.tmpfile('.',False)
tmp=saltsafeio.openascii(tmpfile,'w')
tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword,value,comment))
saltsafeio.closeascii(tmp)
iraf.noao.artdata.mkheader(file,tmpfile,append='y',verbose='n')
saltsafeio.delete(tmpfile,False)
except:
raise SaltIOError('Cannot edit keyword '+keyword+' in '+file)
def maketempreadmefast(pid, sdb, readme):
"""Creates the readme file to send to the PI. It adds on the observing log for those
observations
"""
#read in the readme
f=saltio.openascii(readme, 'r')
rstring=f.read()
saltio.closeascii(f)
#replace propcode with the actual propcode
if pid: rstring=rstring.replace('PROPCODE', pid)
#add in the observing log
return rstring
def saltfpcalring(images,outfile, waves, method=None, thresh=5, minsize=10, niter=3, conv=0.05,
axc=None, ayc=None,
clobber=False,logfile='salt.log',verbose=True):
"""Fits rings in Fabry-Perot ring images"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# if the outfile exists and not clobber, fail
saltio.overwrite(outfile,clobber)
#open the output file
fout = saltio.openascii(outfile, 'w')
#make sure that the list of waves is convertable to a numpy array
#convert to floats in case the wave is given as a string
if isinstance(waves, str):
waves=[float(x) for x in waves.split(',')]
try:
waves=np.array(waves)
except:
raise SaltError('%s is not convertable to a numpy array' % waves)
#check the value of method
method=saltio.checkfornone(method)
#setup the output file
fout.write('#Comment\n')
fout.write('# radius err xc yc z ut wave dn file\n')
# open each image and detect the ring
for img,w in zip(infiles, waves):
#open the image
hdu=saltio.openfits(img)
#measure the ring in each file
xc, yc, radius, err, z, ut=make_calring(hdu, method=method, thresh=thresh, niter=niter, conv=conv, minsize=minsize, axc=axc, ayc=ayc)
#output the results
outstr=' %7.2f %6.2f %7.2f %7.2f %6.2f %7.4f %8.3f 0 %s\n' % (radius, err, xc, yc, z, ut, w, img)
fout.write(outstr)
log.message(outstr.strip(), with_stdout=verbose, with_header=False)
fout.close()
def readkeyfile(keyfile, log=None, verbose=True):
"""Read in the keyword file"""
#open the key word file
kfile = saltio.openascii(keyfile,'r')
#extract keyword edits
keyedits={}
for line in kfile:
if len(line.strip()) > 0 and not line.startswith('#'):
try:
frange, finstr, fstar, fend=getfrange(line)
#print frange, finstr, fstar, fend
keyedits[frange]=[finstr, fstar, fend, getitems(line, frange)]
#print frange, keyedits[frange]
#print getitems(line, frange)
except EditKeyError,e :
message="Could not extract keyword edits for %s because %s" % (line, e)
log.error(message)
def checkfordata(rawpath, prefix, obsdate, log):
"""Check to see if the data have downloaded correctly"""
lastnum=1
saltio.fileexists(rawpath+'disk.file')
content = saltio.openascii(rawpath+'disk.file','r')
for line in content:
lastnum = saltstring.filenumber(line)
saltio.closeascii(content)
lastfile = saltstring.filename(prefix,obsdate,lastnum-1)
if lastnum==1: return lastnum
#check to see that the data are present
if (os.path.isfile(rawpath+lastfile) or os.path.isfile(rawpath+lastfile.replace('fits','bin'))):
message = 'Data download complete for %s\n' % rawpath
log.message(message)
else:
message = 'Data download incomplete to %s' % rawpath
log.error(message)
return lastnum
def write_extract_text(ofile, ap_list, clobber=False):
"""Write out the extracted spectrum to a text file. If the file already
exists, this will not overwrite it. The first
For each spectrum in ap_list, it will add a columns onto the output file
so that the first column is always wavelength, the second column is
flux, and the third column is sigma, and then repeat the flux and sigma
columns
ofile: Output file to write
ap_list: List of extracted spectrum
clobber: delete ofile if it already exists
"""
if os.path.isfile(ofile) and not clobber:
return
# open the file
dout = saltio.openascii(ofile, 'w')
# first extract warr, assume it is the same for all frames
warr = ap_list[0].wave
# write out the spectrum
for i in range(len(warr)):
outstr = '%7.3f ' % warr[i]
for ap in ap_list:
flux = ap.ldata[i]
try:
fvar = abs(ap.lvar[i])**0.5
except:
fvar = 1
outstr += "%7.3f %7.3f " % (flux, fvar)
outstr += '\n'
dout.write(outstr)
dout.close()
return
def write_extract_text(ofile, ap_list, clobber=False):
"""Write out the extracted spectrum to a text file. If the file already
exists, this will not overwrite it. The first
For each spectrum in ap_list, it will add a columns onto the output file
so that the first column is always wavelength, the second column is
flux, and the third column is sigma, and then repeat the flux and sigma
columns
ofile: Output file to write
ap_list: List of extracted spectrum
clobber: delete ofile if it already exists
"""
if os.path.isfile(ofile) and not clobber:
return
# open the file
dout = saltio.openascii(ofile, "w")
# first extract warr, assume it is the same for all frames
warr = ap_list[0].wave
# write out the spectrum
for i in range(len(warr)):
outstr = "%7.3f " % warr[i]
for ap in ap_list:
flux = ap.ldata[i]
try:
fvar = abs(ap.lvar[i]) ** 0.5
except:
fvar = 1
outstr += "%7.3f %7.3f " % (flux, fvar)
outstr += "\n"
dout.write(outstr)
dout.close()
return
def fitsconfig(config):
"""reads in the configuration file and calculates the appropriate parameters
for the sections
returns dictiionary
"""
condict={}
#open and read the config file
construct=saltio.openascii(config,'r')
try:
condata = construct.read().split('\n')
except:
message = 'ERROR: VID2BIN.FITSCONFIG -- cannot read in ' + file
raise SaltError(message)
for conline in condata:
ckey,cval=processline(conline)
if ckey and cval: condict[ckey]=cval
return condict
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep = 0
with logging(logfile, debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps > 0: sleep = 1.0 / (fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr = st.readlcfile(
indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2 = st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1', struct[1])
naxis2 = saltkey.get('NAXIS2', struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit = App.exec_()
# Check if GUI was executed succesfully
if app_exit != 0:
raise SALTError('InterIdentify GUI has unexpected exit status ' +
str(exit))
ratio, tflux, cflux, gframe, newphot = aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc = saltio.openascii(fileout, 'w')
for i in range(len(ratio)):
x['target'] = tx[i]
x['comparison'] = cx[i]
y['target'] = ty[i]
y['comparison'] = cy[i]
reltime = False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep=0
with logging(logfile,debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps>0: sleep=1.0/(fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr=st.readlcfile(indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1',struct[1])
naxis2 = saltkey.get('NAXIS2',struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit=App.exec_()
# Check if GUI was executed succesfully
if app_exit!=0:
raise SALTError('InterIdentify GUI has unexpected exit status '+str(exit))
ratio, tflux, cflux, gframe, newphot=aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc=saltio.openascii(fileout,'w')
for i in range(len(ratio)):
x['target']=tx[i]
x['comparison']=cx[i]
y['target']=ty[i]
y['comparison']=cy[i]
reltime=False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
print pids, propids
try:
pids=saltio.cleanpropcode(pids, propids)
pids=saltio.removebadpids(pids)
pids=saltio.removeengineeringpids(pids)
except SaltIOError:
msg="No notifications necessary for %s" % obsdate
log.warning
return
#loop through each of the pids and send the email
for pid in pids:
propinfo=findpropinfo(pid, sdb)
if propinfo:
for pi, email in zip(propinfo[pid][0], propinfo[pid][1]):
letter=saltio.openascii(readme,'r')
msg=letter.read()
msg=msg.replace('yourname',pi)
msg=msg.replace('YYYY-INST-PID',pid.upper())
msg=msg.replace('yyyymmdd',obsdate)
saltio.closeascii(letter)
#set up the message to be sent
recip = []
#uncomment the following lines if you just want to send the email
#to yourself
#email='*****@*****.**'
#bcclist=[]
recip.append(email)
for bccobj in bcclist:
recip.append(bccobj)
def saltquery(selection, logic, startdate, enddate, outfile=None, sdbhost='sdb.saao', sdbname='sdb', \
sdbuser='', password='', clobber=False, logfile='saltlog.log', verbose=True):
"""Query the salt database for FITS files
"""
with logging(logfile,debug) as log:
#check the outfiles
if not saltio.checkfornone(outfile):
outfile=None
#check that the output file can be deleted
if outfile:
saltio.overwrite(outfile, clobber)
#open outfile
if outfile:
fout=saltio.openascii(outfile, 'w')
#connect to the database
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser,password)
#Create a list of the selection and then unpack it
selection_list = saltio.argunpack('Selection', selection)
selection=','.join(selection_list)
#write out the header for the outfile
outstr='#'+' '.join(['%s' % x for x in selection_list])
if outfile:
fout.write(outstr+'\n')
else:
print outstr
#set up the table
rsstable=''' FileData
left join FitsHeaderImage using (FileData_Id)
inner join FitsHeaderRss using (FileData_Id)
'''
#set up the table
scamtable=''' FileData
left join FitsHeaderImage using (FileData_Id)
inner join FitsHeaderSalticam using (FileData_Id)
'''
#set up the logic
logic=makelogic(logic, startdate, enddate)
for tab in [rsstable, scamtable]:
msg='''
Mysql querying data is:
SELECT %s
FROM %s
WHERE %s
''' % (selection, tab, logic)
log.message(msg, with_stdout=verbose)
record=saltmysql.select(sdb, selection, tab, logic)
print record
for r in record:
outstr=' '.join(['%s' % x for x in r])
if outfile:
fout.write(outstr+'\n')
else:
print outstr
#close outfile
if outfile: fout.close()
def makenightstats(els, sdb, outfile, obsdate, clobber=False):
"""Retrieve data for a given observation during an observation date for
a proposal
"""
fout=saltio.openascii(outfile, 'w')
headerstr="""<html>
<head><title>SALT Night Report for %s</title></head>
<body bgcolor="white" text="black" link="blue" vlink="blue">
<center><h1>
SALT Night Report for %s<br>
</h1></center>
""" % (obsdate, obsdate)
fout.write(headerstr)
#set up the Observing Statistics
fout.write('<h2> A. Observing Information </h2>\n')
nid=saltmysql.getnightinfoid(sdb, obsdate)
print nid
#get duty information
selcmd='sa.Surname, so.Surname, ct.surname'
tabcmd='NightInfo join SaltOperator as so on (SO1_Id=SO_Id) join Investigator as sa on (SA_Id=sa.Investigator_Id) join Investigator as ct on (CTDuty_Id=ct.Investigator_Id)'
record=saltmysql.select(sdb, selcmd, tabcmd, 'NightInfo_Id=%i' % nid)
try:
sa,so,ct=record[0]
dutystr='SA: %s <br>\nSO: %s <br> \nCT: %s <br>\n<br>' % (sa, so,ct)
except:
dutystr='SA: %s <br>\n SO: %s <br> \n CT: %s <br>\n<br>' % ('', '', '')
fout.write(dutystr)
#get night time information
selcmd='SunSet, SunRise, MoonSet, MoonRise, MoonPhase_Percent, EveningTwilightEnd, MorningTwilightStart'
record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
statlist=['Sun Set', 'Sun Rise', 'Moon Set', 'Moon Rise', 'Moon Phase', 'Evening Twilight', 'Morning Twilight']
statstr='<table border=1><tr><th colspan=2>Nighttime Statistics </th></tr>\n'
for s,r in zip (statlist, record):
statstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
statstr+='</table><br>\n'
fout.write(statstr)
mintime=record[0]-datetime.timedelta(seconds=1*3600)
maxtime=record[1]+datetime.timedelta(seconds=1*3600)
obsstatlist=['Science Time', 'Engineering Time', 'Lost to Weather', 'Lost to Problems']
selcmd='ScienceTime, EngineeringTime, TimeLostToWeather, TimeLostToProblems'
record=saltmysql.select(sdb, selcmd, 'NightInfo', 'NightInfo_Id=%i' % nid)[0]
obsstatstr='<table border=1><tr><th colspan=2>Observing Statistics </th></tr>\n'
for s,r in zip (obsstatlist, record):
obsstatstr+='<tr><td>%20s</td><td> %s </td></tr>\n' % (s, r)
obsstatstr+='</table><br>\n'
fout.write(obsstatstr)
#Set up the Envirnmental Statistics
fout.write('<h2> B. Environmental Statistics </h2>\n')
#create the tables
print mintime, maxtime
#guihdu=guidertable(els, mintime, maxtime)
#temperature plot
fout.write('<table>')
weahdu=weathertable(els, mintime, maxtime)
tempplot=temperatureplot(weahdu, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(tempplot))
windfile=windplot(weahdu, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s></td></tr>\n' % os.path.basename(windfile))
#seeing plot
seeplot=makeseeingplot(sdb, mintime, maxtime, obsdate)
fout.write('<tr><td><img width=700 height=200 src=%s><br></td></tr>\n' % os.path.basename(seeplot))
fout.write('</table>\n')
#Set up the Pipeline Statistics
fout.write('<h2> C. Pipeline Statistics </h2>\n')
fout.write('<table>\n')
record=saltmysql.select(sdb, 'PipelineStatus, RawSize, ReducedSize, PipelineRunTime', 'PipelineStatistics join PipelineStatus using (PipelineStatus_Id)', 'NightInfo_Id=%i' % nid)[0]
print record
print record[1]/3600.0
pipelinestatus='Completed'
fout.write('<tr><td>Pipeline Status:</td><td>%s</td></tr>' % record[0])
fout.write('<tr><td>Raw Data:</td><td>%3.2f Gb</td></tr>' % (record[1]/1e9))
fout.write('<tr><td>Reduced Data:</td><td>%3.2f Gb</td></tr>' % (record[2]/1e9))
fout.write('<tr><td>Run Time:</td><td>%3.2f min</td></tr>' % (record[3]/60.0))
fout.write('</table>\n')
#Set up the Proposal/Block Statistics
fout.write('<h2> D. Proposal Statistics </h2>\n')
selcmd='Proposal_Code, Block_Name, Accepted, ObsTime, BlockRejectedReason_Id, BlockVisit_Id'
tabcmd='Block join BlockVisit as bv using (Block_Id) join Proposal using (Proposal_Id) join ProposalCode using (ProposalCode_Id)'
logcmd='NightInfo_Id=%i' % nid
record=saltmysql.select(sdb, selcmd, tabcmd, logcmd)
print record
fout.write('<table border=1>\n')
fout.write('<tr><th>Proposal</th><th>Block</th><th>Obs Time</th><th>Accepted?</th><th>Rejected Reason</th></tr>\n')
for r in record:
if r[2]>=1:
accept='Yes'
reason=''
else:
accept='No'
print r
reason=saltmysql.select(sdb, 'RejectedReason', 'BlockRejectedReason', 'BlockRejectedReason_Id=%i' % int(r[4]))[0][0]
bstr='<tr><td><a href="https://www.salt.ac.za/wm/proposal/%s/">%s</a></td><td>%s</td><td>%3.2f</td><td>%s</td><td>%s</td></tr>\n' % (r[0], r[0], r[1], float(r[3])/3600.0, accept, reason)
fout.write(bstr)
fout.write('</table>\n')
#Set up the Data Quality Statistics
fout.write('<h2> E. Data Quality </h2>\n')
fout.write('</body> \n</hmtl>')
fout.close()