def runinsert(index=None, f1=None, f2=None, auto=False):
ad = AstroData(f1)
md = AstroData(f2)
pstr = "\n\n >>>>>>> AD <<<<<<<<\n"
pstr += str(ad.infostr())
pstr += "\n\n >>>>>>> AD APPEND <<<<<<<<\n"
pstr += str(md.infostr())
ad.insert(index=index, moredata=md, auto_number=auto)
pstr +="\n\n >>>>>>> NEW AD <<<<<<<<\n"
pstr += str(ad.infostr())
print(pstr)
return ad
def do_GET(self):
global webserverdone
self.state = ppwstate
rim = self.informers["rim"]
parms = parsepath(self.path)
# Older revisions of adcc may not supply 'verbose' key
try:
self.informers["verbose"]
except KeyError:
self.informers["verbose"] = True
try:
if self.path == "/":
page = """
<html>
<head>
</head>
<body>
<h4>prsproxy engineering interface</h4>
<ul>
<li><a href="/engineering">Engeering Interface</a></li>
<li><a href="qap/engineering.html">Engeering AJAX App</a></li>
<li><a href="datadir">Data Directory View</a></li>
<li><a href="killprs">Kill this server</a> (%(numinsts)d """ +\
"""copies ofreduce registered)</li>
</ul>
<body>
</html>"""
page % {"numinsts":rim.numinsts}
self.send_response(200)
self.send_header("content-type", "text-html")
self.end_headers()
self.wfile.write(page)
return
if parms["path"].startswith("/rqlog.json"):
self.send_response(200)
self.send_header('Content-type', "application/json")
self.end_headers()
if "file" in parms:
logfile = parms["file"][0]
print logfile
if not os.path.exists(logfile):
msg = "Log file not available"
else:
f = open(logfile, "r")
msg = f.read()
f.close()
else:
msg = "No log file available"
tdic = {"log":msg}
self.wfile.write(json.dumps(tdic, sort_keys=True, indent=4))
return
# ------------------------------------------------------------------
# Server time
if parms["path"].startswith("/rqsite.json"):
self.send_response(200)
self.send_header('Content-type', "application/json")
self.end_headers()
tdic = server_time()
self.wfile.write(json.dumps(tdic, sort_keys=True, indent=4))
return
# ------------------------------------------------------------------
# Metrics query employing fitsstore
if parms["path"].startswith("/cmdqueue.json"):
self.send_header('Content-type', "application/json")
self._handle_cmdqueue_json(rim, parms)
return
# ------------------------------------------------------------------
if parms["path"].startswith("/cmdqueue.xml"):
self.send_response(200)
self.send_header('Content-type','text/xml')
self.end_headers()
if "lastcmd" in parms:
start = int(parms["lastcmd"][0])+1
else:
start = 0
elist = self.state.rim.displayCmdHistory.peekSince(cmdNum=start)
print "prsw 200:", repr(elist)
xml = '<commandQueue lastCmd="%d">' % (start-1)
for cmd in elist:
# this is because there should be only one top key
# in the cmd dict
cmdname = cmd.keys()[0]
cmdbody = cmd[cmdname]
xml += '<command name="%s">' % cmdname
if "files" in cmdbody:
basenames = cmdbody["files"].keys()
for basename in basenames:
fileitem = cmdbody["files"][basename]
if "url" not in fileitem or fileitem["url"] == None:
url = "None"
else:
url = fileitem["url"]
xml += """<file basename="%(basename)s"
url = "%(url)s"
cmdnum = "%(cn)d"/>""" % {
"basename": basename,
"url": "" if "file" not in fileitem else fileitem["url"],
"cn":int(cmdbody["cmdNum"])}
# now any extension in the extdict
if "extdict" in fileitem:
extdict = fileitem["extdict"]
for name in extdict.keys():
xml += """\n<file basename="%(basename)s"
url="%(url)s"
ext="%(ext)s"
cmdnum="%(cn)d"/>""" % {
"basename": basename,
"ext": name,
"url": extdict[name],
"cn":int(cmdbody["cmdNum"])}
xml += '</command>'
xml += "</commandQueue>"
self.wfile.write(xml)
return
if parms["path"] == "/recipeindex.xml":
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
self.wfile.write(rl.getRecipeIndex(as_xml=True))
return
if parms["path"].startswith("/summary"):
from fitsstore.FitsStorageWebSummary.Summary import summary
from fitsstore.FitsStorageWebSummary.Selection import getselection
selection = getselection({})
rec = PRec()
summary(rec, "summary", selection, [], links=False)
buff = rec._buff
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
self.wfile.write(buff)
return
if parms["path"].startswith("/calmgr"):
from FitsStorageWebSummary.Selection import getselection
from FitsStorageWebSummary.CalMGR import calmgr
things = parms["path"].split("/")[2:]
# print "ppw457:"+ repr(things)
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
# Parse the rest of the URL.
selection=getselection(things)
# If we want other arguments like order by
# we should parse them here
req = PRec()
retval = calmgr(req, selection)
print "-------\n"*3,"ppw469:", req._buff
self.wfile.write(req._buff)
return
if parms["path"] == "/calsearch.xml":
import searcher
cparms = {}
cparms.update(parms)
print "pproxy466:"+repr(cparms)
if "datalab" in parms:
cparms.update({"datalab":parms["datalab"][0]})
if "filename" in parms:
print "ppw481:", repr(parms["filename"])
cparms.update({"filename":parms["filename"][0]})
if "caltype" in parms:
cparms.update({"caltype":parms["caltype"][0]})
else:
cparms.update({"caltype":"processed_bias"})
buff = searcher.search(cparms)
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
self.wfile.write(buff)
return
if parms["path"].startswith("/globalcalsearch.xml"):
from prsproxyutil import calibration_search
flattenParms(parms)
resultb = None
resultf = None
if "caltype" in parms:
caltype = parms["caltype"]
if caltype == "processed_bias" or caltype == "all":
parms.update({"caltype":"processed_bias"})
resultb = calibration_search(parms, fullResult=True)
if caltype == "processed_flat" or caltype == "all":
parms.update({"caltype":"processed_flat"})
resultf = calibration_search(parms, fullResult = True)
if caltype == "all":
try:
domb = minidom.parseString(resultb)
domf = minidom.parseString(resultf)
except:
return None # can't parse input... no calibration
calnodefs = domf.getElementsByTagName("calibration")
if len(calnodefs) > 0:
calnodef = calnodefs[0]
else:
calnodef = None
calnodebs = domb.getElementsByTagName("dataset")
if len(calnodebs) > 0:
calnodeb = calnodebs[0]
#print calnodef.toxml()
#print calnodeb.toxml()
# domb.importNode(calnodef, True)
if calnodef and calnodeb:
calnodeb.appendChild(calnodef)
elif calnodef:
result=domb.toxml()
else:
result=domb.toxml()
result = domb.toxml()
print "prsw207:", result
self.send_response(200)
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
self.wfile.write(result)
return
if parms["path"] == "/recipecontent":
if "recipe" in parms:
recipe = parms["recipe"][0]
content = rl.retrieve_recipe(recipe)
self.send_response(200)
self.send_header('Content-type', 'text/plain')
self.end_headers()
self.wfile.write(content)
return
if parms["path"] == "/adinfo":
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
if "filename" not in parms:
return "Error: Need Filename Parameter"
if "filename" in parms:
try:
ad = AstroData(parms["filename"][0])
except:
self.wfile.write("Can't use AstroData to open %s" % parms["filename"])
return
if "fullpage" in parms:
self.wfile.write("<html><body>")
if "fullpage" not in parms:
# defaults to false
self.wfile.write("<b>Name</b>: %s \n" % os.path.basename(ad.filename))
self.wfile.write("<br/><b>Path</b>: %s \n" % os.path.abspath(ad.filename))
self.wfile.write("<br/><b>Types</b>: %s\n" % ", ".join(ad.types))
recdict = rl.get_applicable_recipes(ad, collate=True)
keys = recdict.keys()
keys.sort()
for key in keys:
recname = recdict[key]
self.wfile.write("<br/><b>Default Recipe(s)</b>:%s "+\
"(<i>due to type</i>: %s)" % (recname, key))
alldesc = ad.all_descriptors()
self.wfile.write("<br/><b>Descriptors</b>:\n")
self.wfile.write('<table style="margin-left:4em">\n')
adkeys = alldesc.keys()
adkeys.sort()
self.wfile.flush()
for desc in adkeys:
value = str(alldesc[desc])
if "ERROR" in value:
value = '<span style="color:red">' + value + '</span>'
self.wfile.write("<tr><td>%s</td><td>%s</td></tr>\n" % (desc, value))
self.wfile.flush()
self.wfile.write("</table>")
if "fullpage" in parms:
self.wfile.write("</body></html>")
return
if parms["path"] == "/recipes.xml":
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.send_header("Access-Control-Allow-Origin", "http://localhost")
self.end_headers()
self.wfile.write(rl.list_recipes(as_xml = True) )
return
if parms["path"] == "/reduceconfigs.xml":
import glob
rcfgs = glob.glob("./*.rcfg")
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
retxml = '<?xml version="1.0" encoding="UTF-8" ?>\n'
retxml += "<reduceconfigs>\n"
for rcfg in rcfgs:
retxml += """\t<reduceconfig name="%s"/>\n""" % rcfg
retxml += "</reduceconfigs>\n"
self.wfile.write(retxml)
return
if parms["path"].startswith("/datadir.xml"):
dirdict = self.getDirdict()
ds = dirdict.dataSpider
xml = dirdict.as_xml()
self.send_response(200)
self.send_header('Content-type', 'text/xml')
self.end_headers()
self.wfile.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
self.wfile.write("<datasetDict>\n")
self.wfile.write(xml)
self.wfile.write("</datasetDict>")
self.wfile.flush()
return
if parms["path"] == "/runreduce":
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write("<html><head></head><body>\n")
from StringIO import StringIO
rout = StringIO()
cmdlist = ["reduce", "--invoked", "--verbose=6"]
cmdlist.extend(parms["p"])
logdir = ".autologs"
if not os.path.exists(logdir):
os.mkdir(logdir)
reducelog = os.path.join(logdir,
"reduce-addcinvokedlog-%d%s" % (
os.getpid(), str(time.time())
))
f = open(reducelog, "w")
loglink = "reducelog-latest"
if os.path.exists(loglink):
os.remove(loglink)
os.symlink(reducelog, loglink)
# WARNING, this call had used Popen and selected on the
# subprocess.PIPE... now uses call there is kruft remaining
# (may move it back to old style soon but there was a bug)
print "adcc running: \n\t" + " ".join(cmdlist)
pid = subprocess.call( cmdlist,
stdout = f,
stderr = f)
self.wfile.write('<b style="font-size=150%">REDUCTION STARTED</b>')
self.wfile.write("<pre>")
# self.wfile.flush()
f.close()
f = open(reducelog, "r")
txt = f.read()
# pretty the text
ptxt = txt
if (True): # make pretty
ptxt = re.sub("STARTING RECIPE:(.*)\n",
'<b>STARTING RECIPE:</b><span style="color:blue">\g<1></span>\n', ptxt)
ptxt = re.sub("STARTING PRIMITIVE:(.*)\n",
'<i>STARTING PRIMITIVE:</i><span style="color:green">\g<1></span>\n', ptxt)
ptxt = re.sub("ENDING PRIMITIVE:(.*)\n",
'<i>ENDING PRIMITIVE:</i> <span style="color:green">\g<1></span>\n', ptxt)
ptxt = re.sub("ENDING RECIPE:(.*)\n",
'<b>ENDING RECIPE:</b> <span style="color:blue">\g<1></span>\n', ptxt)
ptxt = re.sub("(STATUS|INFO|FULLINFO|WARNING|CRITICAL|ERROR)(.*?)-(.*?)-",
'<span style="font-size:70%">\g<1>\g<2>-\g<3>- </span>', ptxt)
self.wfile.write(ptxt) # f.read())
f.close()
try:
while False:
error = False
while(True):
stdout = None
stderr = None
r,v,w = select.select([pid.stdout],[],[],.1)
print "prsw112:", repr(r)
if len(r):
stdout = r[0].read()
print "prsw487:", stdout
break;
else:
r,v,w = select.select([pid.stderr],[],[],.1)
if len(r):
stderr = pid.stderr.read()
print "prsw494:", stderr
break;
if stdout:
self.wfile.write(str(stdout))
if stderr:
self.wfile.write("{"+stderr+"}")
self.wfile.flush()
if pid.poll()!= None:
self.wfile.flush()
break
except:
print "PRSW516 EMERGENCY:"
self.wfile.write("</pre>")
if False:
r,v,x = select.select([pid.stderr], [], [], .1)
if len(r):
stderr = pid.stderr.read()
else:
stderr = None
# stderr = pid.stderr.read(100)
if stderr != None:
self.wfile.write("<b><pre>\n")
self.wfile.write(str(stderr))
self.wfile.write("</pre></b>")
self.wfile.write('<b style="font-size=150%">REDUCTION ENDED</b>')
self.wfile.write("\n</body></html>")
self.wfile.flush()
return
if self.path == "/reducelist": #our dynamic content
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
# this is the tag in head that autopolls if wanted
front = """
<html>
<head>
<meta http-equiv="refresh" content="2" />
</head>
<body>"""
page = front + """
%(body)s
<body>
</html>"""
self.wfile.write(page)
if True:
body = ""
body += "<b>date</b>: %s<br/>\n" \
% datetime.datetime.now().strftime("%A, %Y-%m-%d %H:%M:%S")
body += "<u>Reduce Instances</u><br/>\n"
body += "n.o. instances: %d\n" % rim.numinsts
body += "<ul>"
rdict = copy(rim.reducedict)
rpids = rim.reducedict.keys()
for rpid in rpids:
body += "<li>client pid = %d at port %d</li>\n" \
% (rpid, rdict[rpid]["port"])
body += "</ul>"
self.wfile.write(page % {"body":body})
self.wfile.flush()
return
if self.path == "/killprs":
import datetime
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write("Killed this prsproxy instance, pid = %d at %s" \
%(os.getpid(), str(datetime.datetime.now())))
webserverdone = True
return
if self.path.startswith("/displaycache"):
from CacheManager import get_cache_dir, get_cache_file
path = os.path.split(self.path)
print "prsw 569:", self.path
if len (path)>1:
slot = path[-1]
tfile = get_cache_file(slot)
try:
f = open(tfile)
except:
return
self.send_response(200)
self.send_header('Content-type', 'image/png')
self.end_headers()
while True:
t = f.read(102400)
if t == "":
self.wfile.flush()
break
self.wfile.write(t)
return
if self.path.startswith("/fullheader"):
realpath = self.path.split('/')
realpath = realpath[1:]
dirdict = self.getDirdict()
print "prsw514:", repr(realpath)
name = realpath[-1]
fname = dirdict.get_full_path(name)
ad = AstroData(fname)
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write("<html><body>\n")
self.wfile.write('<h2>%s</h2>\n' % name)
self.wfile.write(ad.infostr(as_html=True))
alld = ad.all_descriptors()
self.wfile.write(
"""
<table cellspacing="2px">
<COLGROUP align="right" />
<COLGROUP align="left" />
<thead>
<tr>
<td style="background-color:grey">Descriptor</td>
<td style="background-color:grey">Value</td>
</tr>
</thead>
""")
alldkeys = alld.keys()
alldkeys.sort()
for dname in alldkeys:
if type(alld[dname]) == str and "ERROR" in alld[dname]:
redval = '<span style="color:red">'+str(alld[dname])+"</span>"
dval = redval
else:
# print "ppw864:",type(alld[dname])
if not alld[dname].collapse_value():
import pprint
dval = """<pre>%s</pre> """ \
% pprint.pformat(alld[dname].dict_val, indent=4, width=80)
else:
dval = str(alld[dname])
self.wfile.write("""
<tr>
<td style="text-align:right;border-bottom:solid grey 1px">
%(dname)s =
</td>
<td style="border-bottom:solid grey 1px">
%(value)s
</td>
</tr>
""" % { "dname":dname,
"value":dval})
self.wfile.write("</table>")
self.wfile.write("</body></html>\n")
return
if self.path.startswith("/htmldocs"):
import FitsStorage
realpath = self.path.split('/')
realpath = realpath[1:]
dirname = os.path.dirname(FitsStorage.__file__)
fname = os.path.join(dirname, "htmldocroot", *realpath)
#print "psrw456: %s\n" % repr(fname)*10
fnamelocal = os.path.join(
os.path.dirname(fname),
"FS_LOCALMODE_"+os.path.basename(fname)
)
if os.path.exists(fnamelocal):
fname = fnamelocal
try:
f = open(fname, "r")
data = f.read()
print repr(data)
f.close()
except IOError:
data = "<b>NO SUCH RESOURCE FOUND</b>"
self.send_response(200)
if fname.endswith(".css"):
self.send_header('Content-type', "text/css")
elif fname.endswith(".png"):
self.send_header('Content-type', "image/png")
else:
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(data)
return
#what's the problem with this.
if self.path.startswith("/cmd_queue"):
self.counter += 1
data = str(self.counter)
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(data)
return
if self.path.startswith("/engineering"):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
if "rim" in ADCCHandler.informers:
rim = ADCCHandler.informers["rim"]
evman = rim.events_manager
import pprint
data = "<u>Events</u><br/><pre>"
data += "num events: %d\n" % len(evman.event_list)
for mevent in evman.event_list:
data += pprint.pformat(mevent)
data += "------------------------------\n"
data += "</pre>"
self.wfile.write(data)
return
if self.path.startswith("/event_report.json"):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
if "timestamp" in parms:
timestamp = parms["timestamp"][0]
else:
timestamp = 0
print "prsprox:",timestamp
self.wfile.write({"youbo":"mompun"})
return
if self.path.startswith("/qap"):
if ".." in self.path:
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
data = "<b>bad path error</b>"
self.wfile.write(data)
dirname = os.path.dirname(__file__)
joinlist = [dirname, "../../scripts/adcc_faceplate/"]
# Split out any parameters in the URL
self.path = self.path.split("?")[0]
#append any further directory info.
joinlist.append(self.path[5:])
fname = os.path.join(*joinlist)
self.log_message('"%s" %s %s', "Loading " + \
joinlist[1] + os.path.basename(fname),
203, '-')
try:
f = open(fname, "r")
data = f.read()
f.close()
except IOError:
data = "<b>NO SUCH RESOURCE AVAILABLE</b>"
self.send_response(200)
if self.path.endswith(".js"):
self.send_header('Content-type', 'text/javascript')
elif self.path.endswith(".css"):
self.send_header("Content-type", "text/css")
elif fname.endswith(".png"):
self.send_header('Content-type', "image/png")
else:
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(data)
return
else:
print "not qap"
if self.path == "/":
self.path = "/KitchenSink.html"
dirname = os.path.dirname(__file__)
fname = os.path.join(dirname, "pyjamaface/prsproxygui/output", *(self.path[1:]))
try:
f = open(fname, "r")
data = f.read()
f.close()
except IOError:
data = "<b>NO SUCH RESOURCE FOUND</b>"
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(data)
return
except IOError:
raise
print "handling IOError"
self.send_error(404,'File Not Found: %s' % self.path)
from astrodata import AstroData
from astrodata.structuredslice import pixel_exts,bintable_exts
# some files for tests, of === the one used
ffile = "../../../../test_data/gmosspect/gsN20011222S027.fits"
od = "../../../../test_data/gndeploy1/N20110826S0336.fits"
of = nf = "../../../../test_data/multibins.fits"
od = AstroData(of) #original od
print od.infostr()
pixad = od[pixel_exts]
print pixad.infostr()
binad = od[bintable_exts]
print binad.infostr()
print od.gain()
for ext in od:
try:
print "<<"*20
print "gain", ext.gain()
print ">>"*20
except:
print "failed on "
print ext.infostr()
raise