コード例 #1
0
ファイル: testutil.py プロジェクト: pyrrho314/recipesystem
def runinsert(index=None, f1=None, f2=None, auto=False):
    ad = AstroData(f1)
    md = AstroData(f2)
    pstr = "\n\n             >>>>>>>     AD     <<<<<<<<\n"
    pstr += str(ad.infostr())
    pstr += "\n\n             >>>>>>>    AD APPEND   <<<<<<<<\n"
    pstr += str(md.infostr())
    ad.insert(index=index, moredata=md, auto_number=auto)
    pstr +="\n\n             >>>>>>>  NEW AD <<<<<<<<\n"
    pstr += str(ad.infostr())
    print(pstr)
    return ad
コード例 #2
0
ファイル: prsproxyweb.py プロジェクト: pyrrho314/recipesystem
    def do_GET(self):
        global webserverdone
        self.state = ppwstate
        rim = self.informers["rim"]
        parms = parsepath(self.path)

        # Older revisions of adcc may not supply 'verbose' key
        try: 
            self.informers["verbose"]
        except KeyError: 
            self.informers["verbose"] = True

        try:
            if self.path == "/":
                page = """
                <html>
                <head>
                </head>
                <body>
                <h4>prsproxy engineering interface</h4>
                <ul>
                <li><a href="/engineering">Engeering Interface</a></li>
                <li><a href="qap/engineering.html">Engeering AJAX App</a></li>
                <li><a href="datadir">Data Directory View</a></li>
                <li><a href="killprs">Kill this server</a> (%(numinsts)d """ +\
                    """copies ofreduce registered)</li>
                </ul>
                <body>
                </html>"""
                page % {"numinsts":rim.numinsts}
                self.send_response(200)
                self.send_header("content-type", "text-html")
                self.end_headers()
                self.wfile.write(page)
                return
                
            if parms["path"].startswith("/rqlog.json"):
                self.send_response(200)
                self.send_header('Content-type', "application/json")
                self.end_headers()
                
                if "file" in parms:
                    logfile = parms["file"][0]
                    print logfile
                    if not os.path.exists(logfile):
                        msg = "Log file not available"
                    else:
                        f = open(logfile, "r")      
                        msg = f.read()
                        f.close()
                else:
                    msg = "No log file available"

                tdic = {"log":msg}

                self.wfile.write(json.dumps(tdic, sort_keys=True, indent=4))
                return
 
           # ------------------------------------------------------------------
           # Server time

            if parms["path"].startswith("/rqsite.json"):
                self.send_response(200)
                self.send_header('Content-type', "application/json")
                self.end_headers()
                tdic = server_time()
                self.wfile.write(json.dumps(tdic, sort_keys=True, indent=4))
                return

            # ------------------------------------------------------------------
            # Metrics query employing fitsstore

            if parms["path"].startswith("/cmdqueue.json"):
                self.send_header('Content-type', "application/json")
                self._handle_cmdqueue_json(rim, parms)
                return

            # ------------------------------------------------------------------
            
            if parms["path"].startswith("/cmdqueue.xml"):
                self.send_response(200)
                self.send_header('Content-type','text/xml')
                self.end_headers()
                
                if "lastcmd" in parms:
                    start = int(parms["lastcmd"][0])+1
                else:
                    start = 0   
                elist = self.state.rim.displayCmdHistory.peekSince(cmdNum=start)
                print "prsw 200:", repr(elist)
                xml = '<commandQueue lastCmd="%d">' % (start-1)
                for cmd in elist:
                    # this is because there should be only one top key
                    #   in the cmd dict
                    cmdname = cmd.keys()[0] 
                                            
                    cmdbody = cmd[cmdname]
                    xml += '<command name="%s">' % cmdname
                    
                    if "files" in cmdbody:
                    
                        basenames = cmdbody["files"].keys()
                        for basename in basenames:
                            fileitem = cmdbody["files"][basename]
                            if "url" not in fileitem or fileitem["url"] == None:
                                url = "None"
                            else:
                                url = fileitem["url"]
                            xml += """<file basename="%(basename)s"
                            url = "%(url)s"
                            cmdnum = "%(cn)d"/>""" % {
                                "basename": basename,
                                "url": "" if "file" not in fileitem else fileitem["url"],
                                "cn":int(cmdbody["cmdNum"])}
                                            
                            # now any extension in the extdict
                            if "extdict" in fileitem:
                                extdict = fileitem["extdict"]
                                for name in extdict.keys():
                                    xml += """\n<file basename="%(basename)s"
                                             url="%(url)s"
                                             ext="%(ext)s"
                                             cmdnum="%(cn)d"/>""" % {
                                            "basename": basename,
                                            "ext": name,
                                            "url": extdict[name],
                                            "cn":int(cmdbody["cmdNum"])}
 
                    xml += '</command>'
                xml += "</commandQueue>"
                self.wfile.write(xml)
                return 
                
            if parms["path"] == "/recipeindex.xml":
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                
                self.wfile.write(rl.getRecipeIndex(as_xml=True))
                return
             
            if parms["path"].startswith("/summary"):
                from fitsstore.FitsStorageWebSummary.Summary import summary
                from fitsstore.FitsStorageWebSummary.Selection import getselection
                
                selection = getselection({})
                
                rec =  PRec()
                summary(rec, "summary", selection, [], links=False)
                buff = rec._buff
                self.send_response(200)
                self.send_header("Content-type", "text/html")
                self.end_headers()
                self.wfile.write(buff)
                return
                
            if parms["path"].startswith("/calmgr"):
                from FitsStorageWebSummary.Selection import getselection
                from FitsStorageWebSummary.CalMGR import calmgr
                things = parms["path"].split("/")[2:]
                # print "ppw457:"+ repr(things)
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                
                # Parse the rest of the URL.
                selection=getselection(things)
            
                # If we want other arguments like order by
                # we should parse them here
                req = PRec()
                retval = calmgr(req, selection)
                print "-------\n"*3,"ppw469:", req._buff
                self.wfile.write(req._buff)
                return 
                
            if parms["path"] == "/calsearch.xml":
                import searcher
                cparms = {}
                cparms.update(parms)
                print "pproxy466:"+repr(cparms)
                if "datalab" in parms:
                    cparms.update({"datalab":parms["datalab"][0]})
                if "filename" in parms:
                    print "ppw481:", repr(parms["filename"])
                    cparms.update({"filename":parms["filename"][0]})
                if "caltype" in parms:
                    cparms.update({"caltype":parms["caltype"][0]})
                else:
                    cparms.update({"caltype":"processed_bias"})
                    
                buff = searcher.search(cparms)
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                
                self.wfile.write(buff)
                return 
                
            if parms["path"].startswith("/globalcalsearch.xml"):
                from prsproxyutil import calibration_search
                flattenParms(parms)
                resultb = None
                resultf = None
                
                if "caltype" in parms:
                    caltype = parms["caltype"]
                    if caltype == "processed_bias" or caltype == "all":
                        parms.update({"caltype":"processed_bias"})
                        resultb = calibration_search(parms, fullResult=True)
                    if caltype == "processed_flat" or caltype == "all":
                        parms.update({"caltype":"processed_flat"})
                        resultf = calibration_search(parms, fullResult = True)
                
                if caltype == "all":
                    try:
                        domb = minidom.parseString(resultb)
                        domf = minidom.parseString(resultf)
                    except:
                        return None # can't parse input... no calibration
                    calnodefs = domf.getElementsByTagName("calibration")
                    if len(calnodefs) > 0:
                        calnodef = calnodefs[0]
                    else:
                        calnodef = None
                    calnodebs = domb.getElementsByTagName("dataset")
                    if len(calnodebs) > 0:
                        calnodeb = calnodebs[0]
                    
                    #print calnodef.toxml()
                    #print calnodeb.toxml()
                    # domb.importNode(calnodef, True)
                    if calnodef and calnodeb:
                        calnodeb.appendChild(calnodef)
                    elif calnodef:
                        result=domb.toxml()                        
                    else:
                        result=domb.toxml()
                    result = domb.toxml()
                
                print "prsw207:", result
                self.send_response(200)
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                
                self.wfile.write(result)
                return
                
            if parms["path"] == "/recipecontent":
                if "recipe" in parms:
                    recipe = parms["recipe"][0]
                    content = rl.retrieve_recipe(recipe)
                    self.send_response(200)
                    self.send_header('Content-type', 'text/plain')
                    self.end_headers()

                    self.wfile.write(content)
                    return

            if parms["path"] == "/adinfo":
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()

                if "filename" not in parms:
                    return "Error: Need Filename Parameter"
                if "filename" in parms:
                    try:
                        ad = AstroData(parms["filename"][0])
                    except:
                        self.wfile.write("Can't use AstroData to open %s" % parms["filename"])
                        return
                    if "fullpage" in parms:
                        self.wfile.write("<html><body>")
                    if "fullpage" not in parms:
                    # defaults to false
                        self.wfile.write("<b>Name</b>: %s \n" % os.path.basename(ad.filename))
                        self.wfile.write("<br/><b>Path</b>: %s \n" % os.path.abspath(ad.filename))
                        self.wfile.write("<br/><b>Types</b>: %s\n" % ", ".join(ad.types))
                        recdict = rl.get_applicable_recipes(ad, collate=True)
                        keys = recdict.keys()
                        keys.sort()
                        for key in keys:
                            recname = recdict[key]                        
                            self.wfile.write("<br/><b>Default Recipe(s)</b>:%s "+\
                                             "(<i>due to type</i>: %s)" % (recname, key))
                        alldesc = ad.all_descriptors()
                        self.wfile.write("<br/><b>Descriptors</b>:\n")
                        self.wfile.write('<table style="margin-left:4em">\n')
                        adkeys = alldesc.keys()
                        adkeys.sort()
                        self.wfile.flush()
                        for desc in adkeys:
                            value = str(alldesc[desc])
                            if "ERROR" in value:
                                value = '<span style="color:red">' + value + '</span>'
                            self.wfile.write("<tr><td>%s</td><td>%s</td></tr>\n" % (desc, value))
                            self.wfile.flush()
                        self.wfile.write("</table>")
                    if "fullpage" in parms:
                        self.wfile.write("</body></html>")
                return
                
            if parms["path"] == "/recipes.xml":
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.send_header("Access-Control-Allow-Origin", "http://localhost")
                self.end_headers()
                self.wfile.write(rl.list_recipes(as_xml = True) )
                return

            if parms["path"] == "/reduceconfigs.xml":
                import glob
                rcfgs = glob.glob("./*.rcfg")
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                retxml = '<?xml version="1.0" encoding="UTF-8" ?>\n'
                retxml += "<reduceconfigs>\n"
                for rcfg in rcfgs:
                    retxml += """\t<reduceconfig name="%s"/>\n""" % rcfg
                retxml += "</reduceconfigs>\n"
                self.wfile.write(retxml)
                return

            if parms["path"].startswith("/datadir.xml"):
                dirdict = self.getDirdict()
                ds = dirdict.dataSpider
                xml = dirdict.as_xml()
                
                self.send_response(200)
                self.send_header('Content-type', 'text/xml')
                self.end_headers()
                
                self.wfile.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
                self.wfile.write("<datasetDict>\n")
                self.wfile.write(xml)
                self.wfile.write("</datasetDict>")
                self.wfile.flush()
                return

            if parms["path"] == "/runreduce":
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write("<html><head></head><body>\n")
                from StringIO import StringIO
                rout = StringIO()
                cmdlist = ["reduce", "--invoked", "--verbose=6"]
                cmdlist.extend(parms["p"])
                
                logdir = ".autologs"
                if not os.path.exists(logdir):
                    os.mkdir(logdir)

                reducelog = os.path.join(logdir, 
                                         "reduce-addcinvokedlog-%d%s" % (
                                             os.getpid(), str(time.time())
                                         ))
                f = open(reducelog, "w")
                
                loglink = "reducelog-latest"
                if os.path.exists(loglink):
                    os.remove(loglink)
                os.symlink(reducelog, loglink)
                            
                # WARNING, this call had used Popen and selected on the 
                # subprocess.PIPE... now uses call there is kruft remaining 
                # (may move it back to old style soon but there was a bug)

                print "adcc running: \n\t" + " ".join(cmdlist)
                pid = subprocess.call( cmdlist,
                                        stdout = f,
                                        stderr = f)
                
                self.wfile.write('<b style="font-size=150%">REDUCTION STARTED</b>')
                self.wfile.write("<pre>")
                # self.wfile.flush()
                f.close()
                f = open(reducelog, "r")      
                txt = f.read()
                # pretty the text
                ptxt = txt
                if (True): # make pretty
                    ptxt = re.sub("STARTING RECIPE:(.*)\n", 
                                  '<b>STARTING RECIPE:</b><span style="color:blue">\g<1></span>\n', ptxt)
                    ptxt = re.sub("STARTING PRIMITIVE:(.*)\n", 
                                  '<i>STARTING PRIMITIVE:</i><span style="color:green">\g<1></span>\n', ptxt)
                    ptxt = re.sub("ENDING PRIMITIVE:(.*)\n", 
                                  '<i>ENDING PRIMITIVE:</i>  <span style="color:green">\g<1></span>\n', ptxt)
                    ptxt = re.sub("ENDING RECIPE:(.*)\n", 
                                  '<b>ENDING RECIPE:</b>  <span style="color:blue">\g<1></span>\n', ptxt)
                    ptxt = re.sub("(STATUS|INFO|FULLINFO|WARNING|CRITICAL|ERROR)(.*?)-(.*?)-", 
                                  '<span style="font-size:70%">\g<1>\g<2>-\g<3>- </span>', ptxt)

                self.wfile.write(ptxt) # f.read())
                f.close()
                try:
                    while False:
                        error = False
                        while(True):
                            stdout = None
                            stderr = None
                            r,v,w = select.select([pid.stdout],[],[],.1)
                            print "prsw112:", repr(r)
                            if len(r):
                                stdout = r[0].read()
                                print "prsw487:", stdout
                                break;
                            else:
                                r,v,w = select.select([pid.stderr],[],[],.1)
                                if len(r):
                                    stderr = pid.stderr.read()
                                    print "prsw494:", stderr
                                    break;

                        if stdout:
                            self.wfile.write(str(stdout))
                        if stderr:
                            self.wfile.write("{"+stderr+"}")

                        self.wfile.flush()
                        if pid.poll()!= None:
                            self.wfile.flush()
                            break
                except:
                    print "PRSW516 EMERGENCY:"
                    
                self.wfile.write("</pre>")
                
                if False:
                    r,v,x = select.select([pid.stderr], [], [], .1)
                    if len(r):
                        stderr = pid.stderr.read()
                    else:
                        stderr = None
                # stderr = pid.stderr.read(100)
                    if stderr != None:
                        self.wfile.write("<b><pre>\n")
                        self.wfile.write(str(stderr))
                        self.wfile.write("</pre></b>")
                self.wfile.write('<b style="font-size=150%">REDUCTION ENDED</b>')
                self.wfile.write("\n</body></html>")
                self.wfile.flush()

                return
            
            if self.path == "/reducelist": #our dynamic content
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()

                # this is the tag in head that autopolls if wanted
                front = """
                <html>
                <head>
                    <meta http-equiv="refresh" content="2" />
                </head>
                <body>"""
                page = front + """
                %(body)s
                <body>
                </html>"""

                self.wfile.write(page)
                if True:
                    body = ""
                    body += "<b>date</b>: %s<br/>\n" \
                            % datetime.datetime.now().strftime("%A, %Y-%m-%d %H:%M:%S")
                    body += "<u>Reduce Instances</u><br/>\n"
                    body += "n.o. instances: %d\n" % rim.numinsts 
                    body += "<ul>"
                    rdict = copy(rim.reducedict)
                    rpids = rim.reducedict.keys()
                    for rpid in rpids:
                        body += "<li>client pid = %d at port %d</li>\n" \
                                % (rpid, rdict[rpid]["port"])
                    body += "</ul>"
                    self.wfile.write(page % {"body":body})
                    self.wfile.flush()
                return 

            if self.path == "/killprs":
                import datetime
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write("Killed this prsproxy instance, pid = %d at %s" \
                                 %(os.getpid(), str(datetime.datetime.now())))
                webserverdone = True
                return
            
            if self.path.startswith("/displaycache"):
                from CacheManager import get_cache_dir, get_cache_file
                
                path = os.path.split(self.path)
                print "prsw 569:", self.path
                if len (path)>1:
                    slot = path[-1]
                    tfile = get_cache_file(slot)
                    
                    try:
                        f = open(tfile)
                    except:
                        return
                    self.send_response(200)
                    self.send_header('Content-type', 'image/png')
                    self.end_headers()

                    while True:
                        t = f.read(102400)
                        if t == "":
                            self.wfile.flush()
                            break
                        self.wfile.write(t)
                return

            if self.path.startswith("/fullheader"):
                realpath = self.path.split('/')
                realpath = realpath[1:]
                
                dirdict = self.getDirdict()
                print "prsw514:", repr(realpath)
                
                name = realpath[-1]
                fname = dirdict.get_full_path(name)
                ad = AstroData(fname)

                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
        
                self.wfile.write("<html><body>\n")
                self.wfile.write('<h2>%s</h2>\n' % name)
                self.wfile.write(ad.infostr(as_html=True))
                alld = ad.all_descriptors()
                self.wfile.write(
                        """
                        <table cellspacing="2px">
                        <COLGROUP align="right" />
                        <COLGROUP align="left" />
                        <thead>
                        <tr>
                        <td style="background-color:grey">Descriptor</td>
                        <td style="background-color:grey">Value</td>
                        </tr>
                        </thead>
                        """)
                alldkeys = alld.keys()
                alldkeys.sort()
                for dname in alldkeys:
                    
                    if type(alld[dname]) == str and "ERROR" in alld[dname]:
                        redval = '<span  style="color:red">'+str(alld[dname])+"</span>"
                        dval = redval
                    else:
                        # print "ppw864:",type(alld[dname])
                        if not alld[dname].collapse_value():
                            import pprint
                            dval = """<pre>%s</pre> """ \
                                   % pprint.pformat(alld[dname].dict_val, indent=4, width=80)
                        else:
                            dval = str(alld[dname])
                    self.wfile.write("""
                        <tr>
                        <td style="text-align:right;border-bottom:solid grey 1px">
                        %(dname)s =
                        </td>
                        <td style="border-bottom:solid grey 1px">
                        %(value)s
                        </td>
                        </tr>
                        """ % { "dname":dname,
                                "value":dval})
                self.wfile.write("</table>")
                self.wfile.write("</body></html>\n")
                                
                return
                
            if self.path.startswith("/htmldocs"):
                import FitsStorage
                realpath = self.path.split('/')
                realpath = realpath[1:]
                dirname = os.path.dirname(FitsStorage.__file__)
                fname = os.path.join(dirname, "htmldocroot", *realpath)
                #print "psrw456: %s\n" % repr(fname)*10
                fnamelocal = os.path.join(
                                os.path.dirname(fname),
                                "FS_LOCALMODE_"+os.path.basename(fname)
                                )
                if os.path.exists(fnamelocal):
                    fname = fnamelocal
                try:
                    f = open(fname, "r")
                    data = f.read()
                    print repr(data)
                    f.close()
                except IOError:
                    data = "<b>NO SUCH RESOURCE FOUND</b>"
                self.send_response(200)
                if fname.endswith(".css"):
                    self.send_header('Content-type', "text/css")
                elif fname.endswith(".png"):
                    self.send_header('Content-type', "image/png")
                else:
                    self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(data)
                return
                
            #what's the problem with this.
            if self.path.startswith("/cmd_queue"):
                self.counter += 1
                data = str(self.counter)
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(data)
                return 
                
            if self.path.startswith("/engineering"):
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                
                if "rim" in ADCCHandler.informers:
                    rim = ADCCHandler.informers["rim"]
                    evman = rim.events_manager
                    import pprint
                    data = "<u>Events</u><br/><pre>"
                    data += "num events: %d\n" % len(evman.event_list)
                    for mevent in evman.event_list:
                        data += pprint.pformat(mevent)
                        data += "------------------------------\n"
                    data += "</pre>"
                    self.wfile.write(data)
                return
            
            if self.path.startswith("/event_report.json"):
                self.send_response(200)
                self.send_header('Content-type', 'text/html')
                self.end_headers()
                
                if "timestamp" in parms:
                    timestamp = parms["timestamp"][0]
                else:
                    timestamp  = 0
                print "prsprox:",timestamp
                self.wfile.write({"youbo":"mompun"})
                return
                
            if self.path.startswith("/qap"):
                if ".." in self.path:
                    self.send_response(200)
                    self.send_header('Content-type', 'text/html')
                    self.end_headers()
                    data = "<b>bad path error</b>"
                    self.wfile.write(data)
                dirname = os.path.dirname(__file__)
                joinlist = [dirname, "../../scripts/adcc_faceplate/"]
                
                # Split out any parameters in the URL
                self.path = self.path.split("?")[0]

                #append any further directory info.
                joinlist.append(self.path[5:])
                fname = os.path.join(*joinlist)
                self.log_message('"%s" %s %s', "Loading " + \
                                 joinlist[1] + os.path.basename(fname), 
                                 203, '-')
                try:
                    f = open(fname, "r")
                    data = f.read()
                    f.close()
                except IOError:
                    data = "<b>NO SUCH RESOURCE AVAILABLE</b>"
                self.send_response(200)
                if  self.path.endswith(".js"):
                    self.send_header('Content-type', 'text/javascript')
                elif self.path.endswith(".css"):
                    self.send_header("Content-type", "text/css")
                elif fname.endswith(".png"):
                    self.send_header('Content-type', "image/png")
                else:
                    self.send_header('Content-type', 'text/html')
                self.end_headers()
                self.wfile.write(data)
                return 
            else:
                print "not qap"    
            if self.path == "/":
                self.path = "/KitchenSink.html"
                
            dirname = os.path.dirname(__file__)
            fname = os.path.join(dirname, "pyjamaface/prsproxygui/output", *(self.path[1:]))
            
            try:
                f = open(fname, "r")
                data = f.read()
                f.close()
            except IOError:
                data = "<b>NO SUCH RESOURCE FOUND</b>"
                
            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.end_headers()
            self.wfile.write(data)
            return 
        except IOError:
            raise
            print "handling IOError"
            self.send_error(404,'File Not Found: %s' % self.path)
コード例 #3
0
ファイル: tstructures.py プロジェクト: pyrrho314/recipesystem
from astrodata import AstroData
from astrodata.structuredslice import pixel_exts,bintable_exts

# some files for tests, of === the one used
ffile = "../../../../test_data/gmosspect/gsN20011222S027.fits"
od = "../../../../test_data/gndeploy1/N20110826S0336.fits"
of = nf = "../../../../test_data/multibins.fits"


od = AstroData(of) #original od
print od.infostr()


pixad = od[pixel_exts]
print pixad.infostr()

binad = od[bintable_exts]
print binad.infostr()

print od.gain()
for ext in od:
    try:
        print "<<"*20
        print "gain", ext.gain()
        
        print ">>"*20
    except:
        print "failed on "
        print ext.infostr()
        raise