def dump(self): if not self.filename: return if not self.qproxy and self.filename.startswith("qserve://"): fn = self.filename[len("qserve://"):] host, jobid=fn.split("/") try: jobid = int(jobid) except ValueError: jobid = jobid.strip('"') if ":" in host: host, port = host.split(":") port = int(port) else: port = 14311 from mwlib.async import rpcclient self.qproxy = rpcclient.serverproxy(host=host, port=port) self.jobid = jobid if self.qproxy: self.qproxy.qsetinfo(jobid=self.jobid, info=self.status) return try: open(self.filename + '.tmp', 'wb').write( json.dumps(self.status).encode('utf-8') ) os.rename(self.filename + '.tmp', self.filename) except Exception, exc: log.ERROR('Could not write status file %r: %s' % ( self.filename, exc ))
def dump(self): if not self.filename: return if not self.qproxy and self.filename.startswith("qserve://"): fn = self.filename[len("qserve://"):] host, jobid = fn.split("/") try: jobid = int(jobid) except ValueError: jobid = jobid.strip('"') if ":" in host: host, port = host.split(":") port = int(port) else: port = 14311 from mwlib. async import rpcclient self.qproxy = rpcclient.serverproxy(host=host, port=port) self.jobid = jobid if self.qproxy: self.qproxy.qsetinfo(jobid=self.jobid, info=self.status) return try: open(self.filename + '.tmp', 'wb').write(json.dumps(self.status).encode('utf-8')) os.rename(self.filename + '.tmp', self.filename) except Exception as exc: log.ERROR('Could not write status file %r: %s' % (self.filename, exc))
def dispatch(self, request): try: command = request.params['command'] except KeyError: log.error("no command given") raise HTTPResponse("no command given", status=400) try: method = getattr(self, 'do_%s' % command) except AttributeError: log.error("no such command: %r" % (command, )) raise HTTPResponse("no such command: %r" % (command, ), status=400) collection_id = request.params.get('collection_id') if not collection_id: collection_id = self.new_collection(request.params) is_new = True else: is_new = False if not self.check_collection_id(collection_id): raise HTTPResponse(status=404) try: qserve = collid2qserve[collection_id] except KeyError: qserve = choose_idle_qserve() if qserve is None: return self.error_response( "system overloaded. please try again later.", queue_full=1) collid2qserve[collection_id] = qserve self.qserve = rpcclient.serverproxy(host=qserve[0], port=qserve[1]) try: return method(collection_id, request.params, is_new) except Exception, exc: print "ERROR while dispatching %r: %s" % ( command, dict(collection_id=collection_id, is_new=is_new, qserve=qserve)) traceback.print_exc() if command == "download": raise exc return self.error_response('error executing command %r: %s' % ( command, exc, ))
def dispatch(self, request): try: command = request.params['command'] except KeyError: log.error("no command given") raise HTTPResponse("no command given", status=400) try: method = getattr(self, 'do_%s' % command) except AttributeError: log.error("no such command: %r" % (command, )) raise HTTPResponse("no such command: %r" % (command, ), status=400) collection_id = request.params.get('collection_id') if not collection_id: collection_id = self.new_collection(request.params) is_new = True else: is_new = False if not self.check_collection_id(collection_id): raise HTTPResponse(status=404) try: qserve = collid2qserve[collection_id] except KeyError: qserve = choose_idle_qserve() if qserve is None: return self.error_response( "system overloaded. please try again later.", queue_full=1) collid2qserve[collection_id] = qserve self.qserve = rpcclient.serverproxy(host=qserve[0], port=qserve[1]) try: return method(collection_id, request.params, is_new) except Exception as exc: print "ERROR while dispatching %r: %s" % (command, dict( collection_id=collection_id, is_new=is_new, qserve=qserve)) traceback.print_exc() if command == "download": raise exc return self.error_response('error executing command %r: %s' % ( command, exc,))
def report_mwzip_status(posturl, jobid, host, port): podclient = PODClient(posturl) status = Status(podclient=podclient) from mwlib.async import rpcclient sp = rpcclient.serverproxy(host, port) last = {} while 1: res = sp.qinfo(jobid=jobid) or {} done = res.get("done", False) if done: break info = res.get("info", {}) if info!=last: status(status=info.get("status", "fetching"), progress=info.get("progress", 0.0)) last = info else: gevent.sleep(0.5)
def _serverproxy(self): return rpcclient.serverproxy(host=self.host, port=self.port)
return result return Response(json.dumps(result), content_type='application/json') return wrapper from mwlib import lrucache busy=dict() collid2qserve = lrucache.lrucache(4000) def wait_idle((host, port), busy): ident = (host, port) busy[ident] = True numerrors = 0 qserve = rpcclient.serverproxy(host=host, port=port) while 1: try: stats = qserve.getstats() numerrors = 0 numrender = stats.get("busy", {}).get("render", 0) if numrender>10: if not busy[ident]: print "SYSTEM OVERLOADED on %r" % (ident, ) busy[ident] = True else: if busy[ident]: print "RESUMING OPERATION on %r" % (ident, ) busy[ident] = False