def dump(self):
if not self.filename:
return
if not self.qproxy and self.filename.startswith("qserve://"):
fn = self.filename[len("qserve://"):]
host, jobid=fn.split("/")
try:
jobid = int(jobid)
except ValueError:
jobid = jobid.strip('"')
if ":" in host:
host, port = host.split(":")
port = int(port)
else:
port = 14311
from mwlib.async import rpcclient
self.qproxy = rpcclient.serverproxy(host=host, port=port)
self.jobid = jobid
if self.qproxy:
self.qproxy.qsetinfo(jobid=self.jobid, info=self.status)
return
try:
open(self.filename + '.tmp', 'wb').write(
json.dumps(self.status).encode('utf-8')
)
os.rename(self.filename + '.tmp', self.filename)
except Exception, exc:
log.ERROR('Could not write status file %r: %s' % (
self.filename, exc
))
def dump(self):
if not self.filename:
return
if not self.qproxy and self.filename.startswith("qserve://"):
fn = self.filename[len("qserve://"):]
host, jobid = fn.split("/")
try:
jobid = int(jobid)
except ValueError:
jobid = jobid.strip('"')
if ":" in host:
host, port = host.split(":")
port = int(port)
else:
port = 14311
from mwlib. async import rpcclient
self.qproxy = rpcclient.serverproxy(host=host, port=port)
self.jobid = jobid
if self.qproxy:
self.qproxy.qsetinfo(jobid=self.jobid, info=self.status)
return
try:
open(self.filename + '.tmp',
'wb').write(json.dumps(self.status).encode('utf-8'))
os.rename(self.filename + '.tmp', self.filename)
except Exception as exc:
log.ERROR('Could not write status file %r: %s' %
(self.filename, exc))
def dispatch(self, request):
try:
command = request.params['command']
except KeyError:
log.error("no command given")
raise HTTPResponse("no command given", status=400)
try:
method = getattr(self, 'do_%s' % command)
except AttributeError:
log.error("no such command: %r" % (command, ))
raise HTTPResponse("no such command: %r" % (command, ), status=400)
collection_id = request.params.get('collection_id')
if not collection_id:
collection_id = self.new_collection(request.params)
is_new = True
else:
is_new = False
if not self.check_collection_id(collection_id):
raise HTTPResponse(status=404)
try:
qserve = collid2qserve[collection_id]
except KeyError:
qserve = choose_idle_qserve()
if qserve is None:
return self.error_response(
"system overloaded. please try again later.", queue_full=1)
collid2qserve[collection_id] = qserve
self.qserve = rpcclient.serverproxy(host=qserve[0], port=qserve[1])
try:
return method(collection_id, request.params, is_new)
except Exception, exc:
print "ERROR while dispatching %r: %s" % (
command,
dict(collection_id=collection_id, is_new=is_new,
qserve=qserve))
traceback.print_exc()
if command == "download":
raise exc
return self.error_response('error executing command %r: %s' % (
command,
exc,
))
def dispatch(self, request):
try:
command = request.params['command']
except KeyError:
log.error("no command given")
raise HTTPResponse("no command given", status=400)
try:
method = getattr(self, 'do_%s' % command)
except AttributeError:
log.error("no such command: %r" % (command, ))
raise HTTPResponse("no such command: %r" % (command, ), status=400)
collection_id = request.params.get('collection_id')
if not collection_id:
collection_id = self.new_collection(request.params)
is_new = True
else:
is_new = False
if not self.check_collection_id(collection_id):
raise HTTPResponse(status=404)
try:
qserve = collid2qserve[collection_id]
except KeyError:
qserve = choose_idle_qserve()
if qserve is None:
return self.error_response(
"system overloaded. please try again later.", queue_full=1)
collid2qserve[collection_id] = qserve
self.qserve = rpcclient.serverproxy(host=qserve[0], port=qserve[1])
try:
return method(collection_id, request.params, is_new)
except Exception as exc:
print "ERROR while dispatching %r: %s" % (command, dict(
collection_id=collection_id, is_new=is_new, qserve=qserve))
traceback.print_exc()
if command == "download":
raise exc
return self.error_response('error executing command %r: %s' % (
command, exc,))
def report_mwzip_status(posturl, jobid, host, port):
podclient = PODClient(posturl)
status = Status(podclient=podclient)
from mwlib.async import rpcclient
sp = rpcclient.serverproxy(host, port)
last = {}
while 1:
res = sp.qinfo(jobid=jobid) or {}
done = res.get("done", False)
if done:
break
info = res.get("info", {})
if info!=last:
status(status=info.get("status", "fetching"),
progress=info.get("progress", 0.0))
last = info
else:
gevent.sleep(0.5)
def _serverproxy(self):
return rpcclient.serverproxy(host=self.host, port=self.port)
def _serverproxy(self):
return rpcclient.serverproxy(host=self.host, port=self.port)
return result
return Response(json.dumps(result), content_type='application/json')
return wrapper
from mwlib import lrucache
busy=dict()
collid2qserve = lrucache.lrucache(4000)
def wait_idle((host, port), busy):
ident = (host, port)
busy[ident] = True
numerrors = 0
qserve = rpcclient.serverproxy(host=host, port=port)
while 1:
try:
stats = qserve.getstats()
numerrors = 0
numrender = stats.get("busy", {}).get("render", 0)
if numrender>10:
if not busy[ident]:
print "SYSTEM OVERLOADED on %r" % (ident, )
busy[ident] = True
else:
if busy[ident]:
print "RESUMING OPERATION on %r" % (ident, )
busy[ident] = False