def __init__(self, wsdir, wsid, writing=False, reading=True):
self.wsid = wsid
self.qdir = os.path.join(wsdir, str(self.wsid))
if writing:
FileEnqueue.recover(self.qdir)
self.enq = FileEnqueue(self.qdir, buffer=200, gzip=None)
else:
self.enq = DummyFileEnqueue(self.qdir)
if reading:
self.deq = FileDequeue(self.qdir)
else:
self.deq = DummyFileDequeue(self.qdir, enq=self.enq)
self.running = True
self.scheduledcount = 0
self.checkedoutcount = 0
self.finishedcount = 0
self.activecount = 0
class WorkSet(object):
def __init__(self, wsdir, wsid, writing=False, reading=True):
self.wsid = wsid
self.qdir = os.path.join(wsdir, str(self.wsid))
if writing:
FileEnqueue.recover(self.qdir)
self.enq = FileEnqueue(self.qdir, buffer=200, gzip=None)
else:
self.enq = DummyFileEnqueue(self.qdir)
if reading:
self.deq = FileDequeue(self.qdir)
else:
self.deq = DummyFileDequeue(self.qdir, enq=self.enq)
self.running = True
self.scheduledcount = 0
self.checkedoutcount = 0
self.finishedcount = 0
self.activecount = 0
def flush(self):
# _flush() should be part of close(), but not now
self.enq._flush()
self.enq.close()
def shutdown(self):
self.flush()
if self.deq:
self.deq.close()
def get_status(self):
r = dict(id=self.wsid, running=self.running,
scheduled=self.scheduledcount,
checkedout=self.checkedoutcount,
finished=self.finishedcount
)
if self.enq: r['enq'] = self.enq.get_status()
if self.deq: r['deq'] = self.deq.get_status()
return r
def schedule(self, curi):
self.enq.queue(curi)
self.scheduledcount += 1
def checkout(self, n):
if not self.running:
return []
r = []
while len(r) < n:
curi = self.deq.get(timeout=0.001)
if curi is None:
# avoid flushing queue too frequently
if self.enq.queue_count > 10000 or self.enq.age() > 1.0:
self.enq.close()
break
r.append(curi)
self.checkedoutcount += len(r)
return r
def deschedule(self, furi):
self.finishedcount += 1
