class JobPool(object):
POOL_SIZE = 5
def __init__(self,
repo,
repo_uri,
jobs_done=True,
poolsize=POOL_SIZE,
queuesize=None):
self.jobs_done = jobs_done
self.queue = AsyncQueue(queuesize or 0)
if self.jobs_done:
self.done = AsyncQueue()
for i in range(poolsize):
rep = repo.copy()
thread = threading.Thread(target=self._job_thread,
args=(rep, repo_uri))
thread.setDaemon(True)
thread.start()
def _job_thread(self, repo, repo_uri):
while True:
job = self.queue.get()
job.run(repo, repo_uri)
self.queue.done()
if self.jobs_done:
self.done.put(job)
def push(self, job):
self.queue.put(job)
# Default timeout is 5 minutes
def get_next_done(self, timeout=(5 * 60)):
if not self.jobs_done:
return None
try:
job = self.done.get(timeout)
self.done.done()
return job
except TimeOut:
return None
def get_next_done_unlocked(self):
if not self.jobs_done:
return None
if self.done.empty_unlocked():
return None
return self.done.get_unlocked()
def join(self):
self.queue.join()
class JobPool(object):
POOL_SIZE = 5
def __init__(self, repo, repo_uri, jobs_done=True, poolsize=POOL_SIZE,
queuesize=None):
self.jobs_done = jobs_done
self.queue = AsyncQueue(queuesize or 0)
if self.jobs_done:
self.done = AsyncQueue()
for i in range(poolsize):
rep = repo.copy()
thread = threading.Thread(target=self._job_thread,
args=(rep, repo_uri))
thread.setDaemon(True)
thread.start()
def _job_thread(self, repo, repo_uri):
while True:
job = self.queue.get()
job.run(repo, repo_uri)
self.queue.done()
if self.jobs_done:
self.done.put(job)
def push(self, job):
self.queue.put(job)
# Default timeout is 5 minutes
def get_next_done(self, timeout=(5 * 60)):
if not self.jobs_done:
return None
try:
job = self.done.get(timeout)
self.done.done()
return job
except TimeOut:
return None
def get_next_done_unlocked(self):
if not self.jobs_done:
return None
if self.done.empty_unlocked():
return None
return self.done.get_unlocked()
def join(self):
self.queue.join()
