class JobPool(object): POOL_SIZE = 5 def __init__(self, repo, repo_uri, jobs_done=True, poolsize=POOL_SIZE, queuesize=None): self.jobs_done = jobs_done self.queue = AsyncQueue(queuesize or 0) if self.jobs_done: self.done = AsyncQueue() for i in range(poolsize): rep = repo.copy() thread = threading.Thread(target=self._job_thread, args=(rep, repo_uri)) thread.setDaemon(True) thread.start() def _job_thread(self, repo, repo_uri): while True: job = self.queue.get() job.run(repo, repo_uri) self.queue.done() if self.jobs_done: self.done.put(job) def push(self, job): self.queue.put(job) # Default timeout is 5 minutes def get_next_done(self, timeout=(5 * 60)): if not self.jobs_done: return None try: job = self.done.get(timeout) self.done.done() return job except TimeOut: return None def get_next_done_unlocked(self): if not self.jobs_done: return None if self.done.empty_unlocked(): return None return self.done.get_unlocked() def join(self): self.queue.join()