class WorkerQueue(object): def __init__(self, num_workers = 20): self.queue = Queue() self.pool = [] self._setup_workers(num_workers) def _setup_workers(self, num_workers): """ Sets up the worker threads NOTE: undefined behaviour if you call this again. """ self.pool = [] for _ in range(num_workers): self.pool.append(Thread(target=self.threadloop)) for a_thread in self.pool: a_thread.setDaemon(True) a_thread.start() def do(self, f, *args, **kwArgs): """ puts a function on a queue for running later. """ self.queue.put((f, args, kwArgs)) def stop(self): """ Stops the WorkerQueue, waits for all of the threads to finish up. """ self.queue.put(STOP) for thread in self.pool: thread.join() def threadloop(self): #, finish = False): """ Loops until all of the tasks are finished. """ while True: args = self.queue.get() if args is STOP: self.queue.put(STOP) self.queue.task_done() break else: try: args[0](*args[1], **args[2]) finally: # clean up the queue, raise the exception. self.queue.task_done() #raise def wait(self): """ waits until all tasks are complete. """ self.queue.join()