class AdaptiveThreadPool(object): """A thread pool which processes WorkItems from a queue. Attributes: requeue: The requeue instance which holds work items for this thread pool. """ def __init__(self, num_threads, queue_size=None, base_thread_name=None, worker_thread_factory=WorkerThread, queue_factory=Queue.Queue): """Initialize an AdaptiveThreadPool. An adaptive thread pool executes WorkItems using a number of WorkerThreads. WorkItems represent items of work that may succeed, soft fail, or hard fail. In addition, a completed work item can signal this AdaptiveThreadPool to enable more or fewer threads. Initially one thread is active. Soft failures are reqeueud to be retried. Hard failures cause this AdaptiveThreadPool to shut down entirely. See the WorkItem class for more details. Args: num_threads: The number of threads to use. queue_size: The size of the work item queue to use. base_thread_name: A string from which worker thread names are derived. worker_thread_factory: A factory which procudes WorkerThreads. queue_factory: Used for dependency injection. """ if queue_size is None: queue_size = num_threads self.requeue = ReQueue(queue_size, queue_factory=queue_factory) self.__thread_gate = ThreadGate(num_threads) self.__num_threads = num_threads self.__threads = [] for i in xrange(num_threads): thread = worker_thread_factory(self, self.__thread_gate) if base_thread_name: base = base_thread_name else: base = thread.__class__.__name__ thread.name = '%s-%d' % (base, i) self.__threads.append(thread) thread.start() def num_threads(self): """Return the number of threads in this thread pool.""" return self.__num_threads def Threads(self): """Yields the registered threads.""" for thread in self.__threads: yield thread def SubmitItem(self, item, block=True, timeout=0.0): """Submit a WorkItem to the AdaptiveThreadPool. Args: item: A WorkItem instance. block: Whether to block on submitting if the submit queue is full. timeout: Time wait for room in the queue if block is True, 0.0 to block indefinitely. Raises: Queue.Full if the submit queue is full. """ self.requeue.put(item, block=block, timeout=timeout) def QueuedItemCount(self): """Returns the number of items currently in the queue.""" return self.requeue.qsize() def Shutdown(self): """Shutdown the thread pool. Tasks may remain unexecuted in the submit queue. """ while not self.requeue.empty(): try: unused_item = self.requeue.get_nowait() self.requeue.task_done() except Queue.Empty: pass for thread in self.__threads: thread.exit_flag = True self.requeue.put(_THREAD_SHOULD_EXIT) self.__thread_gate.EnableAllThreads() def Wait(self): """Wait until all work items have been completed.""" self.requeue.join() def JoinThreads(self): """Wait for all threads to exit.""" for thread in self.__threads: logger.debug('Waiting for %s to exit' % str(thread)) thread.join() def CheckErrors(self): """Output logs for any errors that occurred in the worker threads.""" for thread in self.__threads: thread.CheckError()
class AdaptiveThreadPool(object): """A thread pool which processes WorkItems from a queue. Attributes: requeue: The requeue instance which holds work items for this thread pool. """ def __init__(self, num_threads, queue_size=None, base_thread_name=None, worker_thread_factory=WorkerThread, queue_factory=queue.Queue): """Initialize an AdaptiveThreadPool. An adaptive thread pool executes WorkItems using a number of WorkerThreads. WorkItems represent items of work that may succeed, soft fail, or hard fail. In addition, a completed work item can signal this AdaptiveThreadPool to enable more or fewer threads. Initially one thread is active. Soft failures are reqeueud to be retried. Hard failures cause this AdaptiveThreadPool to shut down entirely. See the WorkItem class for more details. Args: num_threads: The number of threads to use. queue_size: The size of the work item queue to use. base_thread_name: A string from which worker thread names are derived. worker_thread_factory: A factory which procudes WorkerThreads. queue_factory: Used for dependency injection. """ if queue_size is None: queue_size = num_threads self.requeue = ReQueue(queue_size, queue_factory=queue_factory) self.__thread_gate = ThreadGate(num_threads) self.__num_threads = num_threads self.__threads = [] for i in range(num_threads): thread = worker_thread_factory(self, self.__thread_gate) if base_thread_name: base = base_thread_name else: base = thread.__class__.__name__ thread.name = '%s-%d' % (base, i) self.__threads.append(thread) thread.start() def num_threads(self): """Return the number of threads in this thread pool.""" return self.__num_threads def Threads(self): """Yields the registered threads.""" for thread in self.__threads: yield thread def SubmitItem(self, item, block=True, timeout=0.0): """Submit a WorkItem to the AdaptiveThreadPool. Args: item: A WorkItem instance. block: Whether to block on submitting if the submit queue is full. timeout: Time wait for room in the queue if block is True, 0.0 to block indefinitely. Raises: Queue.Full if the submit queue is full. """ self.requeue.put(item, block=block, timeout=timeout) def QueuedItemCount(self): """Returns the number of items currently in the queue.""" return self.requeue.qsize() def Shutdown(self): """Shutdown the thread pool. Tasks may remain unexecuted in the submit queue. """ while not self.requeue.empty(): try: unused_item = self.requeue.get_nowait() self.requeue.task_done() except queue.Empty: pass for thread in self.__threads: thread.exit_flag = True self.requeue.put(_THREAD_SHOULD_EXIT) self.__thread_gate.EnableAllThreads() def Wait(self): """Wait until all work items have been completed.""" self.requeue.join() def JoinThreads(self): """Wait for all threads to exit.""" for thread in self.__threads: logger.debug('Waiting for %s to exit' % str(thread)) thread.join() def CheckErrors(self): """Output logs for any errors that occurred in the worker threads.""" for thread in self.__threads: thread.CheckError()