def __init__(self,
num_threads,
queue_size=None,
base_thread_name=None,
worker_thread_factory=WorkerThread,
queue_factory=queue.Queue):
"""Initialize an AdaptiveThreadPool.
An adaptive thread pool executes WorkItems using a number of
WorkerThreads. WorkItems represent items of work that may
succeed, soft fail, or hard fail. In addition, a completed work
item can signal this AdaptiveThreadPool to enable more or fewer
threads. Initially one thread is active. Soft failures are
reqeueud to be retried. Hard failures cause this
AdaptiveThreadPool to shut down entirely. See the WorkItem class
for more details.
Args:
num_threads: The number of threads to use.
queue_size: The size of the work item queue to use.
base_thread_name: A string from which worker thread names are derived.
worker_thread_factory: A factory which procudes WorkerThreads.
queue_factory: Used for dependency injection.
"""
if queue_size is None:
queue_size = num_threads
self.requeue = ReQueue(queue_size, queue_factory=queue_factory)
self.__thread_gate = ThreadGate(num_threads)
self.__num_threads = num_threads
self.__threads = []
for i in range(num_threads):
thread = worker_thread_factory(self, self.__thread_gate)
if base_thread_name:
base = base_thread_name
else:
base = thread.__class__.__name__
thread.name = '%s-%d' % (base, i)
self.__threads.append(thread)
thread.start()
def __init__(self,
num_threads,
queue_size=None,
base_thread_name=None,
worker_thread_factory=WorkerThread,
queue_factory=Queue.Queue):
"""Initialize an AdaptiveThreadPool.
An adaptive thread pool executes WorkItems using a number of
WorkerThreads. WorkItems represent items of work that may
succeed, soft fail, or hard fail. In addition, a completed work
item can signal this AdaptiveThreadPool to enable more or fewer
threads. Initially one thread is active. Soft failures are
reqeueud to be retried. Hard failures cause this
AdaptiveThreadPool to shut down entirely. See the WorkItem class
for more details.
Args:
num_threads: The number of threads to use.
queue_size: The size of the work item queue to use.
base_thread_name: A string from which worker thread names are derived.
worker_thread_factory: A factory which procudes WorkerThreads.
queue_factory: Used for dependency injection.
"""
if queue_size is None:
queue_size = num_threads
self.requeue = ReQueue(queue_size, queue_factory=queue_factory)
self.__thread_gate = ThreadGate(num_threads)
self.__num_threads = num_threads
self.__threads = []
for i in xrange(num_threads):
thread = worker_thread_factory(self, self.__thread_gate)
if base_thread_name:
base = base_thread_name
else:
base = thread.__class__.__name__
thread.name = '%s-%d' % (base, i)
self.__threads.append(thread)
thread.start()
class AdaptiveThreadPool(object):
"""A thread pool which processes WorkItems from a queue.
Attributes:
requeue: The requeue instance which holds work items for this
thread pool.
"""
def __init__(self,
num_threads,
queue_size=None,
base_thread_name=None,
worker_thread_factory=WorkerThread,
queue_factory=Queue.Queue):
"""Initialize an AdaptiveThreadPool.
An adaptive thread pool executes WorkItems using a number of
WorkerThreads. WorkItems represent items of work that may
succeed, soft fail, or hard fail. In addition, a completed work
item can signal this AdaptiveThreadPool to enable more or fewer
threads. Initially one thread is active. Soft failures are
reqeueud to be retried. Hard failures cause this
AdaptiveThreadPool to shut down entirely. See the WorkItem class
for more details.
Args:
num_threads: The number of threads to use.
queue_size: The size of the work item queue to use.
base_thread_name: A string from which worker thread names are derived.
worker_thread_factory: A factory which procudes WorkerThreads.
queue_factory: Used for dependency injection.
"""
if queue_size is None:
queue_size = num_threads
self.requeue = ReQueue(queue_size, queue_factory=queue_factory)
self.__thread_gate = ThreadGate(num_threads)
self.__num_threads = num_threads
self.__threads = []
for i in xrange(num_threads):
thread = worker_thread_factory(self, self.__thread_gate)
if base_thread_name:
base = base_thread_name
else:
base = thread.__class__.__name__
thread.name = '%s-%d' % (base, i)
self.__threads.append(thread)
thread.start()
def num_threads(self):
"""Return the number of threads in this thread pool."""
return self.__num_threads
def Threads(self):
"""Yields the registered threads."""
for thread in self.__threads:
yield thread
def SubmitItem(self, item, block=True, timeout=0.0):
"""Submit a WorkItem to the AdaptiveThreadPool.
Args:
item: A WorkItem instance.
block: Whether to block on submitting if the submit queue is full.
timeout: Time wait for room in the queue if block is True, 0.0 to
block indefinitely.
Raises:
Queue.Full if the submit queue is full.
"""
self.requeue.put(item, block=block, timeout=timeout)
def QueuedItemCount(self):
"""Returns the number of items currently in the queue."""
return self.requeue.qsize()
def Shutdown(self):
"""Shutdown the thread pool.
Tasks may remain unexecuted in the submit queue.
"""
while not self.requeue.empty():
try:
unused_item = self.requeue.get_nowait()
self.requeue.task_done()
except Queue.Empty:
pass
for thread in self.__threads:
thread.exit_flag = True
self.requeue.put(_THREAD_SHOULD_EXIT)
self.__thread_gate.EnableAllThreads()
def Wait(self):
"""Wait until all work items have been completed."""
self.requeue.join()
def JoinThreads(self):
"""Wait for all threads to exit."""
for thread in self.__threads:
logger.debug('Waiting for %s to exit' % str(thread))
thread.join()
def CheckErrors(self):
"""Output logs for any errors that occurred in the worker threads."""
for thread in self.__threads:
thread.CheckError()
class AdaptiveThreadPool(object):
"""A thread pool which processes WorkItems from a queue.
Attributes:
requeue: The requeue instance which holds work items for this
thread pool.
"""
def __init__(self,
num_threads,
queue_size=None,
base_thread_name=None,
worker_thread_factory=WorkerThread,
queue_factory=queue.Queue):
"""Initialize an AdaptiveThreadPool.
An adaptive thread pool executes WorkItems using a number of
WorkerThreads. WorkItems represent items of work that may
succeed, soft fail, or hard fail. In addition, a completed work
item can signal this AdaptiveThreadPool to enable more or fewer
threads. Initially one thread is active. Soft failures are
reqeueud to be retried. Hard failures cause this
AdaptiveThreadPool to shut down entirely. See the WorkItem class
for more details.
Args:
num_threads: The number of threads to use.
queue_size: The size of the work item queue to use.
base_thread_name: A string from which worker thread names are derived.
worker_thread_factory: A factory which procudes WorkerThreads.
queue_factory: Used for dependency injection.
"""
if queue_size is None:
queue_size = num_threads
self.requeue = ReQueue(queue_size, queue_factory=queue_factory)
self.__thread_gate = ThreadGate(num_threads)
self.__num_threads = num_threads
self.__threads = []
for i in range(num_threads):
thread = worker_thread_factory(self, self.__thread_gate)
if base_thread_name:
base = base_thread_name
else:
base = thread.__class__.__name__
thread.name = '%s-%d' % (base, i)
self.__threads.append(thread)
thread.start()
def num_threads(self):
"""Return the number of threads in this thread pool."""
return self.__num_threads
def Threads(self):
"""Yields the registered threads."""
for thread in self.__threads:
yield thread
def SubmitItem(self, item, block=True, timeout=0.0):
"""Submit a WorkItem to the AdaptiveThreadPool.
Args:
item: A WorkItem instance.
block: Whether to block on submitting if the submit queue is full.
timeout: Time wait for room in the queue if block is True, 0.0 to
block indefinitely.
Raises:
Queue.Full if the submit queue is full.
"""
self.requeue.put(item, block=block, timeout=timeout)
def QueuedItemCount(self):
"""Returns the number of items currently in the queue."""
return self.requeue.qsize()
def Shutdown(self):
"""Shutdown the thread pool.
Tasks may remain unexecuted in the submit queue.
"""
while not self.requeue.empty():
try:
unused_item = self.requeue.get_nowait()
self.requeue.task_done()
except queue.Empty:
pass
for thread in self.__threads:
thread.exit_flag = True
self.requeue.put(_THREAD_SHOULD_EXIT)
self.__thread_gate.EnableAllThreads()
def Wait(self):
"""Wait until all work items have been completed."""
self.requeue.join()
def JoinThreads(self):
"""Wait for all threads to exit."""
for thread in self.__threads:
logger.debug('Waiting for %s to exit' % str(thread))
thread.join()
def CheckErrors(self):
"""Output logs for any errors that occurred in the worker threads."""
for thread in self.__threads:
thread.CheckError()
