def _initPool(self):
self._maxThreads = cf.cf.getData("maxThreads") or 0
#
# Please note that I'm using the q_size parameter of the Threadpool. This is what the
# threadpool documentation says about it:
#
# If q_size > 0 the size of the work request is limited and the
# thread pool blocks when queue is full and it tries to put more
# work requests in it.
#
# This will basically save me from filling-up all the memory with WorkRequest objects
# when somebody performs something like this:
#
# for url in looooooong_list:
# self._tm.startFunction( target=self._do_request, args=(url,) )
#
if self._maxThreads:
self._threadPool = ThreadPool(self._maxThreads, q_size=200)
else:
# if I want to use the restrict argument of startFunction, the thread pool
# MUST have some threads
self._threadPool = ThreadPool(5, 15)
class threadManager(object):
"""
This class manages threads.
@author: Andres Riancho ( [email protected] )
"""
def __init__(self):
self._daemonThreads = []
self.informed = False
self._initialized = False
self._waitForJoin = 2
self._initPool()
def _initPool(self):
self._maxThreads = cf.cf.getData("maxThreads") or 0
#
# Please note that I'm using the q_size parameter of the Threadpool. This is what the
# threadpool documentation says about it:
#
# If q_size > 0 the size of the work request is limited and the
# thread pool blocks when queue is full and it tries to put more
# work requests in it.
#
# This will basically save me from filling-up all the memory with WorkRequest objects
# when somebody performs something like this:
#
# for url in looooooong_list:
# self._tm.startFunction( target=self._do_request, args=(url,) )
#
if self._maxThreads:
self._threadPool = ThreadPool(self._maxThreads, q_size=200)
else:
# if I want to use the restrict argument of startFunction, the thread pool
# MUST have some threads
self._threadPool = ThreadPool(5, 15)
def setMaxThreads(self, num_threads):
if self._maxThreads > num_threads:
self._threadPool.dismissWorkers(self._maxThreads - num_threads)
self._maxThreads = num_threads
elif self._maxThreads < num_threads:
self._threadPool.createWorkers(num_threads - self._maxThreads)
self._maxThreads = num_threads
def getMaxThreads(self):
if not self._initialized:
self._initPool()
return self._maxThreads
def startDaemon(self, threadObj):
om.out.debug("Starting daemon thread: " + str(threadObj))
threadObj.setDaemon(1)
threadObj.start()
self._daemonThreads.append(threadObj)
def stopDaemon(self, threadObj):
for daemon in self._daemonThreads:
if daemon == threadObj:
threadObj.stop()
om.out.debug("Calling join on daemon thread: " + str(threadObj))
threadObj.join(self._waitForJoin)
def stopAllDaemons(self):
om.out.debug("Calling join on all daemon threads")
for thread in self._daemonThreads:
thread.stop()
om.out.debug("Calling join on daemon thread: " + str(thread))
thread.join(self._waitForJoin)
def startFunction(self, target, args=(), kwds={}, restrict=True, ownerObj=None):
if not self._initialized:
self._initPool()
self._initialized = True
if not self._maxThreads and restrict:
# Just start the function
if not self.informed:
om.out.debug("Threading is disabled.")
self.informed = True
target(*args, **kwds)
else:
# Assign a job to a thread in the thread pool
wr = WorkRequest(target, args=args, kwds=kwds, ownerObj=ownerObj)
self._threadPool.putRequest(wr)
msg = "[thread manager] Successfully added function to threadpool. Work queue size: "
msg += str(self._threadPool.requestsQueue.qsize())
om.out.debug(msg)
def join(self, ownerObj=None, joinAll=False):
self._threadPool.wait(ownerObj, joinAll)