def __init__(self, poolsize=None,
processor=None,
output_queue=None):
"""
poolsize - the number of threads to use to process jobs, defaults to 10
processor - an optional 1 argument function which is used to process jobs
placed on the input queue. If none is specified, jobs are assumed
to be zero argument callables.
output_queue - if specified, the return value of proccessing a job will be placed
on this output queue.
"""
self.input_queue = Queue()
self.output_queue = output_queue
if poolsize is None:
poolsize = DEFAULT_POOLSIZE
if processor is not None:
self._do = processor
self._local = threading.local()
self._threads = []
for i in range(poolsize):
t = Thread(target=self._worker)
t.setDaemon(True)
self._threads.append(t)
class ThreadPool:
"""
Simple threadpool that processes
items placed in its input queue.
implement the _do method or pass a single argument function
to the constructor to specify the processing of jobs
placed on the input queue.
By default it is assumed that jobs are 0 argument callables,
a job is processed by calling it.
"""
def __init__(self, poolsize=None,
processor=None,
output_queue=None):
"""
poolsize - the number of threads to use to process jobs, defaults to 10
processor - an optional 1 argument function which is used to process jobs
placed on the input queue. If none is specified, jobs are assumed
to be zero argument callables.
output_queue - if specified, the return value of proccessing a job will be placed
on this output queue.
"""
self.input_queue = Queue()
self.output_queue = output_queue
if poolsize is None:
poolsize = DEFAULT_POOLSIZE
if processor is not None:
self._do = processor
self._local = threading.local()
self._threads = []
for i in range(poolsize):
t = Thread(target=self._worker)
t.setDaemon(True)
self._threads.append(t)
def start(self):
for t in self._threads:
t.start()
def join(self):
self.input_queue.join()
def _worker(self):
while(True):
try:
job = self.input_queue.get()
try:
rc = self._do(job)
if self.output_queue is not None:
self.output_queue.put(rc)
finally:
self.input_queue.task_done()
except:
log.error(traceback.format_exc())
def _do(self, job):
return job()