Esempio n. 1
0
    def __init__(self, poolsize=None,
                 processor=None,
                 output_queue=None):
        """
        poolsize - the number of threads to use to process jobs, defaults to 10
        processor - an optional 1 argument function which is used to process jobs
                    placed on the input queue. If none is specified, jobs are assumed
                    to be zero argument callables.
        output_queue - if specified, the return value of proccessing a job will be placed
                       on this output queue.
        """
        self.input_queue = Queue()
        self.output_queue = output_queue

        if poolsize is None:
            poolsize = DEFAULT_POOLSIZE
 
        if processor is not None:
            self._do = processor

        self._local = threading.local()

        self._threads = []
        for i in range(poolsize):
            t = Thread(target=self._worker)
            t.setDaemon(True)
            self._threads.append(t)
Esempio n. 2
0
class ThreadPool:
    """
    Simple threadpool that processes
    items placed in its input queue. 

    implement the _do method or pass a single argument function 
    to the constructor to specify the processing of jobs
    placed on the input queue.
    
    By default it is assumed that jobs are 0 argument callables,
    a job is processed by calling it.
    """

    def __init__(self, poolsize=None,
                 processor=None,
                 output_queue=None):
        """
        poolsize - the number of threads to use to process jobs, defaults to 10
        processor - an optional 1 argument function which is used to process jobs
                    placed on the input queue. If none is specified, jobs are assumed
                    to be zero argument callables.
        output_queue - if specified, the return value of proccessing a job will be placed
                       on this output queue.
        """
        self.input_queue = Queue()
        self.output_queue = output_queue

        if poolsize is None:
            poolsize = DEFAULT_POOLSIZE
 
        if processor is not None:
            self._do = processor

        self._local = threading.local()

        self._threads = []
        for i in range(poolsize):
            t = Thread(target=self._worker)
            t.setDaemon(True)
            self._threads.append(t)

    def start(self):
        for t in self._threads:
            t.start()
    
    def join(self):
        self.input_queue.join()
            

    def _worker(self):
        while(True):
            try:
                job = self.input_queue.get()
                try:
                    rc = self._do(job)
                    if self.output_queue is not None:
                        self.output_queue.put(rc)
                finally:
                    self.input_queue.task_done()
            except:
                log.error(traceback.format_exc())

    def _do(self, job):
        return job()