def registerProcessor(self, service_name, processor, num_workers=0,
                          max_queued_entries=0):
        """
        Method to register a processor for a thrift service.
        service_name: The well known string service name.
        processor: The processor implementing the service interface
        num_workers: The number of worker threads for this service
        max_queued_entires: The maximum number of entries that can be queued
        for the service, a value < 0 indicates an
        unbounded queue.
        The method has the sideeffect of spawning a num_workers number of
        threads associated with the service.
        """
        # Use an unbounded queue and fail the insert if the queue size is
        # greater than max_queued_entries otherwise we will block inserts which
        # we don't want to do in the multiplex processor.
        service_queue = Queue.Queue()
        workers = []
        thread_name = "{0}-Thread-{1}"

        for i in xrange(num_workers):
            worker = Worker(
                service_queue, name=thread_name.format(service_name, i))
            worker.setDaemon(True)
            worker.start()
            workers.append(worker)

        # Keep track of workers for shutdown.
        self.services[service_name] = ServiceProcessor(service_name, processor,
                                                       service_queue, workers,
                                                       max_queued_entries)
        logging.info("Initialized service %s, with threadpool size %d"
                     % (service_name, num_workers))
Beispiel #2
0
    def test_worker(self):
        """Test Worker thread.
        """
        data = [MagicMock(), MagicMock(), MagicMock(), MagicMock(),
                MagicMock(), MagicMock()]
        service_queue = Queue.Queue()
        workers = []

        def wait_for_processing(timer=20):
            retries = 0
            while (retries < timer):
                if service_queue.empty():
                    break
                retries += 1
                time.sleep(1)

        def produce_task():
            for _ in xrange(100):
                service_queue.put(data)

        for i in xrange(32):
            worker = Worker(service_queue)
            worker.setDaemon(True)
            worker.start()
            workers.append(worker)

        produce_task()
        wait_for_processing()
        self.assertEqual(service_queue.empty(), True)

        # Test queue.get() timeout.
        time.sleep(4)

        produce_task()
        wait_for_processing()
        self.assertEqual(service_queue.empty(), True)
Beispiel #3
0
    def registerProcessor(self,
                          service_name,
                          processor,
                          num_workers=0,
                          max_queued_entries=0):
        """
        Method to register a processor for a thrift service.
        service_name: The well known string service name.
        processor: The processor implementing the service interface
        num_workers: The number of worker threads for this service
        max_queued_entires: The maximum number of entries that can be queued
        for the service, a value < 0 indicates an
        unbounded queue.
        The method has the sideeffect of spawning a num_workers number of
        threads associated with the service.
        """
        # Use an unbounded queue and fail the insert if the queue size is
        # greater than max_queued_entries otherwise we will block inserts which
        # we don't want to do in the multiplex processor.
        service_queue = Queue.Queue()
        workers = []
        thread_name = "{0}-Thread-{1}"

        for i in xrange(num_workers):
            worker = Worker(service_queue,
                            name=thread_name.format(service_name, i))
            worker.setDaemon(True)
            worker.start()
            workers.append(worker)

        # Keep track of workers for shutdown.
        self.services[service_name] = ServiceProcessor(service_name, processor,
                                                       service_queue, workers,
                                                       max_queued_entries)
        logging.info("Initialized service %s, with threadpool size %d" %
                     (service_name, num_workers))