class IndexMaker(threading.Thread): def __init__(self): super(IndexMaker, self).__init__() self.__path = '' self.name = 'IndexMaker' self._in_queue = Queue.Queue(0) self._out_queue = Queue.Queue(0) self.processed = [] self.thread_runner = None self.trhead_class = ReferenceMakerThread def get_processed(self): return self.__processed def set_processed(self, value): self.__processed = value def set_path(self, path): self.__path = path # Read all the files from the given path and put them in a queue self.files = FileManager().get_files_list(path) for file in self.files: self._in_queue.put(file) def get_n_files(self): return self._in_queue.qsize() def make_index(self): self.trhead_class = ReferenceMakerThread self.start() def run(self): log.debug("Start running index maker") #@UndefinedVariable # Run threads self.thread_runner = ThreadRunner(self.trhead_class, self._in_queue, self._out_queue) self.thread_runner.start() while not (self.thread_runner.finished and self._out_queue.empty()): extraction = self._out_queue.get() log.info('Persisting extraction results') #@UndefinedVariable # Persist the extraction ExtractionGateway().persist_extraction(extraction) self.processed.append(extraction) # Commit changes to the database flush_changes() log.debug("Total processed: %d" % len(self.processed)) #@UndefinedVariable processed = property(get_processed, set_processed)
def run(self): log.debug("Start running index maker") #@UndefinedVariable # Run threads self.thread_runner = ThreadRunner(self.trhead_class, self._in_queue, self._out_queue) self.thread_runner.start() while not (self.thread_runner.finished and self._out_queue.empty()): extraction = self._out_queue.get() log.info('Persisting extraction results') #@UndefinedVariable # Persist the extraction ExtractionGateway().persist_extraction(extraction) self.processed.append(extraction) # Commit changes to the database flush_changes() log.debug("Total processed: %d" % len(self.processed)) #@UndefinedVariable
def setUp(self): self.thread_type = TestThread self.in_queue = Queue.Queue(0) self.out_queue = Queue.Queue(0) self.tr = ThreadRunner(self.thread_type, self.in_queue, self.out_queue)
class TestThreading(unittest.TestCase): def setUp(self): self.thread_type = TestThread self.in_queue = Queue.Queue(0) self.out_queue = Queue.Queue(0) self.tr = ThreadRunner(self.thread_type, self.in_queue, self.out_queue) def tearDown(self): pass def test_set_pool_size(self): for i in range(2): self.in_queue.put(i) self.tr._set_pool_size() self.failUnless(self.tr.get_pool_size() == 1) # Let's add two more items for i in range(2): self.in_queue.put(i) self.tr._set_pool_size() self.failUnless(self.tr.get_pool_size() == 2) # Let's add 25 items for i in range(25): self.in_queue.put(i) self.tr._set_pool_size() self.failUnless(self.tr.get_pool_size() == 5) def test_run(self): for i in range(50): #@UnusedVariable self.in_queue.put('somet_object') self.tr.run() self.failUnless(self.out_queue.qsize() == 50)