class IndexMaker(threading.Thread):
def __init__(self):
super(IndexMaker, self).__init__()
self.__path = ''
self.name = 'IndexMaker'
self._in_queue = Queue.Queue(0)
self._out_queue = Queue.Queue(0)
self.processed = []
self.thread_runner = None
self.trhead_class = ReferenceMakerThread
def get_processed(self):
return self.__processed
def set_processed(self, value):
self.__processed = value
def set_path(self, path):
self.__path = path
# Read all the files from the given path and put them in a queue
self.files = FileManager().get_files_list(path)
for file in self.files:
self._in_queue.put(file)
def get_n_files(self):
return self._in_queue.qsize()
def make_index(self):
self.trhead_class = ReferenceMakerThread
self.start()
def run(self):
log.debug("Start running index maker") #@UndefinedVariable
# Run threads
self.thread_runner = ThreadRunner(self.trhead_class, self._in_queue,
self._out_queue)
self.thread_runner.start()
while not (self.thread_runner.finished and self._out_queue.empty()):
extraction = self._out_queue.get()
log.info('Persisting extraction results') #@UndefinedVariable
# Persist the extraction
ExtractionGateway().persist_extraction(extraction)
self.processed.append(extraction)
# Commit changes to the database
flush_changes()
log.debug("Total processed: %d" %
len(self.processed)) #@UndefinedVariable
processed = property(get_processed, set_processed)
class IndexMaker(threading.Thread):
def __init__(self):
super(IndexMaker, self).__init__()
self.__path = ''
self.name = 'IndexMaker'
self._in_queue = Queue.Queue(0)
self._out_queue = Queue.Queue(0)
self.processed = []
self.thread_runner = None
self.trhead_class = ReferenceMakerThread
def get_processed(self):
return self.__processed
def set_processed(self, value):
self.__processed = value
def set_path(self, path):
self.__path = path
# Read all the files from the given path and put them in a queue
self.files = FileManager().get_files_list(path)
for file in self.files:
self._in_queue.put(file)
def get_n_files(self):
return self._in_queue.qsize()
def make_index(self):
self.trhead_class = ReferenceMakerThread
self.start()
def run(self):
log.debug("Start running index maker") #@UndefinedVariable
# Run threads
self.thread_runner = ThreadRunner(self.trhead_class,
self._in_queue, self._out_queue)
self.thread_runner.start()
while not (self.thread_runner.finished and self._out_queue.empty()):
extraction = self._out_queue.get()
log.info('Persisting extraction results') #@UndefinedVariable
# Persist the extraction
ExtractionGateway().persist_extraction(extraction)
self.processed.append(extraction)
# Commit changes to the database
flush_changes()
log.debug("Total processed: %d" % len(self.processed)) #@UndefinedVariable
processed = property(get_processed, set_processed)
def run(self):
log.debug("Start running index maker") #@UndefinedVariable
# Run threads
self.thread_runner = ThreadRunner(self.trhead_class, self._in_queue,
self._out_queue)
self.thread_runner.start()
while not (self.thread_runner.finished and self._out_queue.empty()):
extraction = self._out_queue.get()
log.info('Persisting extraction results') #@UndefinedVariable
# Persist the extraction
ExtractionGateway().persist_extraction(extraction)
self.processed.append(extraction)
# Commit changes to the database
flush_changes()
log.debug("Total processed: %d" %
len(self.processed)) #@UndefinedVariable
def run(self):
log.debug("Start running index maker") #@UndefinedVariable
# Run threads
self.thread_runner = ThreadRunner(self.trhead_class,
self._in_queue, self._out_queue)
self.thread_runner.start()
while not (self.thread_runner.finished and self._out_queue.empty()):
extraction = self._out_queue.get()
log.info('Persisting extraction results') #@UndefinedVariable
# Persist the extraction
ExtractionGateway().persist_extraction(extraction)
self.processed.append(extraction)
# Commit changes to the database
flush_changes()
log.debug("Total processed: %d" % len(self.processed)) #@UndefinedVariable
def setUp(self):
self.thread_type = TestThread
self.in_queue = Queue.Queue(0)
self.out_queue = Queue.Queue(0)
self.tr = ThreadRunner(self.thread_type, self.in_queue,
self.out_queue)
class TestThreading(unittest.TestCase):
def setUp(self):
self.thread_type = TestThread
self.in_queue = Queue.Queue(0)
self.out_queue = Queue.Queue(0)
self.tr = ThreadRunner(self.thread_type, self.in_queue,
self.out_queue)
def tearDown(self):
pass
def test_set_pool_size(self):
for i in range(2):
self.in_queue.put(i)
self.tr._set_pool_size()
self.failUnless(self.tr.get_pool_size() == 1)
# Let's add two more items
for i in range(2):
self.in_queue.put(i)
self.tr._set_pool_size()
self.failUnless(self.tr.get_pool_size() == 2)
# Let's add 25 items
for i in range(25):
self.in_queue.put(i)
self.tr._set_pool_size()
self.failUnless(self.tr.get_pool_size() == 5)
def test_run(self):
for i in range(50): #@UnusedVariable
self.in_queue.put('somet_object')
self.tr.run()
self.failUnless(self.out_queue.qsize() == 50)
