def test_get_multi_with_sentinel(self): log = logging.getLogger() index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log) q = index.get_multi_q(sentinel='BLAH') docs = self.rand_docs.get_docs(5000) docs2 = self.rand_docs.get_docs(5000) for item in docs + ['BLAH'] + docs2: q.put(item) index.join_indexer() self.assertEqual(docs + docs2, index.get_all_json_from_indexq())
def test_get_multi_with_sentinel(self): log = logging.getLogger() index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log) q = index.get_multi_q(sentinel='BLAH') docs = self.rand_docs.get_docs(5000) docs2 = self.rand_docs.get_docs(5000) for item in docs + ['BLAH'] + docs2: q.put(item) index.join_indexer() self.assertEqual(docs+docs2, index.get_all_json_from_indexq())
def test_get_multi_q2(self): log = logging.getLogger() index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log) q = index.get_multi_q() docs = self.rand_docs.get_docs(50000) for item in docs: q.put(item) q.put('STOP') index.join_indexer() self.assertEqual(docs, index.get_all_json_from_indexq())
def test_get_multi_q2(self): log = logging.getLogger() index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log) q = index.get_multi_q() docs = self.rand_docs.get_docs(50000) for item in docs: q.put(item) q.put('STOP') index.join_indexer() self.assertEqual(docs, index.get_all_json_from_indexq())
def test_thread_pool_mid(self): ''' Index data using multiple threads. Verity that each thread ''' docs = self.rand_docs.get_docs(5000) threads = 5 index = IndexQ(test_config['indexqbase'],'testq', size = 1) with ThreadPool(threads) as p: p.map(index.add, docs) index.add(finalize=True) d = index.get_all_json_from_indexq() self.assertEqual(sorted(d, key=lambda x: x['id']), sorted(docs, key=lambda x: x['id']))
def test_thread_pool_mid(self): ''' Index data using multiple threads. Verity that each thread ''' docs = self.rand_docs.get_docs(5000) threads = 5 index = IndexQ(test_config['indexqbase'], 'testq', size=1) with ThreadPool(threads) as p: p.map(index.add, docs) index.add(finalize=True) d = index.get_all_json_from_indexq() self.assertEqual(sorted(d, key=lambda x: x['id']), sorted(docs, key=lambda x: x['id']))
def test_thread_pool_high(self): ''' Index data using multiple threads. Verity that each thread ''' docs = self.rand_docs.get_docs(25000) index = IndexQ(test_config['indexqbase'], 'testq', size=.1, devel=True) for dir in ['_todo_dir', '_done_dir']: [os.remove(x) for x in index.get_all_as_list(dir=dir)] threads = 25 with ThreadPool(threads) as p: p.map(index.add, docs) index.add(finalize=True) d = index.get_all_json_from_indexq() self.assertEqual(len(d), len(docs)) self.assertEqual(sorted(d, key=lambda x: x['id']), sorted(docs, key=lambda x: x['id']))