def test_get_multi_with_sentinel(self):
log = logging.getLogger()
index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log)
q = index.get_multi_q(sentinel='BLAH')
docs = self.rand_docs.get_docs(5000)
docs2 = self.rand_docs.get_docs(5000)
for item in docs + ['BLAH'] + docs2:
q.put(item)
index.join_indexer()
self.assertEqual(docs + docs2, index.get_all_json_from_indexq())
def test_get_multi_with_sentinel(self):
log = logging.getLogger()
index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log)
q = index.get_multi_q(sentinel='BLAH')
docs = self.rand_docs.get_docs(5000)
docs2 = self.rand_docs.get_docs(5000)
for item in docs + ['BLAH'] + docs2:
q.put(item)
index.join_indexer()
self.assertEqual(docs+docs2, index.get_all_json_from_indexq())
def test_get_multi_q2(self):
log = logging.getLogger()
index = IndexQ(test_config['indexqbase'], 'testq', size=1, log=log)
q = index.get_multi_q()
docs = self.rand_docs.get_docs(50000)
for item in docs:
q.put(item)
q.put('STOP')
index.join_indexer()
self.assertEqual(docs, index.get_all_json_from_indexq())
def test_get_multi_q2(self):
log = logging.getLogger()
index = IndexQ(test_config['indexqbase'], 'testq', size = 1, log = log)
q = index.get_multi_q()
docs = self.rand_docs.get_docs(50000)
for item in docs:
q.put(item)
q.put('STOP')
index.join_indexer()
self.assertEqual(docs, index.get_all_json_from_indexq())
def test_thread_pool_mid(self):
'''
Index data using multiple threads.
Verity that each thread
'''
docs = self.rand_docs.get_docs(5000)
threads = 5
index = IndexQ(test_config['indexqbase'],'testq', size = 1)
with ThreadPool(threads) as p:
p.map(index.add, docs)
index.add(finalize=True)
d = index.get_all_json_from_indexq()
self.assertEqual(sorted(d, key=lambda x: x['id']), sorted(docs, key=lambda x: x['id']))
def test_thread_pool_mid(self):
'''
Index data using multiple threads.
Verity that each thread
'''
docs = self.rand_docs.get_docs(5000)
threads = 5
index = IndexQ(test_config['indexqbase'], 'testq', size=1)
with ThreadPool(threads) as p:
p.map(index.add, docs)
index.add(finalize=True)
d = index.get_all_json_from_indexq()
self.assertEqual(sorted(d, key=lambda x: x['id']),
sorted(docs, key=lambda x: x['id']))
def test_thread_pool_high(self):
'''
Index data using multiple threads.
Verity that each thread
'''
docs = self.rand_docs.get_docs(25000)
index = IndexQ(test_config['indexqbase'], 'testq', size=.1, devel=True)
for dir in ['_todo_dir', '_done_dir']:
[os.remove(x) for x in index.get_all_as_list(dir=dir)]
threads = 25
with ThreadPool(threads) as p:
p.map(index.add, docs)
index.add(finalize=True)
d = index.get_all_json_from_indexq()
self.assertEqual(len(d), len(docs))
self.assertEqual(sorted(d, key=lambda x: x['id']),
sorted(docs, key=lambda x: x['id']))
def test_thread_pool_high(self):
'''
Index data using multiple threads.
Verity that each thread
'''
docs = self.rand_docs.get_docs(25000)
index = IndexQ(test_config['indexqbase'],
'testq',
size=.1,
devel=True)
for dir in ['_todo_dir', '_done_dir']:
[os.remove(x) for x in index.get_all_as_list(dir=dir)]
threads = 25
with ThreadPool(threads) as p:
p.map(index.add, docs)
index.add(finalize=True)
d = index.get_all_json_from_indexq()
self.assertEqual(len(d), len(docs))
self.assertEqual(sorted(d, key=lambda x: x['id']),
sorted(docs, key=lambda x: x['id']))
