def test_asyncwriter(self):
self.make_dir("testindex")
schema = fields.Schema(id=fields.ID, text=fields.TEXT)
ix = index.create_in("testindex", schema)
domain = (u"alfa", u"bravo", u"charlie", u"delta", u"echo", u"foxtrot",
u"golf", u"hotel", u"india")
writers = []
for i in xrange(20):
w = writing.AsyncWriter(ix.writer)
# Simulate doing 20 (near-)simultaneous commits. If we weren't using
# AsyncWriter, at least some of these would fail because the first
# writer wouldn't be finished yet.
writers.append(w)
w.add_document(id=unicode(i),
text=u" ".join(random.sample(domain, 5)))
w.commit()
# Wait for all writers to finish before checking the results
for w in writers:
if w.running:
w.join()
# Check whether all documents made it into the index.
r = ix.reader()
self.assertEqual(sorted([int(id) for id in r.lexicon("id")]),
range(20))
r.close()
ix.close()
self.clean_dir("test_index")
def test_asyncwriter_no_stored():
schema = fields.Schema(id=fields.ID, text=fields.TEXT)
with TempIndex(schema, "asyncnostored") as ix:
domain = (u("alfa"), u("bravo"), u("charlie"), u("delta"), u("echo"),
u("foxtrot"), u("golf"), u("hotel"), u("india"))
writers = []
# Simulate doing 20 (near-)simultaneous commits. If we weren't using
# AsyncWriter, at least some of these would fail because the first
# writer wouldn't be finished yet.
for i in xrange(20):
w = writing.AsyncWriter(ix)
writers.append(w)
w.add_document(id=text_type(i),
text=u(" ").join(random.sample(domain, 5)))
w.commit()
# Wait for all writers to finish before checking the results
for w in writers:
if w.running:
w.join()
# Check whether all documents made it into the index.
with ix.reader() as r:
assert_equal(sorted([int(id) for id in r.lexicon("id")]),
list(range(20)))
def add_documents(self, documents: list):
with writing.AsyncWriter(self.__index) as writer:
for doc in documents:
doc["message_tr"] = doc["message"]
doc["meta_content_tr"] = doc["meta_content"]
writer.add_document(**doc)
def delete_document(self, docnum: int):
with writing.AsyncWriter(self.__index) as writer:
writer.delete_document(docnum)
def get_writer(self):
return writing.AsyncWriter(self.get_index())
