def reindex(self):
if self.args.all:
if self.args.drop:
self.drop()
self.prepare()
t1 = datetime.now()
updated = 0
created = 0
for post in BasePost.find():
doc = post.get_indexer_document()
if doc is None:
continue
resp = ctx.es.index(index="posts", id=post._id, body=doc)
if resp["result"] == "created":
created += 1
elif resp["result"] == "updated":
updated += 1
t2 = datetime.now()
dt = (t2 - t1).total_seconds()
ctx.log.info(
"Posts full reindexing completed. Created %d, updated %d documents in %.3f seconds",
created, updated, dt)
else:
raise NotImplementedError("partial reindex is not implemented")
def rt_new_post(task):
post: Optional[BasePost] = BasePost.get(task.post_id)
if not post:
ctx.log.error("error processing new post %s: post not found",
task.post_id)
return
ctx.log.info("generating events for a new post %s body='%s'",
task.post_id, cut(post.body, max_len=50))
post.generate_new_post_events()
def __next__(self):
if self.idx >= len(self.data):
raise StopIteration
while True:
doc = self.data[self.idx]
post = BasePost.find_one({"_id": ObjectId(doc["_id"])})
self.idx += 1
if post:
break
if self.idx >= len(self.data):
raise StopIteration
post._score = doc["_score"]
return post
def rt_post_indexer(task: PostIndexerTask):
if not hasattr(ctx, "es") or ctx.es is None:
ctx.log.error("elasticsearch is not configured, indexing skipped")
return
if task.delete:
ctx.es.delete(index="posts", id=task.post_id)
ctx.log.info("post %s has been deleted from index", task.post_id)
else:
post: BasePost = BasePost.get(task.post_id)
if post is None:
ctx.log.error("error indexing post %s: post not found",
task.post_id)
return
doc = post.get_indexer_document()
resp = ctx.es.index(index="posts", id=task.post_id, body=doc)
if resp["result"] == "created":
ctx.log.info("post %s has been indexed", task.post_id)
elif resp["result"] == "updated":
ctx.log.info("post %s index has been updated", task.post_id)
def setUp(self) -> None:
BasePost.destroy_all()
Tag.destroy_all()
def setUp(self) -> None:
super(TestEvent, self).setUp()
Event.destroy_all()
BasePost.destroy_all()
User.destroy_all()
self.clear_queue()