def reindex_all(cls, chunk_size=500, index=None, percent=100):
"""Rebuild ElasticSearch indexes.
:arg chunk_size: how many documents to bulk index as a single chunk.
:arg index: the `Index` object to reindex into. Uses the current
promoted index if none provided.
:arg percent: 1 to 100--the percentage of the db to index.
"""
from kuma.search.models import Index
from kuma.search.tasks import prepare_index, finalize_index
from kuma.wiki.tasks import index_documents
index = index or Index.objects.get_current()
# Get the list of document IDs to index.
indexable = WikiDocumentType.get_indexable(percent)
total = len(indexable)
total_chunks = int(ceil(total / chunk_size))
pre_task = prepare_index.si(index.pk)
post_task = finalize_index.si(index.pk)
if not total:
# If there's no data we still create the index and finalize it.
chain(pre_task, post_task).apply_async()
else:
index_tasks = [
index_documents.si(chunk, index.pk)
for chunk in chunked(indexable, chunk_size)
]
chord_flow(pre_task, index_tasks, post_task).apply_async()
message = _(
"Indexing %(total)d documents into %(total_chunks)d chunks of "
"size %(size)d into index %(index)s."
% {
"total": total,
"total_chunks": total_chunks,
"size": chunk_size,
"index": index.prefixed_name,
}
)
return message
def reindex_all(cls, chunk_size=500, index=None, percent=100):
"""Rebuild ElasticSearch indexes.
:arg chunk_size: how many documents to bulk index as a single chunk.
:arg index: the `Index` object to reindex into. Uses the current
promoted index if none provided.
:arg percent: 1 to 100--the percentage of the db to index.
"""
from kuma.search.models import Index
from kuma.search.tasks import prepare_index, finalize_index
from kuma.wiki.tasks import index_documents
index = index or Index.objects.get_current()
# Get the list of document IDs to index.
indexable = WikiDocumentType.get_indexable(percent)
total = len(indexable)
total_chunks = int(ceil(total / chunk_size))
pre_task = prepare_index.si(index.pk)
post_task = finalize_index.si(index.pk)
if not total:
# If there's no data we still create the index and finalize it.
chain(pre_task, post_task).apply_async()
else:
index_tasks = [index_documents.si(chunk, index.pk)
for chunk in chunked(indexable, chunk_size)]
chord_flow(pre_task, index_tasks, post_task).apply_async()
message = _(
'Indexing %(total)d documents into %(total_chunks)d chunks of '
'size %(size)d into index %(index)s.' % {
'total': total,
'total_chunks': total_chunks,
'size': chunk_size,
'index': index.prefixed_name
}
)
return message
def render_stale_documents(log=None):
"""Simple task wrapper for rendering stale documents"""
stale_docs = Document.objects.get_by_stale_rendering().distinct()
stale_docs_count = stale_docs.count()
if stale_docs_count == 0:
# not stale documents to render
return
if log is None:
# fetch a logger in case none is given
log = render_stale_documents.get_logger()
log.info("Found %s stale documents" % stale_docs_count)
stale_pks = stale_docs.values_list("pk", flat=True)
pre_task = acquire_render_lock.si()
render_tasks = [render_document_chunk.si(pks) for pks in chunked(stale_pks, 5)]
post_task = release_render_lock.si()
chord_flow(pre_task, render_tasks, post_task).apply_async()
def render_stale_documents(log=None):
"""Simple task wrapper for rendering stale documents"""
stale_docs = Document.objects.get_by_stale_rendering().distinct()
stale_docs_count = stale_docs.count()
if stale_docs_count == 0:
# not stale documents to render
return
if log is None:
# fetch a logger in case none is given
log = render_stale_documents.get_logger()
log.info('Found %s stale documents' % stale_docs_count)
stale_pks = stale_docs.values_list('pk', flat=True)
pre_task = acquire_render_lock.si()
render_tasks = [render_document_chunk.si(pks)
for pks in chunked(stale_pks, 5)]
post_task = release_render_lock.si()
chord_flow(pre_task, render_tasks, post_task).apply_async()
