def reindex_all(cls, chunk_size=500, index=None, percent=100): """Rebuild ElasticSearch indexes. :arg chunk_size: how many documents to bulk index as a single chunk. :arg index: the `Index` object to reindex into. Uses the current promoted index if none provided. :arg percent: 1 to 100--the percentage of the db to index. """ from kuma.search.models import Index from kuma.search.tasks import prepare_index, finalize_index from kuma.wiki.tasks import index_documents index = index or Index.objects.get_current() # Get the list of document IDs to index. indexable = WikiDocumentType.get_indexable(percent) total = len(indexable) total_chunks = int(ceil(total / chunk_size)) pre_task = prepare_index.si(index.pk) post_task = finalize_index.si(index.pk) if not total: # If there's no data we still create the index and finalize it. chain(pre_task, post_task).apply_async() else: index_tasks = [ index_documents.si(chunk, index.pk) for chunk in chunked(indexable, chunk_size) ] chord_flow(pre_task, index_tasks, post_task).apply_async() message = _( "Indexing %(total)d documents into %(total_chunks)d chunks of " "size %(size)d into index %(index)s." % { "total": total, "total_chunks": total_chunks, "size": chunk_size, "index": index.prefixed_name, } ) return message
def reindex_all(cls, chunk_size=500, index=None, percent=100): """Rebuild ElasticSearch indexes. :arg chunk_size: how many documents to bulk index as a single chunk. :arg index: the `Index` object to reindex into. Uses the current promoted index if none provided. :arg percent: 1 to 100--the percentage of the db to index. """ from kuma.search.models import Index from kuma.search.tasks import prepare_index, finalize_index from kuma.wiki.tasks import index_documents index = index or Index.objects.get_current() # Get the list of document IDs to index. indexable = WikiDocumentType.get_indexable(percent) total = len(indexable) total_chunks = int(ceil(total / chunk_size)) pre_task = prepare_index.si(index.pk) post_task = finalize_index.si(index.pk) if not total: # If there's no data we still create the index and finalize it. chain(pre_task, post_task).apply_async() else: index_tasks = [index_documents.si(chunk, index.pk) for chunk in chunked(indexable, chunk_size)] chord_flow(pre_task, index_tasks, post_task).apply_async() message = _( 'Indexing %(total)d documents into %(total_chunks)d chunks of ' 'size %(size)d into index %(index)s.' % { 'total': total, 'total_chunks': total_chunks, 'size': chunk_size, 'index': index.prefixed_name } ) return message
def render_stale_documents(log=None): """Simple task wrapper for rendering stale documents""" stale_docs = Document.objects.get_by_stale_rendering().distinct() stale_docs_count = stale_docs.count() if stale_docs_count == 0: # not stale documents to render return if log is None: # fetch a logger in case none is given log = render_stale_documents.get_logger() log.info("Found %s stale documents" % stale_docs_count) stale_pks = stale_docs.values_list("pk", flat=True) pre_task = acquire_render_lock.si() render_tasks = [render_document_chunk.si(pks) for pks in chunked(stale_pks, 5)] post_task = release_render_lock.si() chord_flow(pre_task, render_tasks, post_task).apply_async()
def render_stale_documents(log=None): """Simple task wrapper for rendering stale documents""" stale_docs = Document.objects.get_by_stale_rendering().distinct() stale_docs_count = stale_docs.count() if stale_docs_count == 0: # not stale documents to render return if log is None: # fetch a logger in case none is given log = render_stale_documents.get_logger() log.info('Found %s stale documents' % stale_docs_count) stale_pks = stale_docs.values_list('pk', flat=True) pre_task = acquire_render_lock.si() render_tasks = [render_document_chunk.si(pks) for pks in chunked(stale_pks, 5)] post_task = release_render_lock.si() chord_flow(pre_task, render_tasks, post_task).apply_async()