def handle_reindex(request): """Caculates and kicks off indexing tasks""" # This is truthy if the user wants us to delete and recreate # the index first. delete_index_first = bool(request.POST.get('delete_index')) if delete_index_first: # Coming from the delete form, so we reindex all models. mapping_types_to_index = None else: # Coming from the reindex form, so we reindex whatever we're # told. mapping_types_to_index = [ name.replace('check_', '') for name in request.POST.keys() if name.startswith('check_') ] # TODO: If this gets fux0rd, then it's possible this could be # non-zero and we really want to just ignore it. Need the ability # to ignore it. try: client = redis_client('default') val = client.get(OUTSTANDING_INDEX_CHUNKS) if val is not None and int(val) > 0: raise ReindexError('There are %s outstanding chunks.' % val) # We don't know how many chunks we're building, but we do want # to make sure another reindex request doesn't slide in here # and kick off a bunch of chunks. # # There is a race condition here. client.set(OUTSTANDING_INDEX_CHUNKS, 1) except RedisError: log.warning('Redis not running. Can not check if there are ' 'outstanding tasks.') batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(mapping_types=mapping_types_to_index): chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) if delete_index_first: # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() chunks_count = len(chunks) try: client = redis_client('default') client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count) except RedisError: log.warning('Redis not running. Can\'t denote outstanding tasks.') for chunk in chunks: index_chunk_task.delay(write_index(), batch_id, chunk) return HttpResponseRedirect(request.path)
def handle_reindex(request): """Caculates and kicks off indexing tasks""" # This is truthy if the user wants us to delete and recreate # the index first. delete_index_first = bool(request.POST.get('delete_index')) if delete_index_first: # Coming from the delete form, so we reindex all models. mapping_types_to_index = None else: # Coming from the reindex form, so we reindex whatever we're # told. mapping_types_to_index = [name.replace('check_', '') for name in request.POST.keys() if name.startswith('check_')] # TODO: If this gets fux0rd, then it's possible this could be # non-zero and we really want to just ignore it. Need the ability # to ignore it. try: client = redis_client('default') val = client.get(OUTSTANDING_INDEX_CHUNKS) if val is not None and int(val) > 0: raise ReindexError('There are %s outstanding chunks.' % val) # We don't know how many chunks we're building, but we do want # to make sure another reindex request doesn't slide in here # and kick off a bunch of chunks. # # There is a race condition here. client.set(OUTSTANDING_INDEX_CHUNKS, 1) except RedisError: log.warning('Redis not running. Can not check if there are ' 'outstanding tasks.') batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(mapping_types=mapping_types_to_index): chunks.extend( (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) if delete_index_first: # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() chunks_count = len(chunks) try: client = redis_client('default') client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count) except RedisError: log.warning('Redis not running. Can\'t denote outstanding tasks.') for chunk in chunks: index_chunk_task.delay(write_index(), batch_id, chunk) return HttpResponseRedirect(request.path)
def setup_indexes(self, empty=False, wait=True): """(Re-)create WRITE_INDEX""" from kitsune.search.es_utils import recreate_index recreate_index() get_es().health(wait_for_status='yellow')