def handle_reindex(request): """Caculate chunks and kick off indexing tasks.""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend( (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list)) return HttpResponseRedirect(request.path)
def test_index_chunk_task(self): simple_items = [simple(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) self.refresh() # Verify there's nothing in the index. eq_(len(SimpleIndex.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = record(batch_id=batch_id, save=True) chunk = (SimpleIndex, [item.id for item in simple_items]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) # Verify everything is in the index now. eq_(len(SimpleIndex.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def handle_reindex(request): """Caculate chunks and kick off indexing tasks.""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (cls, id_list)) return HttpResponseRedirect(request.path)
def test_index_chunk_task(self): simple_items = [simple(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) self.refresh() # Verify there's nothing in the index. eq_(len(SimpleIndex.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = "ou812" rec = record(batch_id=batch_id, save=True) chunk = (SimpleIndex, [item.id for item in simple_items]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) # Verify everything is in the index now. eq_(len(SimpleIndex.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def test_index_chunk_task(self): responses = ResponseFactory.create_batch(10) # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. assert ResponseDocType.docs.search().count() == 0 # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = RecordFactory(batch_id=batch_id) chunk = ( to_class_path(ResponseDocType), [item.id for item in responses] ) index_chunk_task.delay(get_index_name(), batch_id, rec.id, chunk) self.refresh() # Verify everything is in the index now. assert ResponseDocType.docs.search().count() == 10 # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) assert rec.status == Record.STATUS_SUCCESS
def test_index_chunk_task(self): responses = ResponseFactory.create_batch(10) # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. eq_(len(ResponseMappingType.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = RecordFactory(batch_id=batch_id) chunk = (to_class_path(ResponseMappingType), [item.id for item in responses]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) ResponseMappingType.refresh_index() # Verify everything is in the index now. eq_(len(ResponseMappingType.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def reindex(): """Calculates and creates indexing chunks""" index = get_index_name() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls._doc_type.name, id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))
def reindex(): """Calculates and creates indexing chunks""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend( (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))