def handle_reindex(request):
"""Caculate chunks and kick off indexing tasks."""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
return HttpResponseRedirect(request.path)
def test_index_chunk_task(self):
simple_items = [simple(save=True) for i in range(10)]
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
self.refresh()
# Verify there's nothing in the index.
eq_(len(SimpleIndex.search()), 0)
# Create the record and the chunk and then run it through
# celery.
batch_id = 'ou812'
rec = record(batch_id=batch_id, save=True)
chunk = (SimpleIndex, [item.id for item in simple_items])
index_chunk_task.delay(get_index(), batch_id, rec.id, chunk)
# Verify everything is in the index now.
eq_(len(SimpleIndex.search()), 10)
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
eq_(rec.status, Record.STATUS_SUCCESS)
def handle_reindex(request):
"""Caculate chunks and kick off indexing tasks."""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0],
id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id, (cls, id_list))
return HttpResponseRedirect(request.path)
def test_index_chunk_task(self):
simple_items = [simple(save=True) for i in range(10)]
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
self.refresh()
# Verify there's nothing in the index.
eq_(len(SimpleIndex.search()), 0)
# Create the record and the chunk and then run it through
# celery.
batch_id = "ou812"
rec = record(batch_id=batch_id, save=True)
chunk = (SimpleIndex, [item.id for item in simple_items])
index_chunk_task.delay(get_index(), batch_id, rec.id, chunk)
# Verify everything is in the index now.
eq_(len(SimpleIndex.search()), 10)
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
eq_(rec.status, Record.STATUS_SUCCESS)
def test_index_chunk_task(self):
responses = ResponseFactory.create_batch(10)
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
# Verify there's nothing in the index.
assert ResponseDocType.docs.search().count() == 0
# Create the record and the chunk and then run it through
# celery.
batch_id = 'ou812'
rec = RecordFactory(batch_id=batch_id)
chunk = (
to_class_path(ResponseDocType),
[item.id for item in responses]
)
index_chunk_task.delay(get_index_name(), batch_id, rec.id, chunk)
self.refresh()
# Verify everything is in the index now.
assert ResponseDocType.docs.search().count() == 10
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
assert rec.status == Record.STATUS_SUCCESS
def test_index_chunk_task(self):
responses = ResponseFactory.create_batch(10)
# With live indexing, that'll create items in the index. Since
# we want to test index_chunk_test, we need a clean index to
# start with so we delete and recreate it.
self.setup_indexes(empty=True)
# Verify there's nothing in the index.
eq_(len(ResponseMappingType.search()), 0)
# Create the record and the chunk and then run it through
# celery.
batch_id = 'ou812'
rec = RecordFactory(batch_id=batch_id)
chunk = (to_class_path(ResponseMappingType),
[item.id for item in responses])
index_chunk_task.delay(get_index(), batch_id, rec.id, chunk)
ResponseMappingType.refresh_index()
# Verify everything is in the index now.
eq_(len(ResponseMappingType.search()), 10)
# Verify the record was marked succeeded.
rec = Record.objects.get(pk=rec.id)
eq_(rec.status, Record.STATUS_SUCCESS)
def reindex():
"""Calculates and creates indexing chunks"""
index = get_index_name()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls._doc_type.name, id_list[0],
id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
def reindex():
"""Calculates and creates indexing chunks"""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
