def test_chunked():
# chunking nothing yields nothing.
assert list(chunked([], 1)) == []
# chunking list where len(list) < n
assert list(chunked([1], 10)) == [(1,)]
# chunking a list where len(list) == n
assert list(chunked([1, 2], 2)) == [(1, 2)]
# chunking list where len(list) > n
assert list(chunked([1, 2, 3, 4, 5], 2)) == [(1, 2), (3, 4), (5,)]
def test_chunked():
# chunking nothing yields nothing.
assert list(chunked([], 1)) == []
# chunking list where len(list) < n
assert list(chunked([1], 10)) == [(1, )]
# chunking a list where len(list) == n
assert list(chunked([1, 2], 2)) == [(1, 2)]
# chunking list where len(list) > n
assert list(chunked([1, 2, 3, 4, 5], 2)) == [(1, 2), (3, 4), (5, )]
def test_chunked(self):
# chunking nothing yields nothing.
eq_(list(chunked([], 1)), [])
# chunking list where len(list) < n
eq_(list(chunked([1], 10)), [(1, )])
# chunking a list where len(list) == n
eq_(list(chunked([1, 2], 2)), [(1, 2)])
# chunking list where len(list) > n
eq_(list(chunked([1, 2, 3, 4, 5], 2)), [(1, 2), (3, 4), (5, )])
def test_chunked(self):
# chunking nothing yields nothing.
eq_(list(chunked([], 1)), [])
# chunking list where len(list) < n
eq_(list(chunked([1], 10)), [(1,)])
# chunking a list where len(list) == n
eq_(list(chunked([1, 2], 2)), [(1, 2)])
# chunking list where len(list) > n
eq_(list(chunked([1, 2, 3, 4, 5], 2)),
[(1, 2), (3, 4), (5,)])
def handle_reindex(request):
"""Caculate chunks and kick off indexing tasks."""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
return HttpResponseRedirect(request.path)
def handle_reindex(request):
"""Caculate chunks and kick off indexing tasks."""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0],
id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id, (cls, id_list))
return HttpResponseRedirect(request.path)
def reindex():
"""Calculates and creates indexing chunks"""
index = get_index_name()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls._doc_type.name, id_list[0],
id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
def reindex():
"""Calculates and creates indexing chunks"""
index = get_index()
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable():
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record(batch_id=batch_id, name=chunk_name)
rec.save()
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
