def reindex(mapping_type_names):
"""Reindex all instances of a given mapping type with celery tasks
:arg mapping_type_names: list of mapping types to reindex
"""
outstanding = Record.objects.outstanding().count()
if outstanding > 0:
raise ReindexError('There are %s outstanding chunks.' % outstanding)
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_type_names):
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
index = cls.get_index()
chunk_name = 'Indexing: %s %d -> %d' % (cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record.objects.create(batch_id=batch_id, name=chunk_name)
index_chunk_task.delay(index, batch_id, rec.id,
(to_class_path(cls), id_list))
def reindex(mapping_type_names):
"""Reindex all instances of a given mapping type with celery tasks
:arg mapping_type_names: list of mapping types to reindex
"""
outstanding = Record.objects.outstanding().count()
if outstanding > 0:
raise ReindexError('There are %s outstanding chunks.' % outstanding)
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_type_names):
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
for cls, id_list in chunks:
index = cls.get_index()
chunk_name = 'Indexing: %s %d -> %d' % (
cls.get_mapping_type_name(), id_list[0], id_list[-1])
rec = Record.objects.create(batch_id=batch_id, name=chunk_name)
index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))
def reindex_with_scoreboard(mapping_type_names):
"""Reindex all instances of a given mapping type with celery tasks.
This will use Redis to keep track of outstanding tasks so nothing
gets screwed up by two jobs running at once.
"""
# TODO: If this gets fux0rd, then it's possible this could be
# non-zero and we really want to just ignore it. Need the ability
# to ignore it.
try:
client = redis_client('default')
val = client.get(OUTSTANDING_INDEX_CHUNKS)
if val is not None and int(val) > 0:
raise ReindexError('There are %s outstanding chunks.' % val)
# We don't know how many chunks we're building, but we do want
# to make sure another reindex request doesn't slide in here
# and kick off a bunch of chunks.
#
# There is a race condition here.
client.set(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
log.warning('Redis not running. Can not check if there are '
'outstanding tasks.')
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size. Also generate
# reconcile_tasks.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_type_names):
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
reconcile_task.delay(cls.get_index(), batch_id,
cls.get_mapping_type_name())
chunks_count = len(chunks)
try:
client = redis_client('default')
client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count)
except RedisError:
log.warning('Redis not running. Can\'t denote outstanding tasks.')
for chunk in chunks:
index = chunk[0].get_index()
index_chunk_task.delay(index, batch_id, chunk)
def reindex_with_scoreboard(mapping_type_names):
"""Reindex all instances of a given mapping type with celery tasks.
This will use Redis to keep track of outstanding tasks so nothing
gets screwed up by two jobs running at once.
"""
# TODO: If this gets fux0rd, then it's possible this could be
# non-zero and we really want to just ignore it. Need the ability
# to ignore it.
try:
client = redis_client('default')
val = client.get(OUTSTANDING_INDEX_CHUNKS)
if val is not None and int(val) > 0:
raise ReindexError('There are %s outstanding chunks.' % val)
# We don't know how many chunks we're building, but we do want
# to make sure another reindex request doesn't slide in here
# and kick off a bunch of chunks.
#
# There is a race condition here.
client.set(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
log.warning('Redis not running. Can not check if there are '
'outstanding tasks.')
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size. Also generate
# reconcile_tasks.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_type_names):
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
reconcile_task.delay(cls.get_index(), batch_id,
cls.get_mapping_type_name())
chunks_count = len(chunks)
try:
client = redis_client('default')
client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count)
except RedisError:
log.warning('Redis not running. Can\'t denote outstanding tasks.')
for chunk in chunks:
index = chunk[0].get_index()
index_chunk_task.delay(index, batch_id, chunk)
def handle_reindex(request):
"""Caculates and kicks off indexing tasks"""
# This is truthy if the user wants us to delete and recreate
# the index first.
delete_index_first = bool(request.POST.get('delete_index'))
if delete_index_first:
# Coming from the delete form, so we reindex all models.
mapping_types_to_index = None
else:
# Coming from the reindex form, so we reindex whatever we're
# told.
mapping_types_to_index = [name.replace('check_', '')
for name in request.POST.keys()
if name.startswith('check_')]
# TODO: If this gets fux0rd, then it's possible this could be
# non-zero and we really want to just ignore it. Need the ability
# to ignore it.
try:
client = redis_client('default')
val = client.get(OUTSTANDING_INDEX_CHUNKS)
if val is not None and int(val) > 0:
raise ReindexError('There are %s outstanding chunks.' % val)
# We don't know how many chunks we're building, but we do want
# to make sure another reindex request doesn't slide in here
# and kick off a bunch of chunks.
#
# There is a race condition here.
client.set(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
log.warning('Redis not running. Can not check if there are '
'outstanding tasks.')
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_types_to_index):
chunks.extend(
(cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
if delete_index_first:
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
chunks_count = len(chunks)
try:
client = redis_client('default')
client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count)
except RedisError:
log.warning('Redis not running. Can\'t denote outstanding tasks.')
for chunk in chunks:
index_chunk_task.delay(write_index(), batch_id, chunk)
return HttpResponseRedirect(request.path)
def handle_reindex(request):
"""Caculates and kicks off indexing tasks"""
# This is truthy if the user wants us to delete and recreate
# the index first.
delete_index_first = bool(request.POST.get('delete_index'))
if delete_index_first:
# Coming from the delete form, so we reindex all models.
mapping_types_to_index = None
else:
# Coming from the reindex form, so we reindex whatever we're
# told.
mapping_types_to_index = [
name.replace('check_', '') for name in request.POST.keys()
if name.startswith('check_')
]
# TODO: If this gets fux0rd, then it's possible this could be
# non-zero and we really want to just ignore it. Need the ability
# to ignore it.
try:
client = redis_client('default')
val = client.get(OUTSTANDING_INDEX_CHUNKS)
if val is not None and int(val) > 0:
raise ReindexError('There are %s outstanding chunks.' % val)
# We don't know how many chunks we're building, but we do want
# to make sure another reindex request doesn't slide in here
# and kick off a bunch of chunks.
#
# There is a race condition here.
client.set(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
log.warning('Redis not running. Can not check if there are '
'outstanding tasks.')
batch_id = create_batch_id()
# Break up all the things we want to index into chunks. This
# chunkifies by class then by chunk size.
chunks = []
for cls, indexable in get_indexable(mapping_types=mapping_types_to_index):
chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))
if delete_index_first:
# The previous lines do a lot of work and take some time to
# execute. So we wait until here to wipe and rebuild the
# index. That reduces the time that there is no index by a little.
recreate_index()
chunks_count = len(chunks)
try:
client = redis_client('default')
client.set(OUTSTANDING_INDEX_CHUNKS, chunks_count)
except RedisError:
log.warning('Redis not running. Can\'t denote outstanding tasks.')
for chunk in chunks:
index_chunk_task.delay(write_index(), batch_id, chunk)
return HttpResponseRedirect(request.path)