def index_chunk_task(write_index, batch_id, chunk):
"""Index a chunk of things.
:arg write_index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg chunk: a (class, id_list) of things to index
"""
# Need to import Record here to prevent circular import
from kitsune.search.models import Record
cls, id_list = chunk
task_name = '{0} {1} -> {2}'.format(cls.get_mapping_type_name(),
id_list[0], id_list[-1])
rec = Record.objects.create(
starttime=datetime.datetime.now(),
text=u'Batch: {0} Task: {1}: Reindexing into {2}'.format(
batch_id, task_name, write_index))
try:
# Pin to master db to avoid replication lag issues and stale
# data.
pin_this_thread()
index_chunk(cls, id_list, reraise=True)
except Exception:
rec.text = u'{0}: Errored out {1} {2}'.format(
rec.text, sys.exc_type,
sys.exc_value)[:255] # Truncate at 255 chars.
log.exception('Error while indexing a chunk')
# Some exceptions aren't pickleable and we need this to throw
# things that are pickleable.
raise IndexingTaskError()
finally:
unpin_this_thread()
rec.endtime = datetime.datetime.now()
rec.save()
try:
client = redis_client('default')
client.decr(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
# If Redis isn't running, then we just log that the task
# was completed.
log.info('Index task %s completed.', task_name)
def index_chunk_task(write_index, batch_id, chunk):
"""Index a chunk of things.
:arg write_index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg chunk: a (class, id_list) of things to index
"""
# Need to import Record here to prevent circular import
from kitsune.search.models import Record
cls, id_list = chunk
task_name = '{0} {1} -> {2}'.format(
cls.get_mapping_type_name(), id_list[0], id_list[-1])
rec = Record.objects.create(
starttime=datetime.datetime.now(),
text=u'Batch: {0} Task: {1}: Reindexing into {2}'.format(
batch_id, task_name, write_index))
try:
# Pin to master db to avoid replication lag issues and stale
# data.
pin_this_thread()
index_chunk(cls, id_list, reraise=True)
except Exception:
rec.text = u'{0}: Errored out {1} {2}'.format(
rec.text, sys.exc_type, sys.exc_value)[:255] # Truncate at 255 chars.
log.exception('Error while indexing a chunk')
# Some exceptions aren't pickleable and we need this to throw
# things that are pickleable.
raise IndexingTaskError()
finally:
unpin_this_thread()
rec.endtime = datetime.datetime.now()
rec.save()
try:
client = redis_client('default')
client.decr(OUTSTANDING_INDEX_CHUNKS, 1)
except RedisError:
# If Redis isn't running, then we just log that the task
# was completed.
log.info('Index task %s completed.', task_name)
def index_chunk_task(write_index, batch_id, rec_id, chunk):
"""Index a chunk of things.
:arg write_index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg rec_id: the id for the record for this task
:arg chunk: a (class, id_list) of things to index
"""
cls_path, id_list = chunk
cls = from_class_path(cls_path)
rec = None
# Need to import Record here to prevent circular import
from kitsune.search.models import Record
try:
# Pin to master db to avoid replication lag issues and stale data.
pin_this_thread()
# Update record data.
rec = Record.objects.get(pk=rec_id)
rec.start_time = datetime.datetime.now()
rec.message = "Reindexing into %s" % write_index
rec.status = Record.STATUS_IN_PROGRESS
rec.save()
index_chunk(cls, id_list, reraise=True)
rec.mark_success()
except Exception:
if rec is not None:
rec.mark_fail("Errored out %s %s" %
(sys.exc_info()[0], sys.exc_info()[1]))
log.exception("Error while indexing a chunk")
# Some exceptions aren't pickleable and we need this to throw
# things that are pickleable.
raise IndexingTaskError()
finally:
unpin_this_thread()
def index_chunk_task(write_index, batch_id, rec_id, chunk):
"""Index a chunk of things.
:arg write_index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg rec_id: the id for the record for this task
:arg chunk: a (class, id_list) of things to index
"""
cls_path, id_list = chunk
cls = from_class_path(cls_path)
rec = None
# Need to import Record here to prevent circular import
from kitsune.search.models import Record
try:
# Pin to master db to avoid replication lag issues and stale data.
pin_this_thread()
# Update record data.
rec = Record.objects.get(pk=rec_id)
rec.start_time = datetime.datetime.now()
rec.message = u'Reindexing into %s' % write_index
rec.status = Record.STATUS_IN_PROGRESS
rec.save()
index_chunk(cls, id_list, reraise=True)
rec.mark_success()
except Exception:
if rec is not None:
rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value))
log.exception('Error while indexing a chunk')
# Some exceptions aren't pickleable and we need this to throw
# things that are pickleable.
raise IndexingTaskError()
finally:
unpin_this_thread()