def index_task(cls_path, id_list, **kw):
"""Index documents specified by cls and ids"""
cls = from_class_path(cls_path)
try:
# Pin to master db to avoid replication lag issues and stale
# data.
pin_this_thread()
qs = cls.get_model().objects.filter(pk__in=id_list).values_list(
"pk", flat=True)
for id_ in qs:
try:
cls.index(cls.extract_document(id_), id_=id_)
except UnindexMeBro:
# If extract_document throws this, then we need to
# remove this item from the index.
cls.unindex(id_)
except Exception as exc:
retries = index_task.request.retries
if retries >= MAX_RETRIES:
# Some exceptions aren't pickleable and we need this to
# throw things that are pickleable.
raise IndexingTaskError()
index_task.retry(exc=exc,
max_retries=MAX_RETRIES,
countdown=RETRY_TIMES[retries])
finally:
unpin_this_thread()
def index_chunk_task(write_index, batch_id, rec_id, chunk):
"""Index a chunk of things.
:arg write_index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg rec_id: the id for the record for this task
:arg chunk: a (class, id_list) of things to index
"""
cls_path, id_list = chunk
cls = from_class_path(cls_path)
rec = None
# Need to import Record here to prevent circular import
from kitsune.search.models import Record
try:
# Pin to master db to avoid replication lag issues and stale data.
pin_this_thread()
# Update record data.
rec = Record.objects.get(pk=rec_id)
rec.start_time = datetime.datetime.now()
rec.message = u'Reindexing into %s' % write_index
rec.status = Record.STATUS_IN_PROGRESS
rec.save()
index_chunk(cls, id_list, reraise=True)
rec.mark_success()
except Exception:
if rec is not None:
rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value))
log.exception('Error while indexing a chunk')
# Some exceptions aren't pickleable and we need this to throw
# things that are pickleable.
raise IndexingTaskError()
finally:
unpin_this_thread()
def test_from_class_path():
eq_(from_class_path('kitsune.search.tests.test_utils:FooBarClassOfAwesome'),
FooBarClassOfAwesome)
def test_from_class_path():
eq_(
from_class_path(
"kitsune.search.tests.test_utils:FooBarClassOfAwesome"),
FooBarClassOfAwesome,
)