def index_task(cls_path, id_list, **kw): """Index documents specified by cls and ids""" cls = from_class_path(cls_path) try: # Pin to master db to avoid replication lag issues and stale # data. pin_this_thread() qs = cls.get_model().objects.filter(pk__in=id_list).values_list( "pk", flat=True) for id_ in qs: try: cls.index(cls.extract_document(id_), id_=id_) except UnindexMeBro: # If extract_document throws this, then we need to # remove this item from the index. cls.unindex(id_) except Exception as exc: retries = index_task.request.retries if retries >= MAX_RETRIES: # Some exceptions aren't pickleable and we need this to # throw things that are pickleable. raise IndexingTaskError() index_task.retry(exc=exc, max_retries=MAX_RETRIES, countdown=RETRY_TIMES[retries]) finally: unpin_this_thread()
def index_chunk_task(write_index, batch_id, rec_id, chunk): """Index a chunk of things. :arg write_index: the name of the index to index to :arg batch_id: the name for the batch this chunk belongs to :arg rec_id: the id for the record for this task :arg chunk: a (class, id_list) of things to index """ cls_path, id_list = chunk cls = from_class_path(cls_path) rec = None # Need to import Record here to prevent circular import from kitsune.search.models import Record try: # Pin to master db to avoid replication lag issues and stale data. pin_this_thread() # Update record data. rec = Record.objects.get(pk=rec_id) rec.start_time = datetime.datetime.now() rec.message = u'Reindexing into %s' % write_index rec.status = Record.STATUS_IN_PROGRESS rec.save() index_chunk(cls, id_list, reraise=True) rec.mark_success() except Exception: if rec is not None: rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value)) log.exception('Error while indexing a chunk') # Some exceptions aren't pickleable and we need this to throw # things that are pickleable. raise IndexingTaskError() finally: unpin_this_thread()
def test_from_class_path(): eq_(from_class_path('kitsune.search.tests.test_utils:FooBarClassOfAwesome'), FooBarClassOfAwesome)
def test_from_class_path(): eq_( from_class_path( "kitsune.search.tests.test_utils:FooBarClassOfAwesome"), FooBarClassOfAwesome, )