Beispiel #1
0
def index_task(cls_path, id_list, **kw):
    """Index documents specified by cls and ids"""
    cls = from_class_path(cls_path)
    try:
        # Pin to master db to avoid replication lag issues and stale
        # data.
        pin_this_thread()

        qs = cls.get_model().objects.filter(pk__in=id_list).values_list(
            "pk", flat=True)
        for id_ in qs:
            try:
                cls.index(cls.extract_document(id_), id_=id_)
            except UnindexMeBro:
                # If extract_document throws this, then we need to
                # remove this item from the index.
                cls.unindex(id_)

    except Exception as exc:
        retries = index_task.request.retries
        if retries >= MAX_RETRIES:
            # Some exceptions aren't pickleable and we need this to
            # throw things that are pickleable.
            raise IndexingTaskError()

        index_task.retry(exc=exc,
                         max_retries=MAX_RETRIES,
                         countdown=RETRY_TIMES[retries])
    finally:
        unpin_this_thread()
Beispiel #2
0
def index_chunk_task(write_index, batch_id, rec_id, chunk):
    """Index a chunk of things.

    :arg write_index: the name of the index to index to
    :arg batch_id: the name for the batch this chunk belongs to
    :arg rec_id: the id for the record for this task
    :arg chunk: a (class, id_list) of things to index
    """
    cls_path, id_list = chunk
    cls = from_class_path(cls_path)
    rec = None

    # Need to import Record here to prevent circular import
    from kitsune.search.models import Record

    try:
        # Pin to master db to avoid replication lag issues and stale data.
        pin_this_thread()

        # Update record data.
        rec = Record.objects.get(pk=rec_id)
        rec.start_time = datetime.datetime.now()
        rec.message = u'Reindexing into %s' % write_index
        rec.status = Record.STATUS_IN_PROGRESS
        rec.save()

        index_chunk(cls, id_list, reraise=True)
        rec.mark_success()

    except Exception:
        if rec is not None:
            rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value))

        log.exception('Error while indexing a chunk')
        # Some exceptions aren't pickleable and we need this to throw
        # things that are pickleable.
        raise IndexingTaskError()

    finally:
        unpin_this_thread()
Beispiel #3
0
def test_from_class_path():
    eq_(from_class_path('kitsune.search.tests.test_utils:FooBarClassOfAwesome'),
        FooBarClassOfAwesome)
Beispiel #4
0
def test_from_class_path():
    eq_(
        from_class_path(
            "kitsune.search.tests.test_utils:FooBarClassOfAwesome"),
        FooBarClassOfAwesome,
    )