Пример #1
0
def index_chunk_task(index, batch_id, rec_id, chunk):
    """Index a chunk of things.

    :arg index: the name of the index to index to
    :arg batch_id: the name for the batch this chunk belongs to
    :arg rec_id: the id for the record for this task
    :arg chunk: a (cls_path, id_list) of things to index
    """
    cls_path, id_list = chunk
    cls = from_class_path(cls_path)
    rec = None

    try:
        # Pin to master db to avoid replication lag issues and stale
        # data.
        pin_this_thread()

        # Update record data.
        rec = Record.objects.get(pk=rec_id)
        rec.start_time = datetime.datetime.now()
        rec.message = u'Reindexing into %s' % index
        rec.status = Record.STATUS_IN_PROGRESS
        rec.save()

        index_chunk(cls, id_list)

        rec.mark_success()

    except Exception:
        if rec is not None:
            rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value))
        raise

    finally:
        unpin_this_thread()
Пример #2
0
def unindex_item_task(cls_path, item_id, **kwargs):
    """Remove item from index, given it's mapping_type class_path and id"""
    mapping_type = from_class_path(cls_path)
    try:
        mapping_type.unindex(item_id)

    except Exception as exc:
        retries = kwargs.get('task_retries', 0)
        log.exception("Error while live unindexing %s %d: %s",
                      mapping_type, item_id, exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (mapping_type, item_id)
        if not kwargs:
            # Celery is lame. It requires that kwargs be non empty, but when
            # EAGER is true, it provides empty kwargs.
            kwargs['_dummy'] = True
        unindex_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #3
0
def unindex_item_task(cls_path, item_id, **kwargs):
    """Remove item from index, given it's mapping_type class_path and id"""
    mapping_type = from_class_path(cls_path)
    try:
        mapping_type.unindex(item_id)

    except Exception as exc:
        retries = kwargs.get('task_retries', 0)
        log.exception("Error while live unindexing %s %d: %s", mapping_type,
                      item_id, exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (mapping_type, item_id)
        if not kwargs:
            # Celery is lame. It requires that kwargs be non empty, but when
            # EAGER is true, it provides empty kwargs.
            kwargs['_dummy'] = True
        unindex_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #4
0
def index_item_task(cls_path, item_id, **kwargs):
    """Index an item given it's mapping_type cls_path and id"""
    mapping_type = from_class_path(cls_path)
    retries = kwargs.get('task_retries', 0)
    log.debug('Index attempt #%s', retries)
    try:
        doc = mapping_type.extract_document(item_id)
        mapping_type.index(doc, item_id)

    except Exception as exc:
        log.exception("Error while live indexing %s %d: %s",
                      mapping_type, item_id, exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (mapping_type, item_id)
        if not kwargs:
            # Celery is lame. It requires that kwargs be non empty, but when
            # EAGER is true, it provides empty kwargs.
            kwargs['_dummy'] = True
        index_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #5
0
def index_item_task(cls_path, item_id, **kwargs):
    """Index an item given it's mapping_type cls_path and id"""
    mapping_type = from_class_path(cls_path)
    retries = kwargs.get('task_retries', 0)
    log.debug('Index attempt #%s', retries)
    try:
        doc = mapping_type.extract_document(item_id)
        mapping_type.index(doc, item_id)

    except Exception as exc:
        log.exception("Error while live indexing %s %d: %s", mapping_type,
                      item_id, exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (cls_path, item_id)
        if not kwargs:
            # Celery requires that kwargs be non empty, but when EAGER
            # is true, it provides empty kwargs. Yay.
            kwargs['_dummy'] = True
        index_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #6
0
def index_item_task(cls_path, item_id, **kwargs):
    """Index an item given it's DocType cls_path and id"""
    doctype = from_class_path(cls_path)
    retries = kwargs.get('task_retries', 0)
    log.debug('Index attempt #%s', retries)
    try:
        resp = doctype.get_model().objects.get(id=item_id)
        doc = doctype.extract_doc(resp)
        doctype.docs.bulk_index(docs=[doc])

    except Exception as exc:
        log.exception('Error while live indexing %s %d: %s',
                      doctype, item_id, exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (cls_path, item_id)
        if not kwargs:
            # Celery requires that kwargs be non empty, but when EAGER
            # is true, it provides empty kwargs. Yay.
            kwargs['_dummy'] = True
        index_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #7
0
def index_item_task(cls_path, item_id, **kwargs):
    """Index an item given it's DocType cls_path and id"""
    doctype = from_class_path(cls_path)
    retries = kwargs.get('task_retries', 0)
    log.debug('Index attempt #%s', retries)
    try:
        resp = doctype.get_model().objects.get(id=item_id)
        doc = doctype.extract_doc(resp)
        doctype.docs.bulk_index(docs=[doc])

    except Exception as exc:
        log.exception('Error while live indexing %s %d: %s', doctype, item_id,
                      exc)
        if retries >= MAX_RETRIES:
            raise
        retry_time = RETRY_TIMES[retries]

        args = (cls_path, item_id)
        if not kwargs:
            # Celery requires that kwargs be non empty, but when EAGER
            # is true, it provides empty kwargs. Yay.
            kwargs['_dummy'] = True
        index_item_task.retry(args, kwargs, exc, countdown=retry_time)
Пример #8
0
def index_chunk_task(index, batch_id, rec_id, chunk):
    """Index a chunk of things.

    :arg index: the name of the index to index to
    :arg batch_id: the name for the batch this chunk belongs to
    :arg rec_id: the id for the record for this task
    :arg chunk: a (cls_path, id_list) of things to index
    """
    cls_path, id_list = chunk
    cls = from_class_path(cls_path)
    rec = None

    try:
        # Pin to master db to avoid replication lag issues and stale
        # data.
        pin_this_thread()

        # Update record data.
        rec = Record.objects.get(pk=rec_id)
        rec.start_time = datetime.datetime.now()
        rec.message = u'Reindexing into %s' % index
        rec.status = Record.STATUS_IN_PROGRESS
        rec.save()

        index_chunk(cls, id_list)

        rec.mark_success()

    except Exception:
        if rec is not None:
            rec.mark_fail(u'Errored out %s %s' % (
                sys.exc_type, sys.exc_value))
        raise

    finally:
        unpin_this_thread()
Пример #9
0
def test_from_class_path():
    eq_(from_class_path('fjord.search.tests.test_utils:FooBarClassOfAwesome'),
        FooBarClassOfAwesome)
Пример #10
0
def test_from_class_path():
    assert(
        from_class_path(
            'fjord.search.tests.test_utils:FooBarClassOfAwesome') ==
        FooBarClassOfAwesome
    )
Пример #11
0
def test_from_class_path():
    eq_(
        from_class_path(
            'fjord.search.tests.test__utils:FooBarClassOfAwesome'),
        FooBarClassOfAwesome
    )
Пример #12
0
def test_from_class_path():
    assert (
        from_class_path('fjord.search.tests.test_utils:FooBarClassOfAwesome')
        == FooBarClassOfAwesome)