def index_chunk_task(index, batch_id, rec_id, chunk):
"""Index a chunk of things.
:arg index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg rec_id: the id for the record for this task
:arg chunk: a (cls_path, id_list) of things to index
"""
cls_path, id_list = chunk
cls = from_class_path(cls_path)
rec = None
try:
# Pin to master db to avoid replication lag issues and stale
# data.
pin_this_thread()
# Update record data.
rec = Record.objects.get(pk=rec_id)
rec.start_time = datetime.datetime.now()
rec.message = u'Reindexing into %s' % index
rec.status = Record.STATUS_IN_PROGRESS
rec.save()
index_chunk(cls, id_list)
rec.mark_success()
except Exception:
if rec is not None:
rec.mark_fail(u'Errored out %s %s' % (sys.exc_type, sys.exc_value))
raise
finally:
unpin_this_thread()
def unindex_item_task(cls_path, item_id, **kwargs):
"""Remove item from index, given it's mapping_type class_path and id"""
mapping_type = from_class_path(cls_path)
try:
mapping_type.unindex(item_id)
except Exception as exc:
retries = kwargs.get('task_retries', 0)
log.exception("Error while live unindexing %s %d: %s",
mapping_type, item_id, exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (mapping_type, item_id)
if not kwargs:
# Celery is lame. It requires that kwargs be non empty, but when
# EAGER is true, it provides empty kwargs.
kwargs['_dummy'] = True
unindex_item_task.retry(args, kwargs, exc, countdown=retry_time)
def unindex_item_task(cls_path, item_id, **kwargs):
"""Remove item from index, given it's mapping_type class_path and id"""
mapping_type = from_class_path(cls_path)
try:
mapping_type.unindex(item_id)
except Exception as exc:
retries = kwargs.get('task_retries', 0)
log.exception("Error while live unindexing %s %d: %s", mapping_type,
item_id, exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (mapping_type, item_id)
if not kwargs:
# Celery is lame. It requires that kwargs be non empty, but when
# EAGER is true, it provides empty kwargs.
kwargs['_dummy'] = True
unindex_item_task.retry(args, kwargs, exc, countdown=retry_time)
def index_item_task(cls_path, item_id, **kwargs):
"""Index an item given it's mapping_type cls_path and id"""
mapping_type = from_class_path(cls_path)
retries = kwargs.get('task_retries', 0)
log.debug('Index attempt #%s', retries)
try:
doc = mapping_type.extract_document(item_id)
mapping_type.index(doc, item_id)
except Exception as exc:
log.exception("Error while live indexing %s %d: %s",
mapping_type, item_id, exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (mapping_type, item_id)
if not kwargs:
# Celery is lame. It requires that kwargs be non empty, but when
# EAGER is true, it provides empty kwargs.
kwargs['_dummy'] = True
index_item_task.retry(args, kwargs, exc, countdown=retry_time)
def index_item_task(cls_path, item_id, **kwargs):
"""Index an item given it's mapping_type cls_path and id"""
mapping_type = from_class_path(cls_path)
retries = kwargs.get('task_retries', 0)
log.debug('Index attempt #%s', retries)
try:
doc = mapping_type.extract_document(item_id)
mapping_type.index(doc, item_id)
except Exception as exc:
log.exception("Error while live indexing %s %d: %s", mapping_type,
item_id, exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (cls_path, item_id)
if not kwargs:
# Celery requires that kwargs be non empty, but when EAGER
# is true, it provides empty kwargs. Yay.
kwargs['_dummy'] = True
index_item_task.retry(args, kwargs, exc, countdown=retry_time)
def index_item_task(cls_path, item_id, **kwargs):
"""Index an item given it's DocType cls_path and id"""
doctype = from_class_path(cls_path)
retries = kwargs.get('task_retries', 0)
log.debug('Index attempt #%s', retries)
try:
resp = doctype.get_model().objects.get(id=item_id)
doc = doctype.extract_doc(resp)
doctype.docs.bulk_index(docs=[doc])
except Exception as exc:
log.exception('Error while live indexing %s %d: %s',
doctype, item_id, exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (cls_path, item_id)
if not kwargs:
# Celery requires that kwargs be non empty, but when EAGER
# is true, it provides empty kwargs. Yay.
kwargs['_dummy'] = True
index_item_task.retry(args, kwargs, exc, countdown=retry_time)
def index_item_task(cls_path, item_id, **kwargs):
"""Index an item given it's DocType cls_path and id"""
doctype = from_class_path(cls_path)
retries = kwargs.get('task_retries', 0)
log.debug('Index attempt #%s', retries)
try:
resp = doctype.get_model().objects.get(id=item_id)
doc = doctype.extract_doc(resp)
doctype.docs.bulk_index(docs=[doc])
except Exception as exc:
log.exception('Error while live indexing %s %d: %s', doctype, item_id,
exc)
if retries >= MAX_RETRIES:
raise
retry_time = RETRY_TIMES[retries]
args = (cls_path, item_id)
if not kwargs:
# Celery requires that kwargs be non empty, but when EAGER
# is true, it provides empty kwargs. Yay.
kwargs['_dummy'] = True
index_item_task.retry(args, kwargs, exc, countdown=retry_time)
def index_chunk_task(index, batch_id, rec_id, chunk):
"""Index a chunk of things.
:arg index: the name of the index to index to
:arg batch_id: the name for the batch this chunk belongs to
:arg rec_id: the id for the record for this task
:arg chunk: a (cls_path, id_list) of things to index
"""
cls_path, id_list = chunk
cls = from_class_path(cls_path)
rec = None
try:
# Pin to master db to avoid replication lag issues and stale
# data.
pin_this_thread()
# Update record data.
rec = Record.objects.get(pk=rec_id)
rec.start_time = datetime.datetime.now()
rec.message = u'Reindexing into %s' % index
rec.status = Record.STATUS_IN_PROGRESS
rec.save()
index_chunk(cls, id_list)
rec.mark_success()
except Exception:
if rec is not None:
rec.mark_fail(u'Errored out %s %s' % (
sys.exc_type, sys.exc_value))
raise
finally:
unpin_this_thread()
def test_from_class_path():
eq_(from_class_path('fjord.search.tests.test_utils:FooBarClassOfAwesome'),
FooBarClassOfAwesome)
def test_from_class_path():
assert(
from_class_path(
'fjord.search.tests.test_utils:FooBarClassOfAwesome') ==
FooBarClassOfAwesome
)
def test_from_class_path():
eq_(
from_class_path(
'fjord.search.tests.test__utils:FooBarClassOfAwesome'),
FooBarClassOfAwesome
)
def test_from_class_path():
assert (
from_class_path('fjord.search.tests.test_utils:FooBarClassOfAwesome')
== FooBarClassOfAwesome)
