def index_chunk_task(write_index, batch_id, chunk): """Index a chunk of things. :arg write_index: the name of the index to index to :arg batch_id: the name for the batch this chunk belongs to :arg chunk: a (class, id_list) of things to index """ # Need to import Record here to prevent circular import from search.models import Record cls, id_list = chunk task_name = '{0} {1} -> {2}'.format( cls.get_model_name(), id_list[0], id_list[-1]) rec = Record( starttime=datetime.datetime.now(), text=(u'Batch: %s Task: %s: Reindexing into %s' % ( batch_id, task_name, write_index))) rec.save() try: # Pin to master db to avoid replication lag issues and stale # data. pin_this_thread() index_chunk(cls, id_list, reraise=True) except Exception: rec.text = (u'%s: Errored out %s %s' % ( rec.text, sys.exc_type, sys.exc_value)) # Some exceptions aren't pickleable and we need this to throw # things that are pickleable. raise IndexingTaskError() finally: unpin_this_thread() rec.endtime = datetime.datetime.now() rec.save() try: client = redis_client('default') client.decr(OUTSTANDING_INDEX_CHUNKS, 1) except RedisError: # If Redis isn't running, then we just log that the task # was completed. log.info('Index task %s completed.', task_name)
def index_chunk_task(write_index, batch_id, chunk): """Index a chunk of things. :arg write_index: the name of the index to index to :arg batch_id: the name for the batch this chunk belongs to :arg chunk: a (class, id_list) of things to index """ # Need to import Record here to prevent circular import from search.models import Record cls, id_list = chunk task_name = '{0} {1} -> {2}'.format( cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record( starttime=datetime.datetime.now(), text=(u'Batch: %s Task: %s: Reindexing into %s' % ( batch_id, task_name, write_index))) rec.save() try: # Pin to master db to avoid replication lag issues and stale # data. pin_this_thread() index_chunk(cls, id_list, reraise=True) except Exception: rec.text = (u'%s: Errored out %s %s' % ( rec.text, sys.exc_type, sys.exc_value)) # Some exceptions aren't pickleable and we need this to throw # things that are pickleable. raise IndexingTaskError() finally: unpin_this_thread() rec.endtime = datetime.datetime.now() rec.save() try: client = redis_client('default') client.decr(OUTSTANDING_INDEX_CHUNKS, 1) except RedisError: # If Redis isn't running, then we just log that the task # was completed. log.info('Index task %s completed.', task_name)
def index_chunk_task(write_index, batch_id, chunk): """Index a chunk of things. :arg write_index: the name of the index to index to :arg batch_id: the name for the batch this chunk belongs to :arg chunk: a (class, id_list) of things to index """ # Need to import Record here to prevent circular import from search.models import Record cls, id_list = chunk task_name = '%s %d -> %d' % (cls.get_model_name(), id_list[0], id_list[-1]) rec = Record( starttime=datetime.datetime.now(), text=(u'Batch: %s Task: %s: Reindexing into %s' % ( batch_id, task_name, write_index))) rec.save() try: index_chunk(cls, id_list, reraise=True) except Exception: rec.text = (u'%s: Errored out %s %s' % ( rec.text, sys.exc_type, sys.exc_value)) raise finally: rec.endtime = datetime.datetime.now() rec.save() try: client = redis_client('default') client.decr(OUTSTANDING_INDEX_CHUNKS, 1) except RedisError: # If Redis isn't running, then we just log that the task # was completed. log.info('Index task %s completed.', task_name)