def _live_index_handler(sender, **kwargs): if (not settings.ES_LIVE_INDEX or 'signal' not in kwargs or 'instance' not in kwargs): return instance = kwargs['instance'] try: if kwargs['signal'] == post_save: cls_path = to_class_path(instance.get_doctype()) index_item_task.delay(cls_path, instance.id) elif kwargs['signal'] == pre_delete: cls_path = to_class_path(instance.get_doctype()) unindex_item_task.delay(cls_path, instance.id) except Exception: # At this point, we're trying to create an indexing task for # some response that's changed. When an indexing task is # created, it uses amqp to connect to rabbitmq to put the # new task in the queue. If a user is leaving feadback and # this fails (which it does with some regularity), the user # gets an HTTP 500 which stinks. # # The problem is exacerbated by the fact I don't know the full # list of exceptions that can get kicked up here. So what # we're going to do is catch them all, look for "amqp" in the # frames and if it's there, we'll ignore the exception and # send an email. We can collect reasons and narrow this down # at some point if that makes sense to do. If "amqp" is not in # the frames, then it's some other kind of error that we want # to show up, so we'll re-raise it. Sorry, user! # # In this way, users will stop seeing HTTP 500 errors during # rabbitmq outages. exc_type, exc_value, exc_tb = sys.exc_info() frames = traceback.extract_tb(exc_tb) for fn, ln, fun, text in frames: if 'amqp' in fn: # This is an amqp frame which indicates that we # should ignore this and send an email. mail_admins( subject='amqp error', message=( 'amqp error:\n\n' + traceback.format_exc() ) ) return # No amqp frames, so re-raise it. raise
def _live_index_handler(sender, **kwargs): if (not settings.ES_LIVE_INDEX or 'signal' not in kwargs or 'instance' not in kwargs): return instance = kwargs['instance'] if kwargs['signal'] == post_save: cls_path = to_class_path(instance.get_mapping_type()) index_item_task.delay(cls_path, instance.id) elif kwargs['signal'] == pre_delete: cls_path = to_class_path(instance.get_mapping_type()) unindex_item_task(cls_path, instance.id)
def _live_index_handler(sender, **kwargs): if (not settings.ES_LIVE_INDEX or 'signal' not in kwargs or 'instance' not in kwargs): return instance = kwargs['instance'] if kwargs['signal'] == post_save: cls_path = to_class_path(instance.get_mapping_type()) index_item_task.delay(cls_path, instance.id) elif kwargs['signal'] == pre_delete: cls_path = to_class_path(instance.get_mapping_type()) unindex_item_task.delay(cls_path, instance.id)
def _live_index_handler(sender, **kwargs): if (not settings.ES_LIVE_INDEX or 'signal' not in kwargs or 'instance' not in kwargs): return instance = kwargs['instance'] try: if kwargs['signal'] == post_save: cls_path = to_class_path(instance.get_mapping_type()) index_item_task.delay(cls_path, instance.id) elif kwargs['signal'] == pre_delete: cls_path = to_class_path(instance.get_mapping_type()) unindex_item_task.delay(cls_path, instance.id) except Exception: # At this point, we're trying to create an indexing task for # some response that's changed. When an indexing task is # created, it uses amqp to connect to rabbitmq to put the # new task in the queue. If a user is leaving feadback and # this fails (which it does with some regularity), the user # gets an HTTP 500 which stinks. # # The problem is exacerbated by the fact I don't know the full # list of exceptions that can get kicked up here. So what # we're going to do is catch them all, look for "amqp" in the # frames and if it's there, we'll ignore the exception and # send an email. We can collect reasons and narrow this down # at some point if that makes sense to do. If "amqp" is not in # the frames, then it's some other kind of error that we want # to show up, so we'll re-raise it. Sorry, user! # # In this way, users will stop seeing HTTP 500 errors during # rabbitmq outages. exc_type, exc_value, exc_tb = sys.exc_info() frames = traceback.extract_tb(exc_tb) for fn, ln, fun, text in frames: if 'amqp' in fn: # This is an amqp frame which indicates that we # should ignore this and send an email. mail_admins(subject='amqp error', message=('amqp error:\n\n' + traceback.format_exc())) return # No amqp frames, so re-raise it. raise
def handle_reindex(request): """Caculate chunks and kick off indexing tasks.""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend( (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list)) return HttpResponseRedirect(request.path)
def test_index_chunk_task(self): responses = ResponseFactory.create_batch(10) # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. assert ResponseDocType.docs.search().count() == 0 # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = RecordFactory(batch_id=batch_id) chunk = ( to_class_path(ResponseDocType), [item.id for item in responses] ) index_chunk_task.delay(get_index_name(), batch_id, rec.id, chunk) self.refresh() # Verify everything is in the index now. assert ResponseDocType.docs.search().count() == 10 # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) assert rec.status == Record.STATUS_SUCCESS
def handle_reindex(request): """Caculate chunks and kick off indexing tasks.""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) # The previous lines do a lot of work and take some time to # execute. So we wait until here to wipe and rebuild the # index. That reduces the time that there is no index by a little. recreate_index() for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list)) return HttpResponseRedirect(request.path)
def test_index_chunk_task(self): responses = ResponseFactory.create_batch(10) # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. eq_(len(ResponseMappingType.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = RecordFactory(batch_id=batch_id) chunk = (to_class_path(ResponseMappingType), [item.id for item in responses]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) ResponseMappingType.refresh_index() # Verify everything is in the index now. eq_(len(ResponseMappingType.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def test_index_chunk_task(self): responses = [response(save=True) for i in range(10)] # With live indexing, that'll create items in the index. Since # we want to test index_chunk_test, we need a clean index to # start with so we delete and recreate it. self.setup_indexes(empty=True) # Verify there's nothing in the index. eq_(len(ResponseMappingType.search()), 0) # Create the record and the chunk and then run it through # celery. batch_id = 'ou812' rec = record(batch_id=batch_id, save=True) chunk = (to_class_path(ResponseMappingType), [item.id for item in responses]) index_chunk_task.delay(get_index(), batch_id, rec.id, chunk) ResponseMappingType.refresh_index() # Verify everything is in the index now. eq_(len(ResponseMappingType.search()), 10) # Verify the record was marked succeeded. rec = Record.objects.get(pk=rec.id) eq_(rec.status, Record.STATUS_SUCCESS)
def _live_index_handler(sender, **kwargs): if (not settings.ES_LIVE_INDEX or 'signal' not in kwargs or 'instance' not in kwargs): return instance = kwargs['instance'] try: if kwargs['signal'] == post_save: cls_path = to_class_path(instance.get_mapping_type()) index_item_task.delay(cls_path, instance.id) elif kwargs['signal'] == pre_delete: cls_path = to_class_path(instance.get_mapping_type()) unindex_item_task.delay(cls_path, instance.id) except socket.error: # If we have a socket error here, it means we couldn't connect # to amqp. FIXME: Remove this at some point. But we have it # here now for cron jobs. pass
def reindex(): """Calculates and creates indexing chunks""" index = get_index_name() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend((cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls._doc_type.name, id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))
def reindex(): """Calculates and creates indexing chunks""" index = get_index() batch_id = create_batch_id() # Break up all the things we want to index into chunks. This # chunkifies by class then by chunk size. chunks = [] for cls, indexable in get_indexable(): chunks.extend( (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE)) for cls, id_list in chunks: chunk_name = '%s %d -> %d' % (cls.get_mapping_type_name(), id_list[0], id_list[-1]) rec = Record(batch_id=batch_id, name=chunk_name) rec.save() index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))
def test_to_class_path(): eq_(to_class_path(FooBarClassOfAwesome), 'fjord.search.tests.test_utils:FooBarClassOfAwesome')
def test_to_class_path(): assert( to_class_path(FooBarClassOfAwesome) == 'fjord.search.tests.test_utils:FooBarClassOfAwesome' )
def test_to_class_path(): eq_( to_class_path(FooBarClassOfAwesome), 'fjord.search.tests.test__utils:FooBarClassOfAwesome' )
def test_to_class_path(): assert (to_class_path(FooBarClassOfAwesome) == 'fjord.search.tests.test_utils:FooBarClassOfAwesome')