def test_delete(self):
note1 = Note.objects.create(title="A test note", content="Because everyone loves test data.", author="Daniel")
note2 = Note.objects.create(title="Another test note", content="More test data.", author="Daniel")
note3 = Note.objects.create(title="Final test note", content="The test data. All done.", author="Joe")
# Dump the queue in preparation for the deletes.
queues.delete_queue(get_queue_name())
self.queue = queues.Queue(get_queue_name())
self.assertEqual(len(self.queue), 0)
note1.delete()
self.assertEqual(len(self.queue), 1)
note2.delete()
self.assertEqual(len(self.queue), 2)
note3.delete()
self.assertEqual(len(self.queue), 3)
# Pull the whole queue.
messages = []
try:
while True:
messages.append(self.queue.read())
except QueueException:
# We're out of queued bits.
pass
self.assertEqual(messages, [u"delete:tests.note.1", u"delete:tests.note.2", u"delete:tests.note.3"])
def setUp(self):
super(QueuedSearchIndexTestCase, self).setUp()
# Nuke the queue.
queues.delete_queue(get_queue_name())
# Nuke the index.
call_command('clear_index', interactive=False, verbosity=0)
# Get a queue connection so we can poke at it.
self.queue = queues.Queue(get_queue_name())
def setUp(self):
super(QueuedSearchIndexTestCase, self).setUp()
# Nuke the queue.
queues.delete_queue(get_queue_name())
# Nuke the index.
call_command('clear_index', interactive=False, verbosity=0)
# Get a queue connection so we can poke at it.
self.queue = queues.Queue(get_queue_name())
def setUp(self):
super(ProcessSearchQueueTestCase, self).setUp()
# Nuke the queue.
queues.delete_queue(get_queue_name())
# Nuke the index.
call_command('clear_index', interactive=False, verbosity=0)
# Get a queue connection so we can poke at it.
self.queue = queues.Queue(get_queue_name())
# Clear out and capture log messages.
AssertableHandler.stowed_messages = []
self.psqc = ProcessSearchQueueCommand()
def handle_noargs(self, **options):
self.batchsize = options.get('batchsize', DEFAULT_BATCH_SIZE) or 1000
self.using = options.get('using')
# Setup the queue.
self.queue = queues.Queue(get_queue_name())
# Check if enough is there to process.
if not len(self.queue):
self.log.info("Not enough items in the queue to process.")
self.log.info("Starting to process the queue.")
# Consume the whole queue first so that we can group update/deletes
# for efficiency.
try:
while True:
message = self.queue.read()
if not message:
break
self.process_message(message)
except QueueException:
# We've run out of items in the queue.
pass
self.log.info("Queue consumed.")
try:
self.handle_updates()
self.handle_deletes()
except Exception, e:
self.log.error('Exception seen during processing: %s' % e)
self.requeue()
raise e
def setUp(self):
super(ProcessSearchQueueTestCase, self).setUp()
# Nuke the queue.
queues.delete_queue(get_queue_name())
# Nuke the index.
call_command('clear_index', interactive=False, verbosity=0)
# Get a queue connection so we can poke at it.
self.queue = queues.Queue(get_queue_name())
# Clear out and capture log messages.
AssertableHandler.stowed_messages = []
self.psqc = ProcessSearchQueueCommand()
def test_delete(self):
note1 = Note.objects.create(
title='A test note',
content='Because everyone loves test data.',
author='Daniel'
)
note2 = Note.objects.create(
title='Another test note',
content='More test data.',
author='Daniel'
)
note3 = Note.objects.create(
title='Final test note',
content='The test data. All done.',
author='Joe'
)
# Dump the queue in preparation for the deletes.
queues.delete_queue(get_queue_name())
self.queue = queues.Queue(get_queue_name())
self.assertEqual(len(self.queue), 0)
note1.delete()
self.assertEqual(len(self.queue), 1)
note2.delete()
self.assertEqual(len(self.queue), 2)
note3.delete()
self.assertEqual(len(self.queue), 3)
# Pull the whole queue.
messages = []
try:
while True:
messages.append(self.queue.read())
except QueueException:
# We're out of queued bits.
pass
self.assertEqual(messages, [u'delete:tests.note.1', u'delete:tests.note.2', u'delete:tests.note.3'])
def enqueue(self, action, instance):
"""
Shoves a message about how to update the index into the queue.
This is a standardized string, resembling something like::
``update:notes.note.23``
# ...or...
``delete:weblog.entry.8``
"""
message = "%s:%s" % (action, get_identifier(instance))
queue = queues.Queue(get_queue_name())
return queue.write(message)
def enqueue(self, action, instance):
"""
Shoves a message about how to update the index into the queue.
This is a standardized string, resembling something like::
``update:notes.note.23``
# ...or...
``delete:weblog.entry.8``
"""
message = "%s:%s" % (action, get_identifier(instance))
queue = Queue(get_queue_name())
return queue.write(message)
def test_delete(self):
note1 = Note.objects.create(
title='A test note',
content='Because everyone loves test data.',
author='Daniel')
note2 = Note.objects.create(title='Another test note',
content='More test data.',
author='Daniel')
note3 = Note.objects.create(title='Final test note',
content='The test data. All done.',
author='Joe')
# Dump the queue in preparation for the deletes.
queues.delete_queue(get_queue_name())
self.queue = queues.Queue(get_queue_name())
self.assertEqual(len(self.queue), 0)
note1.delete()
self.assertEqual(len(self.queue), 1)
note2.delete()
self.assertEqual(len(self.queue), 2)
note3.delete()
self.assertEqual(len(self.queue), 3)
# Pull the whole queue.
messages = []
try:
while True:
messages.append(self.queue.read())
except QueueException:
# We're out of queued bits.
pass
self.assertEqual(messages, [
u'delete:tests.note.1', u'delete:tests.note.2',
u'delete:tests.note.3'
])
def handle_noargs(self, **options):
self.batchsize = options.get('batchsize', DEFAULT_BATCH_SIZE) or 1000
self.using = options.get('using')
# Setup the queue.
self.queue = queues.Queue(get_queue_name())
# Check if enough is there to process.
if not len(self.queue):
self.log.debug("Not enough items in the queue to process.")
return
self.log.debug("Starting to process the queue.")
start_time = datetime.datetime.now()
items = 0
# Consume the whole queue first so that we can group update/deletes
# for efficiency.
try:
while True:
message = self.queue.read()
if not message:
break
self.process_message(message)
items += 1
except QueueException:
# We've run out of items in the queue.
pass
self.log.debug('Queue consumed {} items in {}.'.format(items, datetime.datetime.now() - start_time))
start_time = datetime.datetime.now()
try:
self.handle_updates()
self.handle_deletes()
except Exception as e:
self.log.error('Exception seen during processing: %s' % e)
self.requeue()
raise e
self.log.info('Processed {} updates {} deletes in {}.'.format(
len(self.processed_updates), len(self.processed_deletes), datetime.datetime.now() - start_time))
def enqueue(self, action, instance):
"""
Shoves a message about how to update the index into the queue.
This is a standardized string, resembling something like::
``update:notes.note.23``
# ...or...
``delete:weblog.entry.8``
"""
if SKIP_NOINDEX:
# Check if the model even has a ``SearchIndex`` implementation.
try:
connections['default'].get_unified_index().get_index(instance.__class__)
except NotHandled:
return False
message = "%s:%s" % (action, default_get_identifier(instance))
queue = queues.Queue(get_queue_name())
return queue.write(message)
def handle_noargs(self, **options):
self.batchsize = options.get('batchsize', DEFAULT_BATCH_SIZE) or 1000
self.using = options.get('using')
# Setup the queue.
self.queue = queues.Queue(get_queue_name())
# Check if enough is there to process.
if not len(self.queue):
self.log.info("Not enough items in the queue to process.")
self.log.info("Starting to process the queue.")
# Consume the whole queue first so that we can group update/deletes
# for efficiency.
try:
while True:
message = self.queue.read()
if not message:
break
self.process_message(message)
except QueueException:
# We've run out of items in the queue.
pass
self.log.info("Queue consumed.")
try:
self.handle_updates()
self.handle_deletes()
except Exception as e:
self.log.error('Exception seen during processing: %s' % e)
self.requeue()
raise e
self.log.info("Processing complete.")