def test_rerun_after_depletion_doesnt_delete_too_much(self):
"""Ensure MessageIterator works when used manually."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
iterator = MessageIterator('tag', 'qn', 1, auto_delete=False)
with patch.object(iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task]
results = [payload for payload in iterator]
self.assertEqual(results, [payload])
# This new work should never be leased, but simulates new pending
# work.
task_1 = Mock(payload='["task_1"]', tag='tag')
queue.lease_tasks_by_tag.return_value = [task_1]
# Iterating again should return the "originally leased" work, not
# new work.
results = [payload for payload in iterator]
self.assertEqual(results, [payload])
# Lease should only have been called a single time.
queue.lease_tasks_by_tag.assert_called_once_with(
60, 1, tag='tag')
# The delete call should delete only the original work.
iterator.delete_messages()
queue.delete_tasks.assert_called_once_with([task])
def test_rerun_after_depletion_doesnt_delete_too_much(self):
"""Ensure MessageIterator works when used manually."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
iterator = MessageIterator('tag', 'qn', 1, auto_delete=False)
with patch.object(iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task]
results = [payload for payload in iterator]
self.assertEqual(results, [payload])
# This new work should never be leased, but simulates new pending
# work.
task_1 = Mock(payload='["task_1"]', tag='tag')
queue.lease_tasks_by_tag.return_value = [task_1]
# Iterating again should return the "originally leased" work, not
# new work.
results = [payload for payload in iterator]
self.assertEqual(results, [payload])
# Lease should only have been called a single time.
queue.lease_tasks_by_tag.assert_called_once_with(60,
1,
tag='tag',
deadline=10)
# The delete call should delete only the original work.
iterator.delete_messages()
queue.delete_tasks.assert_called_once_with([task])
def test_raise_stopiteration_if_no_messages(self):
"""Ensure MessageIterator raises StopIteration if no messages."""
from furious.batcher import MessageIterator
iterator = MessageIterator('tag', 'qn', 1)
with patch.object(iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = []
self.assertRaises(StopIteration, iterator.next)
def test_time_check(self, time):
"""Ensure that a DeadlineExceededError is thrown when the lease takes
over (deadline-0.1) secs.
"""
from google.appengine.runtime import apiproxy_errors
from furious.batcher import MessageIterator
time.side_effect = [0.0, 9.9]
message_iterator = MessageIterator('tag', 'qn', 1)
with patch.object(message_iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = []
self.assertRaises(apiproxy_errors.DeadlineExceededError, iter,
message_iterator)
def test_iterates(self):
"""Ensure MessageIterator instances iterate in loop."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
task1 = Mock(payload=payload, tag='tag')
task2 = Mock(payload=payload, tag='tag')
iterator = MessageIterator('tag', 'qn', 1)
with patch.object(iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task, task1, task2]
results = [payload for payload in iterator]
self.assertEqual(results, [payload, payload, payload])
def test_rerun_after_depletion_calls_once(self):
"""Ensure MessageIterator works when used manually."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
iterator = MessageIterator('tag', 'qn', 1)
with patch.object(iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task]
results = [payload for payload in iterator]
self.assertEqual(results, [payload])
results = [payload for payload in iterator]
queue.lease_tasks_by_tag.assert_called_once_with(60, 1, tag='tag')
def test_calls_lease_exactly_once(self):
"""Ensure MessageIterator calls lease only once."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
message_iterator = MessageIterator('tag', 'qn', 1)
with patch.object(message_iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task]
iterator = iter(message_iterator)
iterator.next()
self.assertRaises(StopIteration, iterator.next)
self.assertRaises(StopIteration, iterator.next)
queue.lease_tasks_by_tag.assert_called_once_with(60, 1, tag='tag')
def test_custom_deadline(self):
"""Ensure that a custom deadline gets passed to lease_tasks."""
from furious.batcher import MessageIterator
payload = '["test"]'
task = Mock(payload=payload, tag='tag')
message_iterator = MessageIterator('tag', 'qn', 1, deadline=2)
with patch.object(message_iterator, 'queue') as queue:
queue.lease_tasks_by_tag.return_value = [task]
iterator = iter(message_iterator)
iterator.next()
self.assertRaises(StopIteration, iterator.next)
queue.lease_tasks_by_tag.assert_called_once_with(60,
1,
tag='tag',
deadline=2)
def process_messages(tag, retries=0):
"""Processes the messages pulled fromm a queue based off the tag passed in.
Will insert another processor if any work was processed or the retry count
is under the max retry count. Will update a aggregated stats object with
the data in the payload of the messages processed.
:param tag: :class: `str` Tag to query the queue on
:param retry: :class: `int` Number of retries the job has processed
"""
from furious.batcher import bump_batch
from furious.batcher import MESSAGE_DEFAULT_QUEUE
from furious.batcher import MessageIterator
from furious.batcher import MessageProcessor
from google.appengine.api import memcache
# since we don't have a flag for checking complete we'll re-insert a
# processor task with a retry count to catch any work that may still be
# filtering in. If we've hit our max retry count we just bail out and
# consider the job complete.
if retries > 5:
logging.info("Process messages hit max retry and is exiting")
return
# create a message iteragor for the tag in batches of 500
message_iterator = MessageIterator(tag, MESSAGE_DEFAULT_QUEUE, 500)
client = memcache.Client()
# get the stats object from cache
stats = client.gets(tag)
# json decode it if it exists otherwise get the default state.
stats = json.loads(stats) if stats else get_default_stats()
work_processed = False
# loop through the messages pulled from the queue.
for message in message_iterator:
work_processed = True
value = int(message.get("value", 0))
color = message.get("color").lower()
# update the total stats with the value pulled
set_stats(stats["totals"], value)
# update the specific color status via the value pulled
set_stats(stats["colors"][color], value)
# insert the stats back into cache
json_stats = json.dumps(stats)
# try and do an add first to see if it's new. We can't trush get due to
# a race condition.
if not client.add(tag, json_stats):
# if we couldn't add than lets do a compare and set to safely
# update the stats
if not client.cas(tag, json_stats):
raise Exception("Transaction Collision.")
# bump the process batch id
bump_batch(tag)
if work_processed:
# reset the retries as we've processed work
retries = 0
else:
# no work was processed so increment the retries
retries += 1
# insert another processor
processor = MessageProcessor(target=process_messages,
args=("colors", ),
kwargs={'retries': retries},
tag="colors")
processor.start()