def test_to_task_has_correct_arguments(self, memcache, time, task,
task_retry):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious.async import MAX_RESTARTS
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
task_retry_object = Mock()
task_retry.return_value = task_retry_object
processor = MessageProcessor('something', queue='test_queue',
id='someid', parent_id='parentid',
context_id="contextid")
processor.to_task()
task_args = {
'name': 'processor-processor-current-batch-3',
'url': '/_ah/queue/async/something',
'countdown': 30,
'headers': {},
'retry_options': task_retry_object,
'payload': json.dumps(processor.to_dict())
}
task.assert_called_once_with(**task_args)
task_retry.assert_called_once_with(task_retry_limit=MAX_RESTARTS)
def test_to_task_has_correct_arguments(self, memcache, time, task,
task_retry):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious. async import MAX_RESTARTS
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
task_retry_object = Mock()
task_retry.return_value = task_retry_object
processor = MessageProcessor('something',
queue='test_queue',
id='someid',
parent_id='parentid',
context_id="contextid")
processor.to_task()
task_args = {
'name': 'processor-processor-current-batch-3',
'url': '/_ah/queue/async/something',
'countdown': 30,
'headers': {},
'retry_options': task_retry_object,
'payload': json.dumps(processor.to_dict())
}
task.assert_called_once_with(**task_args)
task_retry.assert_called_once_with(task_retry_limit=MAX_RESTARTS)
def test_to_task_has_correct_arguments(self, memcache, time, task):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue')
processor.to_task()
task_args = {
'url': '/_ah/queue/async/something',
'headers': {},
'payload': json.dumps({
'queue': 'test_queue',
'job': ["something", None, None],
'task_args': {
'countdown': 30,
'name': 'processor-processor-current-batch-3'
},
}),
'countdown': 30,
'name': 'processor-processor-current-batch-3'
}
task.assert_called_once_with(**task_args)
def test_to_task_with_frequency_passed_in(self, memcache, time):
"""Ensure that if a frequency is passed into the MessageProcessor that
it uses that frequency when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue', freq=100)
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-1')
def test_has_type(self):
"""Ensure that if _type is not furious.async.Async that the correct
subclass is instantiated.
"""
from furious.async import async_from_options
from furious.batcher import MessageProcessor
async_job = MessageProcessor(dir)
options = async_job.to_dict()
result = async_from_options(options)
self.assertIsInstance(result, MessageProcessor)
def test_to_task_with_no_name_passed_in(self, memcache, time):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue')
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-3')
def test_has_type(self):
"""Ensure that if _type is not furious.async.Async that the correct
subclass is instantiated.
"""
from furious. async import async_from_options
from furious.batcher import MessageProcessor
async_job = MessageProcessor(dir)
options = async_job.to_dict()
result = async_from_options(options)
self.assertIsInstance(result, MessageProcessor)
def test_to_task_with_frequency_passed_in(self, memcache, time):
"""Ensure that if a frequency is passed into the MessageProcessor that
it uses that frequency when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue', freq=100)
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-1')
def test_to_task_with_no_name_passed_in(self, memcache, time):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue')
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-3')
def test_to_task_has_correct_arguments(self, memcache, time, task,
task_retry):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious. async import MAX_RESTARTS
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
task_retry_object = Mock()
task_retry.return_value = task_retry_object
processor = MessageProcessor('something',
queue='test_queue',
id='someid')
processor.to_task()
task_args = {
'url':
'/_ah/queue/async/something',
'headers': {},
'payload':
json.dumps({
'queue': 'test_queue',
'job': ("something", None, None),
'id': 'someid',
'task_args': {
'countdown': 30,
'name': 'processor-processor-current-batch-3'
},
'_recursion': {
'current': 1,
'max': 100
},
'_type': 'furious.batcher.MessageProcessor',
}),
'countdown':
30,
'name':
'processor-processor-current-batch-3',
'retry_options':
task_retry_object
}
task.assert_called_once_with(**task_args)
task_retry.assert_called_once_with(task_retry_limit=MAX_RESTARTS)
def test_to_task_with_name_passed_in(self, memcache, time):
"""Ensure that if a name is passed into the MessageProcessor that it
uses that name when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue',
task_args={'name': 'test-name'})
task = processor.to_task()
self.assertEqual(task.name, 'test-name-processor-current-batch-3')
def test_to_task_with_name_passed_in(self, memcache, time):
"""Ensure that if a name is passed into the MessageProcessor that it
uses that name when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something',
queue='test_queue',
task_args={'name': 'test-name'})
task = processor.to_task()
self.assertEqual(task.name, 'test-name-processor-current-batch-3')
def test_to_task_with_tag_not_passed_in(self, memcache, time):
"""Ensure that if a tag is not passed into the MessageProcessor that it
uses a default value when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue')
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-3')
memcache.get.assert_called_once_with('agg-batch-processor')
def test_to_task_with_tag_not_passed_in(self, memcache, time):
"""Ensure that if a tag is not passed into the MessageProcessor that it
uses a default value when creating the task.
"""
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
processor = MessageProcessor('something', queue='test_queue')
task = processor.to_task()
self.assertEqual(task.name, 'processor-processor-current-batch-3')
memcache.get.assert_called_once_with('agg-batch-processor')
def get(self):
from furious import context
from furious.batcher import Message
from furious.batcher import MessageProcessor
try:
color, value, count = self.get_params()
except (KeyError, AssertionError):
response = {
"success": False,
"message": "Invalid parameters."
}
self.response.write(json.dumps(response))
return
payload = {
"color": color,
"value": value,
"timestamp": time.mktime(datetime.datetime.utcnow().timetuple())
}
tag = "color"
# create a context to insert multiple Messages
with context.new() as ctx:
# loop through the count adding a task to the context per increment
for _ in xrange(count):
# insert the message with the payload
ctx.add(Message(task_args={"payload": payload, "tag": tag}))
# insert a processor to fetch the messages in batches
# this should always be inserted. the logic will keep it from inserting
# too many processors
processor = MessageProcessor(
target=process_messages, args=(tag,), tag=tag,
task_args={"countdown": 0})
processor.start()
response = {
"success": True,
"message": "Task inserted successfully with %s" % (payload,)
}
self.response.write(json.dumps(response))
def test_to_task_has_correct_arguments(self, memcache, time, task,
task_retry):
"""Ensure that if no name is passed into the MessageProcessor that it
creates a default unique name when creating the task.
"""
from furious.async import MAX_RESTARTS
from furious.batcher import MessageProcessor
memcache.get.return_value = 'current-batch'
time.time.return_value = 100
task_retry_object = Mock()
task_retry.return_value = task_retry_object
processor = MessageProcessor('something', queue='test_queue',
id='someid')
processor.to_task()
task_args = {
'url': '/_ah/queue/async/something',
'headers': {},
'payload': json.dumps({
'queue': 'test_queue',
'job': ("something", None, None),
'id': 'someid',
'task_args': {
'countdown': 30,
'name': 'processor-processor-current-batch-3'
},
'_recursion': {
'current': 1,
'max': 100
},
'_type': 'furious.batcher.MessageProcessor',
}),
'countdown': 30,
'name': 'processor-processor-current-batch-3',
'retry_options': task_retry_object
}
task.assert_called_once_with(**task_args)
task_retry.assert_called_once_with(task_retry_limit=MAX_RESTARTS)
def get(self):
from furious import context
from furious.batcher import Message
from furious.batcher import MessageProcessor
try:
color, value, count = self.get_params()
except (KeyError, AssertionError):
response = {"success": False, "message": "Invalid parameters."}
self.response.write(json.dumps(response))
return
payload = {
"color": color,
"value": value,
"timestamp": time.mktime(datetime.datetime.utcnow().timetuple())
}
tag = "color"
# create a context to insert multiple Messages
with context.new() as ctx:
# loop through the count adding a task to the context per increment
for _ in xrange(count):
# insert the message with the payload
ctx.add(Message(task_args={"payload": payload, "tag": tag}))
# insert a processor to fetch the messages in batches
# this should always be inserted. the logic will keep it from inserting
# too many processors
processor = MessageProcessor(target=process_messages,
args=(tag, ),
tag=tag,
task_args={"countdown": 0})
processor.start()
response = {
"success": True,
"message": "Task inserted successfully with %s" % (payload, )
}
self.response.write(json.dumps(response))
def test_curent_batch_key_doesnt_exist_in_cache(self, cache):
"""Ensure that if the current batch key doesn't exist in cache that it
inserts the default value of 1 into cache and returns it.
"""
from furious.batcher import MessageProcessor
cache.get.return_value = None
processor = MessageProcessor('something')
current_batch = processor.current_batch
self.assertEqual(current_batch, 1)
cache.get.assert_called_once_with('agg-batch-processor')
cache.add.assert_called_once_with('agg-batch-processor', 1)
def test_curent_batch_key_exists_in_cache(self, cache):
"""Ensure that if the current batch key exists in cache that it uses it
and doesn't update it.
"""
from furious.batcher import MessageProcessor
cache.get.return_value = 1
processor = MessageProcessor('something')
current_batch = processor.current_batch
cache.get.assert_called_once_with('agg-batch-processor')
self.assertEqual(current_batch, 1)
self.assertFalse(cache.add.called)
def process_messages(tag, retries=0):
"""Processes the messages pulled fromm a queue based off the tag passed in.
Will insert another processor if any work was processed or the retry count
is under the max retry count. Will update a aggregated stats object with
the data in the payload of the messages processed.
:param tag: :class: `str` Tag to query the queue on
:param retry: :class: `int` Number of retries the job has processed
"""
from furious.batcher import bump_batch
from furious.batcher import MESSAGE_DEFAULT_QUEUE
from furious.batcher import MessageIterator
from furious.batcher import MessageProcessor
from google.appengine.api import memcache
# since we don't have a flag for checking complete we'll re-insert a
# processor task with a retry count to catch any work that may still be
# filtering in. If we've hit our max retry count we just bail out and
# consider the job complete.
if retries > 5:
logging.info("Process messages hit max retry and is exiting")
return
# create a message iteragor for the tag in batches of 500
message_iterator = MessageIterator(tag, MESSAGE_DEFAULT_QUEUE, 500)
client = memcache.Client()
# get the stats object from cache
stats = client.gets(tag)
# json decode it if it exists otherwise get the default state.
stats = json.loads(stats) if stats else get_default_stats()
work_processed = False
# loop through the messages pulled from the queue.
for message in message_iterator:
work_processed = True
value = int(message.get("value", 0))
color = message.get("color").lower()
# update the total stats with the value pulled
set_stats(stats["totals"], value)
# update the specific color status via the value pulled
set_stats(stats["colors"][color], value)
# insert the stats back into cache
json_stats = json.dumps(stats)
# try and do an add first to see if it's new. We can't trush get due to
# a race condition.
if not client.add(tag, json_stats):
# if we couldn't add than lets do a compare and set to safely
# update the stats
if not client.cas(tag, json_stats):
raise Exception("Transaction Collision.")
# bump the process batch id
bump_batch(tag)
if work_processed:
# reset the retries as we've processed work
retries = 0
else:
# no work was processed so increment the retries
retries += 1
# insert another processor
processor = MessageProcessor(
target=process_messages, args=("colors",),
kwargs={'retries': retries}, tag="colors")
processor.start()
def process_messages(tag, retries=0):
"""Processes the messages pulled fromm a queue based off the tag passed in.
Will insert another processor if any work was processed or the retry count
is under the max retry count. Will update a aggregated stats object with
the data in the payload of the messages processed.
:param tag: :class: `str` Tag to query the queue on
:param retry: :class: `int` Number of retries the job has processed
"""
from furious.batcher import bump_batch
from furious.batcher import MESSAGE_DEFAULT_QUEUE
from furious.batcher import MessageIterator
from furious.batcher import MessageProcessor
from google.appengine.api import memcache
# since we don't have a flag for checking complete we'll re-insert a
# processor task with a retry count to catch any work that may still be
# filtering in. If we've hit our max retry count we just bail out and
# consider the job complete.
if retries > 5:
logging.info("Process messages hit max retry and is exiting")
return
# create a message iteragor for the tag in batches of 500
message_iterator = MessageIterator(tag, MESSAGE_DEFAULT_QUEUE, 500)
client = memcache.Client()
# get the stats object from cache
stats = client.gets(tag)
# json decode it if it exists otherwise get the default state.
stats = json.loads(stats) if stats else get_default_stats()
work_processed = False
# loop through the messages pulled from the queue.
for message in message_iterator:
work_processed = True
value = int(message.get("value", 0))
color = message.get("color").lower()
# update the total stats with the value pulled
set_stats(stats["totals"], value)
# update the specific color status via the value pulled
set_stats(stats["colors"][color], value)
# insert the stats back into cache
json_stats = json.dumps(stats)
# try and do an add first to see if it's new. We can't trush get due to
# a race condition.
if not client.add(tag, json_stats):
# if we couldn't add than lets do a compare and set to safely
# update the stats
if not client.cas(tag, json_stats):
raise Exception("Transaction Collision.")
# bump the process batch id
bump_batch(tag)
if work_processed:
# reset the retries as we've processed work
retries = 0
else:
# no work was processed so increment the retries
retries += 1
# insert another processor
processor = MessageProcessor(target=process_messages,
args=("colors", ),
kwargs={'retries': retries},
tag="colors")
processor.start()