def test_get_default_queue(self):
"""Ensure get_queue returns the default queue if non was given."""
from furious.batcher import Message
message = Message()
self.assertEqual('default_pull', message.get_queue())
def test_get_empty_task_args(self):
"""Ensure get_task_args returns {} if no task_args."""
from furious.batcher import Message
message = Message()
self.assertEqual({}, message.get_task_args())
def test_get_default_queue(self):
"""Ensure get_queue returns the default queue if non was given."""
from furious.batcher import Message
message = Message()
self.assertEqual('default-pull', message.get_queue())
def test_get_empty_task_args(self):
"""Ensure get_task_args returns {} if no task_args."""
from furious.batcher import Message
message = Message()
self.assertEqual({}, message.get_task_args())
def test_insert(self, queue_mock):
"""Ensure the Task is inserted into the specified queue."""
from furious.batcher import Message
message = Message(queue='my_queue')
message.insert()
queue_mock.assert_called_once_with(name='my_queue')
self.assertTrue(queue_mock.return_value.add.called)
def test_insert(self, queue_mock):
"""Ensure the Task is inserted into the specified queue."""
from furious.batcher import Message
message = Message(queue='my_queue')
message.insert()
queue_mock.assert_called_once_with(name='my_queue')
self.assertTrue(queue_mock.return_value.add.called)
def test_get_queue(self):
"""Ensure get_queue returns the message queue."""
from furious.batcher import Message
queue = "test"
message = Message(queue=queue)
self.assertEqual(queue, message.get_queue())
def test_get_queue(self):
"""Ensure get_queue returns the message queue."""
from furious.batcher import Message
queue = "test"
message = Message(queue=queue)
self.assertEqual(queue, message.get_queue())
def test_get_task_args(self):
"""Ensure get_task_args returns the message task_args."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message(**options)
self.assertEqual(task_args, message.get_task_args())
def test_update_options(self):
"""Ensure update_options updates the options."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message.update_options(**options.copy())
self.assertEqual(options, message._options)
def test_get_task_args(self):
"""Ensure get_task_args returns the message task_args."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message(**options)
self.assertEqual(task_args, message.get_task_args())
def test_get_options(self):
"""Ensure get_options returns the message options."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message._options = options
self.assertEqual(options, message.get_options())
def test_update_options(self):
"""Ensure update_options updates the options."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message.update_options(**options.copy())
self.assertEqual(options, message._options)
def test_get_options(self):
"""Ensure get_options returns the message options."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message._options = options
self.assertEqual(options, message.get_options())
def test_to_dict(self):
"""Ensure to_dict returns a dictionary representation of the
Message.
"""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message(**options.copy())
self.assertEqual(options, message.to_dict())
def test_to_dict(self):
"""Ensure to_dict returns a dictionary representation of the
Message.
"""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message(**options.copy())
self.assertEqual(options, message.to_dict())
def test_update_options(self, get_uuid):
"""Ensure update_options updates the options."""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message.update_options(**options.copy())
self.assertEqual(options, message._options)
def test_update_options(self, get_uuid):
"""Ensure update_options updates the options."""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message()
message.update_options(**options.copy())
self.assertEqual(options, message._options)
def test_update_options_supersede_init_opts(self):
"""Ensure update_options supersedes the options set in init."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message(**options)
message.update_options(value=23, other='stuff')
options['value'] = 23
options['other'] = 'stuff'
self.assertEqual(options, message._options)
def test_update_options_supersede_init_opts(self):
"""Ensure update_options supersedes the options set in init."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message(**options)
message.update_options(value=23, other='stuff')
options['value'] = 23
options['other'] = 'stuff'
self.assertEqual(options, message._options)
def test_to_dict(self, get_uuid):
"""Ensure to_dict returns a dictionary representation of the
Message.
"""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
task_args = {'other': 'zzz', 'nested': 1}
options = {'id': _id, 'task_args': task_args}
message = Message(**options.copy())
self.assertEqual(options, message.to_dict())
def test_to_dict(self, get_uuid):
"""Ensure to_dict returns a dictionary representation of the
Message.
"""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
task_args = {'other': 'zzz', 'nested': 1}
options = {'id': _id, 'task_args': task_args}
message = Message(**options.copy())
self.assertEqual(options, message.to_dict())
def test_to_task_with_payload(self):
"""Ensure to_task with a payload produces the right task object."""
import datetime
import time
from google.appengine.ext import testbed
from furious.batcher import Message
testbed = testbed.Testbed()
testbed.activate()
# This just drops the microseconds. It is a total mess, but is needed
# to handle all the rounding crap.
eta = datetime.datetime.now() + datetime.timedelta(minutes=43)
eta_posix = time.mktime(eta.timetuple())
task_args = {'eta': eta_posix, 'payload': [1, 2, 3]}
options = {'task_args': task_args}
task = Message.from_dict(options).to_task()
# App Engine sets these headers by default.
full_headers = {
'Host': 'testbed.example.com',
'X-AppEngine-Current-Namespace': ''
}
self.assertEqual(eta_posix, task.eta_posix)
self.assertEqual(full_headers, task.headers)
self.assertEqual(task_args['payload'], json.loads(task.payload))
def test_purge_tasks_with_queue_names_provided(self, ctime):
"""When a list of queue_names is provided, ensure purge_tasks() clears
the tasks and none are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, ['default'])
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_purge_tasks_with_string_passed_to_queue_names(self, ctime):
"""If a single queue_name is passed to purge_tasks() instead of a list,
ensure that the queue specified is still cleared.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service, 'default')
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure two tasks from the default queue were cleared.
self.assertEqual(2, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_add_pull_and_push_queue_tasks(self):
"""Ensure that push and pull tasks can be added with add_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import get_tasks
from furious.test_stubs.appengine.queues import purge_tasks
# Add tasks the normal way so we can get them and test readding them
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
# purge current tasks so we can verify how many we will add next.
purge_tasks(self.queue_service)
num_added = add_tasks(self.queue_service, task_dict)
# Purge tasks to check how many tasks are in the queues
num_queued = purge_tasks(self.queue_service)
self.assertEqual(3, num_added)
self.assertEqual(3, num_queued)
def test_purge_tasks_with_tasks(self, ctime):
"""After queues are run, ensure no tasks are left to execute.
Ensure the number of tasks cleared is correct.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import run as run_queues
from furious.test_stubs.appengine.queues import purge_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
Message(queue='default-pull').insert()
num_cleared = purge_tasks(self.taskqueue_service)
# Run the tasks to check if tasks remain
run_queues(self.taskqueue_service)
# Ensure three tasks were cleared, from 'default' and 'default-pull'.
self.assertEqual(3, num_cleared)
# Ensure no tasks were run
self.assertEqual(0, ctime.call_count)
def test_to_task_with_payload(self):
"""Ensure to_task with a payload produces the right task object."""
import datetime
import time
from google.appengine.ext import testbed
from furious.batcher import Message
testbed = testbed.Testbed()
testbed.activate()
# This just drops the microseconds. It is a total mess, but is needed
# to handle all the rounding crap.
eta = datetime.datetime.now() + datetime.timedelta(minutes=43)
eta_posix = time.mktime(eta.timetuple())
task_args = {'eta': eta_posix, 'payload': [1, 2, 3]}
options = {'task_args': task_args}
task = Message.from_dict(options).to_task()
# App Engine sets these headers by default.
full_headers = {
'Host': 'testbed.example.com',
'X-AppEngine-Current-Namespace': ''
}
self.assertEqual(eta_posix, task.eta_posix)
self.assertEqual(full_headers, task.headers)
self.assertEqual(task_args['payload'], json.loads(task.payload))
def test_options_are_set(self):
"""Ensure options passed to init are set on the message."""
from furious.batcher import Message
options = {'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message(value=1, other='zzz', nested={1: 1})
self.assertEqual(options, message._options)
def test_reconstitution(self):
"""Ensure to_dict(job.from_dict()) returns the same thing."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(options, message.to_dict())
def test_reconstitution(self):
"""Ensure to_dict(job.from_dict()) returns the same thing."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(options, message.to_dict())
def test_add_async_and_message_tasks(self, ctime):
"""Ensure taskqueue.Task() instances from furious Asyncs and Messages
can be added.
"""
from google.appengine.api import taskqueue
from furious. async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import run as run_queues
# Create asyncs
async = Async(target='time.ctime')
async2 = Async(target='time.ctime')
# Create a message
options = {'task_args': {'payload': 'abcdefg'}}
message = Message(payload='abc', **options)
message_task = message.to_task()
task_dict = {
'default': [async .to_task(), async2.to_task()],
'default-pull': [message_task]
}
num_added = add_tasks(self.queue_service, task_dict)
# Ensure three tasks were added.
self.assertEqual(3, num_added)
# Run the tasks to make sure they were inserted correctly.
run_queues(self.queue_service)
# Ensure both push queue tasks were executed.
self.assertEqual(2, ctime.call_count)
# Lease the pull queue task and make sure it has the correct payload.
tasks = taskqueue.Queue('default-pull').lease_tasks(3600, 100)
returned_task_message = tasks[0]
# Ensure pull queue task payload is the same as the original.
self.assertEqual(returned_task_message.payload, message_task.payload)
def test_from_dict(self):
"""Ensure from_dict returns the correct Message object."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(task_args, message.get_task_args())
def test_from_dict(self):
"""Ensure from_dict returns the correct Message object."""
from furious.batcher import Message
task_args = {'other': 'zzz', 'nested': 1}
options = {'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(task_args, message.get_task_args())
def test_id_is_passed_in_with_options(self):
"""Ensure id is set with options passed to init are set on the message.
"""
from furious.batcher import Message
_id = "id"
options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message(id=_id, value=1, other='zzz', nested={1: 1})
self.assertEqual(options, message._options)
def test_options_are_set(self, get_uuid):
"""Ensure options passed to init are set on the message."""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
options = {'id': _id, 'value': 1, 'other': 'zzz', 'nested': {1: 1}}
message = Message(value=1, other='zzz', nested={1: 1})
self.assertEqual(options, message._options)
def test_add_async_and_message_tasks(self, ctime):
"""Ensure taskqueue.Task() instances from furious Asyncs and Messages
can be added.
"""
from google.appengine.api import taskqueue
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import add_tasks
from furious.test_stubs.appengine.queues import run as run_queues
# Create asyncs
async = Async(target='time.ctime')
async2 = Async(target='time.ctime')
# Create a message
options = {'task_args': {'payload': 'abcdefg'}}
message = Message(payload='abc', **options)
message_task = message.to_task()
task_dict = {'default': [async.to_task(), async2.to_task()],
'default-pull': [message_task]}
num_added = add_tasks(self.queue_service, task_dict)
# Ensure three tasks were added.
self.assertEqual(3, num_added)
# Run the tasks to make sure they were inserted correctly.
run_queues(self.queue_service)
# Ensure both push queue tasks were executed.
self.assertEqual(2, ctime.call_count)
# Lease the pull queue task and make sure it has the correct payload.
tasks = taskqueue.Queue('default-pull').lease_tasks(3600, 100)
returned_task_message = tasks[0]
# Ensure pull queue task payload is the same as the original.
self.assertEqual(returned_task_message.payload, message_task.payload)
def test_reconstitution(self, get_uuid):
"""Ensure to_dict(job.from_dict()) returns the same thing."""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
task_args = {'other': 'zzz', 'nested': 1}
options = {'id': _id, 'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(options, message.to_dict())
def test_reconstitution(self, get_uuid):
"""Ensure to_dict(job.from_dict()) returns the same thing."""
from furious.batcher import Message
_id = "id"
get_uuid.return_value.hex = _id
task_args = {'other': 'zzz', 'nested': 1}
options = {'id': _id, 'task_args': task_args}
message = Message.from_dict(options)
self.assertEqual(options, message.to_dict())
def get(self):
from furious import context
from furious.batcher import Message
from furious.batcher import MessageProcessor
try:
color, value, count = self.get_params()
except (KeyError, AssertionError):
response = {"success": False, "message": "Invalid parameters."}
self.response.write(json.dumps(response))
return
payload = {
"color": color,
"value": value,
"timestamp": time.mktime(datetime.datetime.utcnow().timetuple())
}
tag = "color"
# create a context to insert multiple Messages
with context.new() as ctx:
# loop through the count adding a task to the context per increment
for _ in xrange(count):
# insert the message with the payload
ctx.add(Message(task_args={"payload": payload, "tag": tag}))
# insert a processor to fetch the messages in batches
# this should always be inserted. the logic will keep it from inserting
# too many processors
processor = MessageProcessor(target=process_messages,
args=(tag, ),
tag=tag,
task_args={"countdown": 0})
processor.start()
response = {
"success": True,
"message": "Task inserted successfully with %s" % (payload, )
}
self.response.write(json.dumps(response))
def test_get_tasks_from_all_queues(self):
"""Ensure all tasks are returned from get_tasks()."""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service)
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(3, num_tasks)
def test_to_task_without_payload(self):
"""Ensure to_task without a payload produces the right task object."""
from google.appengine.ext import testbed
from furious.batcher import Message
testbed = testbed.Testbed()
testbed.activate()
options = {'task_args': {}}
task = Message.from_dict(options).to_task()
# App Engine sets these headers by default.
full_headers = {
'Host': 'testbed.example.com',
'X-AppEngine-Current-Namespace': ''
}
self.assertEqual(full_headers, task.headers)
self.assertIsNone(json.loads(task.payload))
def test_to_task_without_payload(self):
"""Ensure to_task without a payload produces the right task object."""
from google.appengine.ext import testbed
from furious.batcher import Message
testbed = testbed.Testbed()
testbed.activate()
options = {'task_args': {}}
task = Message.from_dict(options).to_task()
# App Engine sets these headers by default.
full_headers = {
'Host': 'testbed.example.com',
'X-AppEngine-Current-Namespace': ''
}
self.assertEqual(full_headers, task.headers)
self.assertIsNone(json.loads(task.payload))
def test_get_tasks_when_queue_names_are_specified(self):
"""Ensure queues' tasks are returned from get_tasks() when a list of
queue_names are passed as an argument.
"""
from furious.async import Async
from furious.batcher import Message
from furious.test_stubs.appengine.queues import get_tasks
# Enqueue a couple of tasks
async = Async(target='time.ctime')
async.start()
async2 = Async(target='time.ctime')
async2.start()
# Insert a pull task
Message(queue='default-pull').insert()
task_dict = get_tasks(self.queue_service, ['default'])
num_tasks = sum([len(task_list) for task_list in task_dict.values()])
self.assertEqual(2, num_tasks)