def test_calling_delete_on_a_message_returned_removes_it_from_mongodb( self): collection = Connection().karait_test.queue_test queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1})) self.assertEqual(1, collection.find({}).count()) queue.read()[0].delete() self.assertEqual(0, len(queue.read()))
def test_delete_messages_removes_whole_set_of_messages(self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read() self.assertEqual(3, len(messages)) queue.delete_messages(messages) messages = queue.read() self.assertEqual(0, len(messages))
def test_calling_delete_on_a_message_returned_removes_it_from_mongodb(self): collection = Connection().karait_test.queue_test queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1})) self.assertEqual(1, collection.find({}).count()) queue.read()[0].delete() self.assertEqual(0, len(queue.read()))
def test_setting_visibility_timeout_makes_a_message_unavailable_for_a_set_number_of_milliseconds( self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read(visibility_timeout=0.4) self.assertEqual(3, len(messages)) messages = queue.read() self.assertEqual(0, len(messages)) time.sleep(0.5) messages = queue.read() self.assertEqual(3, len(messages))
def test_delete_messages_removes_whole_set_of_messages(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read() self.assertEqual(3, len(messages)) queue.delete_messages(messages) messages = queue.read() self.assertEqual(0, len(messages))
def test_setting_visibility_timeout_makes_a_message_unavailable_for_a_set_number_of_milliseconds(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read(visibility_timeout=0.4) self.assertEqual(3, len(messages)) messages = queue.read() self.assertEqual(0, len(messages)) time.sleep(0.5) messages = queue.read() self.assertEqual(3, len(messages))
def test_a_message_with_a_unique_key_specified_can_only_be_inserted_once_for_a_given_key_value( self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 'value1'}), unique_key='foo') queue.write(Message({'foo': 'value1'}), unique_key='foo') queue.write(Message({'foo': 'value2'}), unique_key='foo') messages = queue.read() self.assertEqual(len(messages), 2)
def test_read_with_no_routing_key_only_returns_messages_with_no_routing_key( self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1}), routing_key='foobar') queue.write(Message({'foo': 2})) messages = queue.read() self.assertEqual(1, len(messages)) self.assertEqual(2, messages[0].foo)
def test_routing_key_can_optionally_be_used_to_return_only_select_messages( self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1}), routing_key='foobar') queue.write(Message({'foo': 2})) messages = queue.read(routing_key='foobar') self.assertEqual(1, len(messages)) self.assertEqual(1, messages[0].foo)
def test_message_with_expire_set_does_not_immediately_expire(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1}), expire=0.1) self.assertEqual(1, len(queue.read())) self.assertEqual(1, len(queue.read()))
def test_messages_returned_in_fifo_order(self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read() self.assertEqual(1, messages[0].foo) self.assertEqual(2, messages[1].foo) self.assertEqual(3, messages[2].foo)
def test_read_with_no_routing_key_only_returns_messages_with_no_routing_key(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1}), routing_key='foobar') queue.write(Message({'foo': 2})) messages = queue.read() self.assertEqual(1, len(messages)) self.assertEqual(2, messages[0].foo)
def test_routing_key_can_optionally_be_used_to_return_only_select_messages(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1}), routing_key='foobar') queue.write(Message({'foo': 2})) messages = queue.read(routing_key='foobar') self.assertEqual(1, len(messages)) self.assertEqual(1, messages[0].foo)
def test_a_message_with_a_unique_key_specified_can_only_be_inserted_once_for_a_given_key_value(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 'value1'}), unique_key='foo') queue.write(Message({'foo': 'value1'}), unique_key='foo') queue.write(Message({'foo': 'value2'}), unique_key='foo') messages = queue.read() self.assertEqual(len(messages), 2)
def test_expired_messages_not_returned_by_read(self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1}), expire=0.1) time.sleep(0.2) self.assertEqual(0, len(queue.read())) self.assertEqual(0, len(queue.read())) # Make sure the meta._expired key is actually set. collection = Connection().karait_test.queue_test raw_message = collection.find_one({}) self.assertEqual(True, raw_message['_meta']['expired'])
def test_messages_returned_in_fifo_order(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1})) queue.write(Message({'foo': 2})) queue.write(Message({'foo': 3})) messages = queue.read() self.assertEqual(1, messages[0].foo) self.assertEqual(2, messages[1].foo) self.assertEqual(3, messages[2].foo)
def test_reading_from_the_queue_returns_a_message_object(self): queue = Queue(database='karait_test', queue='queue_test') write_message = Message() write_message.apple = 5 write_message.banana = 6 write_message.inner_object = {'foo': 1, 'bar': 2} queue.write(write_message) read_message = queue.read()[0] self.assertEqual(5, read_message.apple) self.assertEqual(2, read_message.inner_object['bar']) self.assertEqual(3, len(read_message.to_dictionary().keys()))
def test_expired_messages_not_returned_by_read(self): queue = Queue( database='karait_test', queue='queue_test' ) queue.write(Message({'foo': 1}), expire=0.1) time.sleep(0.2) self.assertEqual(0, len(queue.read())) self.assertEqual(0, len(queue.read())) # Make sure the meta._expired key is actually set. collection = Connection().karait_test.queue_test raw_message = collection.find_one({}) self.assertEqual(True, raw_message['_meta']['expired'])
def test_reading_from_the_queue_returns_a_message_object(self): queue = Queue( database='karait_test', queue='queue_test' ) write_message = Message() write_message.apple = 5 write_message.banana = 6 write_message.inner_object = { 'foo': 1, 'bar': 2 } queue.write(write_message) read_message = queue.read()[0] self.assertEqual(5, read_message.apple) self.assertEqual(2, read_message.inner_object['bar']) self.assertEqual(3, len(read_message.to_dictionary().keys()))
import time import json from karait import Queue print("Starting python reader.") messages_read = 0.0 start_time = time.time() queue = Queue() while True: messages = queue.read(routing_key='for_reader') for message in messages: messages_read += 1.0 message.messages_read = messages_read message.messages_read_per_second = messages_read / (time.time() - start_time) if (messages_read % 250) == 0.0: print("Message Read: \n%s" % json.dumps(message.to_dictionary())) queue.delete_messages(messages)
def test_message_with_expire_set_does_not_immediately_expire(self): queue = Queue(database='karait_test', queue='queue_test') queue.write(Message({'foo': 1}), expire=0.1) self.assertEqual(1, len(queue.read())) self.assertEqual(1, len(queue.read()))
import logging import logging.config import pymongo from pymongo import Connection if pymongo.version.startswith("2.5"): import bson.code pymongo.code = bson.code sys.modules['pymongo.code'] = bson.code from karait import Message, Queue #queue = Queue() queue = Queue( host='localhost', # MongoDB host. Defaults to localhost. port=2006, # MongoDB port. Defaults to 27017. database='karait', # Database that will store the karait queue. Defaults to karait. queue='messages', # The capped collection that karait writes to. Defaults to messages. average_message_size=8192, # How big do you expect the messages will be in bytes? Defaults to 8192. queue_size=4096 # How many messages should be allowed in the queue. Defaults to 4096. ) message = queue.read()[0] print "%s" % (message.name) message.delete() # or message = queue.read(routing_key='my_routing_key')[0] print "%s" % (message.action) message.delete()
from karait import Message, Queue print('Starting Python process.') queue = Queue() while True: messages = queue.read() if messages and messages[0].action == 'shutdown_broadcast': print('Terminating due to broadcast shutdown message.') break
import time import json from karait import Queue print("Starting python reader.") messages_read = 0.0 start_time = time.time() queue = Queue() while True: messages = queue.read(routing_key='for_reader') for message in messages: messages_read += 1.0 message.messages_read = messages_read message.messages_read_per_second = messages_read / (time.time() - start_time) if (messages_read % 250) == 0.0: print("Message Read: \n%s" % json.dumps(message.to_dictionary())) queue.delete_messages(messages)