Esempio n. 1
0
 def test_message_with_expire_set_does_not_immediately_expire(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}), expire=0.1)
     self.assertEqual(1, len(queue.read()))
     self.assertEqual(1, len(queue.read()))
Esempio n. 2
0
 def test_delete_messages_removes_whole_set_of_messages(self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}))
     queue.write(Message({'foo': 2}))
     queue.write(Message({'foo': 3}))
     messages = queue.read()
     self.assertEqual(3, len(messages))
     queue.delete_messages(messages)
     messages = queue.read()
     self.assertEqual(0, len(messages))
Esempio n. 3
0
    def test_expired_messages_not_returned_by_read(self):
        queue = Queue(database='karait_test', queue='queue_test')
        queue.write(Message({'foo': 1}), expire=0.1)
        time.sleep(0.2)
        self.assertEqual(0, len(queue.read()))
        self.assertEqual(0, len(queue.read()))

        # Make sure the meta._expired key is actually set.
        collection = Connection().karait_test.queue_test
        raw_message = collection.find_one({})
        self.assertEqual(True, raw_message['_meta']['expired'])
Esempio n. 4
0
    def test_reading_from_the_queue_returns_a_message_object(self):
        queue = Queue(database='karait_test', queue='queue_test')

        write_message = Message()
        write_message.apple = 5
        write_message.banana = 6
        write_message.inner_object = {'foo': 1, 'bar': 2}
        queue.write(write_message)

        read_message = queue.read()[0]
        self.assertEqual(5, read_message.apple)
        self.assertEqual(2, read_message.inner_object['bar'])
        self.assertEqual(3, len(read_message.to_dictionary().keys()))
Esempio n. 5
0
 def test_expired_messages_not_returned_by_read(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}), expire=0.1)
     time.sleep(0.2)
     self.assertEqual(0, len(queue.read()))
     self.assertEqual(0, len(queue.read()))
     
     # Make sure the meta._expired key is actually set.
     collection = Connection().karait_test.queue_test
     raw_message = collection.find_one({})
     self.assertEqual(True, raw_message['_meta']['expired'])
Esempio n. 6
0
    def test_writing_a_message_to_the_queue_populates_it_within_mongodb(self):
        queue = Queue(database='karait_test', queue='queue_test')

        message = Message()
        message.apple = 5
        message.banana = 6
        message.inner_object = {'foo': 1, 'bar': 2}
        queue.write(message)

        collection = Connection().karait_test.queue_test
        obj = collection.find_one({})
        self.assertEqual(6, obj['banana'])
        self.assertEqual(2, obj['inner_object']['bar'])
        self.assertTrue(obj['_meta']['expire'])
        self.assertTrue(obj['_meta']['timestamp'])
Esempio n. 7
0
 def test_setting_visibility_timeout_makes_a_message_unavailable_for_a_set_number_of_milliseconds(
         self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}))
     queue.write(Message({'foo': 2}))
     queue.write(Message({'foo': 3}))
     messages = queue.read(visibility_timeout=0.4)
     self.assertEqual(3, len(messages))
     messages = queue.read()
     self.assertEqual(0, len(messages))
     time.sleep(0.5)
     messages = queue.read()
     self.assertEqual(3, len(messages))
Esempio n. 8
0
 def test_delete_messages_removes_whole_set_of_messages(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}))
     queue.write(Message({'foo': 2}))
     queue.write(Message({'foo': 3}))
     messages = queue.read()
     self.assertEqual(3, len(messages))
     queue.delete_messages(messages)
     messages = queue.read()
     self.assertEqual(0, len(messages))
Esempio n. 9
0
    def test_writing_a_dictionary_to_the_queue_populates_it_within_mongodb(
            self):
        queue = Queue(database='karait_test', queue='queue_test')
        queue.write({
            'apple': 5,
            'banana': 6,
            'inner_object': {
                'foo': 1,
                'bar': 2
            }
        })

        collection = Connection().karait_test.queue_test
        obj = collection.find_one({})
        self.assertEqual(6, obj['banana'])
        self.assertEqual(2, obj['inner_object']['bar'])
        self.assertTrue(obj['_meta']['expire'])
        self.assertTrue(obj['_meta']['timestamp'])
Esempio n. 10
0
 def test_calling_delete_on_a_message_returned_removes_it_from_mongodb(
         self):
     collection = Connection().karait_test.queue_test
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}))
     self.assertEqual(1, collection.find({}).count())
     queue.read()[0].delete()
     self.assertEqual(0, len(queue.read()))
Esempio n. 11
0
 def test_a_message_with_a_unique_key_specified_can_only_be_inserted_once_for_a_given_key_value(
         self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 'value1'}), unique_key='foo')
     queue.write(Message({'foo': 'value1'}), unique_key='foo')
     queue.write(Message({'foo': 'value2'}), unique_key='foo')
     messages = queue.read()
     self.assertEqual(len(messages), 2)
Esempio n. 12
0
 def test_routing_key_can_optionally_be_used_to_return_only_select_messages(
         self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}), routing_key='foobar')
     queue.write(Message({'foo': 2}))
     messages = queue.read(routing_key='foobar')
     self.assertEqual(1, len(messages))
     self.assertEqual(1, messages[0].foo)
Esempio n. 13
0
 def test_read_with_no_routing_key_only_returns_messages_with_no_routing_key(
         self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}), routing_key='foobar')
     queue.write(Message({'foo': 2}))
     messages = queue.read()
     self.assertEqual(1, len(messages))
     self.assertEqual(2, messages[0].foo)
Esempio n. 14
0
 def test_reading_from_the_queue_returns_a_message_object(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     
     write_message = Message()
     write_message.apple = 5
     write_message.banana = 6
     write_message.inner_object = {
         'foo': 1,
         'bar': 2
     }
     queue.write(write_message)
     
     read_message = queue.read()[0]
     self.assertEqual(5, read_message.apple)
     self.assertEqual(2, read_message.inner_object['bar'])
     self.assertEqual(3, len(read_message.to_dictionary().keys()))
Esempio n. 15
0
 def test_messages_returned_in_fifo_order(self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}))
     queue.write(Message({'foo': 2}))
     queue.write(Message({'foo': 3}))
     messages = queue.read()
     self.assertEqual(1, messages[0].foo)
     self.assertEqual(2, messages[1].foo)
     self.assertEqual(3, messages[2].foo)
Esempio n. 16
0
 def test_queue_initializes_capped_collection_for_queue_if_collection_does_not_exist(
         self):
     queue = Queue(database='karait_test',
                   queue='queue_test',
                   average_message_size=8192,
                   queue_size=4096)
     collection = Connection().karait_test.queue_test
     options = collection.options()
     self.assertEqual(1, options['capped'])
     self.assertEqual(4096, options['max'])
     self.assertEqual((8192 * 4096), options['size'])
Esempio n. 17
0
    def test_writing_a_dictionary_to_the_queue_populates_it_within_mongodb(self):
        queue = Queue(
            database='karait_test',
            queue='queue_test'
        )
        queue.write({
            'apple': 5,
            'banana': 6,
            'inner_object': {
                'foo': 1,
                'bar': 2
            }
        })

        collection = Connection().karait_test.queue_test
        obj = collection.find_one({})
        self.assertEqual(6, obj['banana'])
        self.assertEqual(2, obj['inner_object']['bar'])
        self.assertTrue(obj['_meta']['expire'])
        self.assertTrue(obj['_meta']['timestamp'])
Esempio n. 18
0
    def test_writing_a_message_to_the_queue_populates_it_within_mongodb(self):
        queue = Queue(
            database='karait_test',
            queue='queue_test'
        )
        
        message = Message()
        message.apple = 5
        message.banana = 6
        message.inner_object = {
            'foo': 1,
            'bar': 2
        }
        queue.write(message)

        collection = Connection().karait_test.queue_test
        obj = collection.find_one({})
        self.assertEqual(6, obj['banana'])
        self.assertEqual(2, obj['inner_object']['bar'])
        self.assertTrue(obj['_meta']['expire'])
        self.assertTrue(obj['_meta']['timestamp'])
Esempio n. 19
0
 def test_queue_object_can_attach_to_a_collection_that_already_exists(self):
     collection = Connection().karait_test.queue_test
     collection.insert({
         'routing_key': 'foobar',
         'message': {
             'apple': 3,
             'banana': 5
         },
         'timestamp': 2523939,
         'expire': 20393
     })
     queue = Queue(database='karait_test', queue='queue_test')
     self.assertEqual(1, collection.find({}).count())
Esempio n. 20
0
 def test_setting_visibility_timeout_makes_a_message_unavailable_for_a_set_number_of_milliseconds(self):
    queue = Queue(
        database='karait_test',
        queue='queue_test'
    )
    queue.write(Message({'foo': 1}))
    queue.write(Message({'foo': 2}))
    queue.write(Message({'foo': 3}))
    messages = queue.read(visibility_timeout=0.4)
    self.assertEqual(3, len(messages))
    messages = queue.read()
    self.assertEqual(0, len(messages))
    time.sleep(0.5)
    messages = queue.read()
    self.assertEqual(3, len(messages))
Esempio n. 21
0
 def test_a_message_with_a_unique_key_specified_can_only_be_inserted_once_for_a_given_key_value(self):
    queue = Queue(
        database='karait_test',
        queue='queue_test'
    )
    queue.write(Message({'foo': 'value1'}), unique_key='foo')
    queue.write(Message({'foo': 'value1'}), unique_key='foo')
    queue.write(Message({'foo': 'value2'}), unique_key='foo')
    messages = queue.read()
    self.assertEqual(len(messages), 2)
Esempio n. 22
0
 def test_routing_key_can_optionally_be_used_to_return_only_select_messages(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}), routing_key='foobar')
     queue.write(Message({'foo': 2}))
     messages = queue.read(routing_key='foobar')
     self.assertEqual(1, len(messages))
     self.assertEqual(1, messages[0].foo)
Esempio n. 23
0
 def test_calling_delete_on_a_message_returned_removes_it_from_mongodb(self):
     collection = Connection().karait_test.queue_test
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}))
     self.assertEqual(1, collection.find({}).count())
     queue.read()[0].delete()
     self.assertEqual(0, len(queue.read()))
Esempio n. 24
0
 def test_read_with_no_routing_key_only_returns_messages_with_no_routing_key(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}), routing_key='foobar')
     queue.write(Message({'foo': 2}))
     messages = queue.read()
     self.assertEqual(1, len(messages))
     self.assertEqual(2, messages[0].foo)
Esempio n. 25
0
 def test_messages_returned_in_fifo_order(self):
     queue = Queue(
         database='karait_test',
         queue='queue_test'
     )
     queue.write(Message({'foo': 1}))
     queue.write(Message({'foo': 2}))
     queue.write(Message({'foo': 3}))
     messages = queue.read()
     self.assertEqual(1, messages[0].foo)
     self.assertEqual(2, messages[1].foo)
     self.assertEqual(3, messages[2].foo)
Esempio n. 26
0
from karait import Message, Queue

print('Starting Python process.')
queue = Queue()

while True:
  messages = queue.read()
  
  if messages and messages[0].action == 'shutdown_broadcast':
    print('Terminating due to broadcast shutdown message.')
    break
Esempio n. 27
0
 def test_message_with_expire_set_does_not_immediately_expire(self):
     queue = Queue(database='karait_test', queue='queue_test')
     queue.write(Message({'foo': 1}), expire=0.1)
     self.assertEqual(1, len(queue.read()))
     self.assertEqual(1, len(queue.read()))
Esempio n. 28
0
import time
from karait import Message, Queue

print 'Starting python writer.'

messages_written = 0
start = time.time()
queue = Queue()

while True:
    queue.write({
        'messages_written': messages_written,
        'sender': 'writer.py',
        'started_running': start,
        'messages_written_per_second': (messages_written / (time.time() - start))
    }, routing_key='for_reader')
    messages_written += 1
Esempio n. 29
0
from karait import Message, Queue

print 'Terminating all processes.'

queue = Queue()
queue.write(
    {
        'action': 'shutdown_broadcast'
    },
    expire=10
)
Esempio n. 30
0
import time
import json
from karait import Queue
print("Starting python reader.")

messages_read = 0.0
start_time = time.time()

queue = Queue()
while True:
  messages = queue.read(routing_key='for_reader')
  for message in messages:
    messages_read += 1.0
        
    message.messages_read = messages_read
    message.messages_read_per_second = messages_read / (time.time() - start_time)
    
    if (messages_read % 250) == 0.0:
      print("Message Read: \n%s" % json.dumps(message.to_dictionary()))

  queue.delete_messages(messages)
Esempio n. 31
0
from karait import Message, Queue

print "Terminating Ruby process."

queue = Queue()
queue.write({"action": "shutdown_routed"}, routing_key="ruby_process")
Esempio n. 32
0
from karait import Message, Queue

print 'Terminating Ruby process.'

queue = Queue()
queue.write(
    {
        'action': 'shutdown_routed'
    },
    routing_key='ruby_process'
)
Esempio n. 33
0
import sys 
import logging
import logging.config
import pymongo
from pymongo import Connection
if pymongo.version.startswith("2.5"):
    import bson.code
    pymongo.code = bson.code
    sys.modules['pymongo.code'] = bson.code
from karait import Message, Queue

#queue = Queue()
queue = Queue(
        host='localhost', # MongoDB host. Defaults to localhost.
        port=2006, # MongoDB port. Defaults to 27017.
        database='karait', # Database that will store the karait queue. Defaults to karait.
        queue='messages', # The capped collection that karait writes to. Defaults to messages.
        average_message_size=8192, # How big do you expect the messages will be in bytes? Defaults to 8192.
        queue_size=4096 # How many messages should be allowed in the queue. Defaults to 4096.
        )

message = queue.read()[0]
print "%s" % (message.name)

message.delete()

# or
message = queue.read(routing_key='my_routing_key')[0]
print "%s" % (message.action)

message.delete()
Esempio n. 34
0
import os
import sys 
import logging
import logging.config
import pymongo
from pymongo import Connection
if pymongo.version.startswith("2.5"):
    import bson.code
    pymongo.code = bson.code
    sys.modules['pymongo.code'] = bson.code
from karait import Message, Queue

queue = Queue(
        host='localhost', # MongoDB host. Defaults to localhost.
        port=2006, # MongoDB port. Defaults to 27017.
        database='karait', # Database that will store the karait queue. Defaults to karait.
        queue='messages', # The capped collection that karait writes to. Defaults to messages.
        average_message_size=8192, # How big do you expect the messages will be in bytes? Defaults to 8192.
        queue_size=4096 # How many messages should be allowed in the queue. Defaults to 4096.
        )

queue.write({
        'name': 'Benjamin',
        'action': 'Rock'
        })

# or

message = Message()
message.name = 'Benjamin'
message.action = 'Rock!'
Esempio n. 35
0
import time
import json
from karait import Queue
print("Starting python reader.")

messages_read = 0.0
start_time = time.time()

queue = Queue()
while True:
    messages = queue.read(routing_key='for_reader')
    for message in messages:
        messages_read += 1.0

        message.messages_read = messages_read
        message.messages_read_per_second = messages_read / (time.time() -
                                                            start_time)

        if (messages_read % 250) == 0.0:
            print("Message Read: \n%s" % json.dumps(message.to_dictionary()))

    queue.delete_messages(messages)