def publish(message): """ Publish message to the indexing queue """ EXCHANGE = "crawler.vehicles" QUEUE = "indexer_queue" q = MessageQueue(EXCHANGE, QUEUE) ROUTING_KEY = "indexer.index" q.publish(ROUTING_KEY, message)
def setup_class(self): self.client = Redis() self.mq = MessageQueue(self.client, "email_queue") self.mq.add_message("*****@*****.**") self.mq.add_message("*****@*****.**") self.mq.add_message("*****@*****.**")
def setUp(self): self.client = Redis(decode_responses=True) self.client.flushdb() self.mq = MessageQueue(self.client, "mq") self.msg = "hello world" self.blocktime = 1
def setUp(self): self.client = Redis(decode_responses=True) self.stream_key = "test_message_queue" self.q = MessageQueue(self.client, self.stream_key) self.message = {"name":"peter", "age":"36", "location":"UK"} self.tearDown()
def setUp(self): self.client = Redis(decode_responses=True) self.stream = "test_stream" self.group = "test_group" # objs self.q = MessageQueue(self.client, self.stream) self.g = Group(self.client, self.stream, self.group) # create a empty stream key self.msg_id = self.q.add_message({"k":"v"}) self.q.remove_message(self.msg_id)
def should_handle_chunks_of_data_stream_sliced_randomly(): stream = [bytes('7|U|46|17\n', 'UTF-8'), b'6|U|46|68\n3|P|46|68\n4|F', b'|4|68\n5|P|', b'5|68\n1', b'|F|46|49\n2|U|46|68\n'] queue = MessageQueue() for chunk in stream: queue.push_data_stream(chunk) prev_seq = 0 for expected in [b'1|F|46|49', b'2|U|46|68', b'3|P|46|68', b'4|F|4|68', b'5|P|5|68', b'6|U|46|68', b'7|U|46|17']: seq, data = queue.pop() assert expected == data assert seq >= prev_seq assert not queue.queue
def should_handle_chunks_of_data_stream_sliced_randomly(): stream = [ bytes('7|U|46|17\n', 'UTF-8'), b'6|U|46|68\n3|P|46|68\n4|F', b'|4|68\n5|P|', b'5|68\n1', b'|F|46|49\n2|U|46|68\n' ] queue = MessageQueue() for chunk in stream: queue.push_data_stream(chunk) prev_seq = 0 for expected in [ b'1|F|46|49', b'2|U|46|68', b'3|P|46|68', b'4|F|4|68', b'5|P|5|68', b'6|U|46|68', b'7|U|46|17' ]: seq, data = queue.pop() assert expected == data assert seq >= prev_seq assert not queue.queue
class TestMessageQueue: def setup_class(self): self.client = Redis() self.mq = MessageQueue(self.client, "email_queue") self.mq.add_message("*****@*****.**") self.mq.add_message("*****@*****.**") self.mq.add_message("*****@*****.**") def test_len(self): assert self.mq.len() == 3 def test_consume(self): assert self.mq.get_message() == "*****@*****.**" assert self.mq.get_message() == "*****@*****.**" assert self.mq.get_message() == "*****@*****.**"
#!/usr/bin/python import socket from threading import Thread from event_handler import EventHandler from message_queue import MessageQueue from user_network import UserNetwork users = {} queue = MessageQueue() event_handler = EventHandler(users, UserNetwork()) def bind_server(port, queue_len=1): serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) serversocket.bind(('localhost', port)) serversocket.listen(queue_len) return serversocket def event_listener(port=9090): serversocket = bind_server(port) data = None clientsocket = None while True: try: if not data: if clientsocket: print( "Event stream has stopped. Press Ctrl-C to stop the server" ) clientsocket.close()
class TestMessageQueue(unittest.TestCase): def setUp(self): self.client = Redis(decode_responses=True) self.stream_key = "test_message_queue" self.q = MessageQueue(self.client, self.stream_key) self.message = {"name":"peter", "age":"36", "location":"UK"} self.tearDown() def tearDown(self): self.client.delete(self.stream_key) def make_a_none_empty_queue(self, queue, size): for i in range(size): key = "key{0}".format(i) value = "value{0}".format(i) self.q.add_message({key:value,}) def test_add_message(self): message_id = self.q.add_message(self.message) self.assertIsNotNone(message_id) def test_add_message_return_utf8_id(self): message_id = self.q.add_message(self.message) self.assertIsInstance(message_id, str) def test_get_message_return_none_when_msg_not_exists(self): self.assertIsNone(self.q.get_message("10086-0")) def test_get_message_return_msg_when_its_exists(self): message_id = self.q.add_message(self.message) self.assertEqual( self.message, self.q.get_message(message_id) ) def test_remove_message(self): message_id = self.q.add_message(self.message) self.q.remove_message(message_id) self.assertIsNone(self.q.get_message(message_id)) def test_len_return_zero_when_queue_empty(self): self.assertEqual(self.q.len(), 0) def test_len_retrun_non_zero_when_queue_not_empty(self): id = self.q.add_message(self.message) self.assertEqual(self.q.len(), 1) self.q.remove_message(id) self.assertEqual(self.q.len(), 0) def test_get_by_range_return_empty_list_when_queue_empty(self): self.assertEqual(self.q.get_by_range("-", "+"), list()) def test_get_by_range_return_none_empty_list_when_queue_not_empty(self): self.make_a_none_empty_queue(self.q, 10) result = self.q.get_by_range("-", "+") self.assertEqual(len(result), 10) self.assertEqual(type(result), list) for msg in result: self.assertEqual(type(msg), dict) def test_get_by_range_return_right_size(self): self.make_a_none_empty_queue(self.q, 10) result = self.q.get_by_range("-", "+", 5) self.assertEqual(len(result), 5) def test_get_by_range_return_empty_list_when_stream_empty(self): self.assertEqual( self.q.get_by_range("-", "+"), list() ) def test_iterate_return_right_size(self): self.make_a_none_empty_queue(self.q, 10) self.assertEqual( len(self.q.iterate(0, 5)), 5 ) self.assertEqual( len(self.q.iterate(0, 10)), 10 ) def test_iterate_return_right_message(self): self.make_a_none_empty_queue(self.q, 10) full_message_list = self.q.get_by_range("-", "+", 10) self.assertEqual( self.q.iterate(0, 5), full_message_list[0:5] ) self.assertEqual( self.q.iterate(0, 10), full_message_list ) def test_iterate_return_empty_list_when_stream_empty(self): self.assertEqual( self.q.iterate(0), list() )
def should_block_for_next_item(): N = 5 queue = MessageQueue() queue._push(N, 'data5') consumed = [] consumer = Thread(target=consume, args=[queue, consumed]) consumer.start() queue._push(1, "data1") consumer.join() assert ['data1'] == consumed queue._push(4, 'data4') consumer = Thread(target=consume, args=[queue, consumed]) consumer.start() consumer.join(.3) #wait assert consumer.is_alive() #Thread is blocked queue._push(2, 'data2') consumer.join() assert ['data1', 'data2'] == consumed queue._push(3, 'data3') consumer = Thread(target=consume, args=[queue, consumed, 3]) consumer.start() consumer.join() assert consumed == ['data1', 'data2', 'data3', 'data4', 'data5']
class TestMessageQueue(unittest.TestCase): def setUp(self): self.client = Redis(decode_responses=True) self.client.flushdb() self.mq = MessageQueue(self.client, "mq") self.msg = "hello world" self.blocktime = 1 def test_add(self): self.mq.add_message(self.msg) self.assertEqual( self.mq.len(), 1 ) def test_get_message_non_block(self): self.mq.add_message(self.msg) self.assertEqual( self.mq.get_message(), self.msg ) def test_get_message_blocking(self): before = time() print("blocking, wait for {0} second(s)".format(self.blocktime)) self.assertIsNone( self.mq.get_message(self.blocktime) ) self.assertTrue( time() - before >= self.blocktime ) def test_len(self): self.assertEqual( self.mq.len(), 0 ) self.mq.add_message(self.msg) self.assertEqual( self.mq.len(), 1 )
async def main(): logging.info("-- Starting message queue... To exit press CTRL+C") mq = MessageQueue(handler=mq_handler) await mq.connect() await mq.listen()
from message_queue import MessageQueue import downloader as dwn EXCHANGE = "crawler.vehicles" QUEUE = "crawler_queue" queue = MessageQueue(EXCHANGE, QUEUE) ## Starting consumer queue.start_consumer(dwn.process_message) ## Expected message format # { # "vehicle_thumbnails": [ # "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/f.jpg?var=1507107710", # "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/r.jpg?var=1507107710", # "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/2.jpg?var=1507107710", # "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/3.jpg?var=1507107710" # ], # "_values": { # "vehicle_drive": "2WD", # "vehicle_body": "Hatchback", # "vehicle_title": "TOYOTA RAUM 2006/9 CBA-NCZ20", # "vehicle_price_cif": "N", # "vehicle_accessories": "", # "vehicle_engine": "1,490cc", # "internal_id": "SBT-MF3332", # "vehicle_make": "TOYOTA RAUM 2006/9 CBA-NCZ20", # "vehicle_mileage": "82,000", # "vehicle_year": "2006/9", # "vehicle_doors": "5",
class TestGroup(unittest.TestCase): def setUp(self): self.client = Redis(decode_responses=True) self.stream = "test_stream" self.group = "test_group" # objs self.q = MessageQueue(self.client, self.stream) self.g = Group(self.client, self.stream, self.group) # create a empty stream key self.msg_id = self.q.add_message({"k":"v"}) self.q.remove_message(self.msg_id) def tearDown(self): self.client.delete(self.stream) def create_group(self): self.g.create(0) def add_msg_to_stream(self): self.q.add_message({"k":"v"}) def test_create(self): self.create_group() self.assertNotEqual( self.g.info(), [] ) def test_read_message(self): self.create_group() self.assertEqual( self.g.read_message("worker1", ">"), [] ) self.add_msg_to_stream() self.assertNotEqual( self.g.read_message("worker1", ">"), [] ) def test_ack_message(self): self.create_group() self.add_msg_to_stream() msgs = self.g.read_message("worker1", ">") msg_id = list(msgs[0].keys())[0] self.g.ack_message(msg_id) pending_count = self.g.info()['pending'] self.assertEqual( pending_count, 0 ) def test_info(self): self.assertEqual( self.g.info(), dict() ) self.create_group() self.assertIsNotNone( self.g.info() ) def test_consumer_info(self): self.create_group() self.assertEqual( self.g.consumer_info(), [] ) self.add_msg_to_stream() msgs = self.g.read_message("worker1", ">") self.assertNotEqual( self.g.consumer_info(), [] ) def test_delete_consumer(self): self.create_group() self.add_msg_to_stream() msgs = self.g.read_message("worker1", ">") self.g.delete_consumer("worker1") self.assertEqual( self.g.consumer_info(), [] ) def test_destroy_group(self): self.create_group() self.g.destroy() self.assertEqual( self.g.info(), dict() )
async def startup_tasks(app): mq = MessageQueue() await mq.connect() app['mq'] = mq