def publish(message):
""" Publish message to the indexing queue """
EXCHANGE = "crawler.vehicles"
QUEUE = "indexer_queue"
q = MessageQueue(EXCHANGE, QUEUE)
ROUTING_KEY = "indexer.index"
q.publish(ROUTING_KEY, message)
def setup_class(self):
self.client = Redis()
self.mq = MessageQueue(self.client, "email_queue")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
def setUp(self):
self.client = Redis(decode_responses=True)
self.client.flushdb()
self.mq = MessageQueue(self.client, "mq")
self.msg = "hello world"
self.blocktime = 1
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream_key = "test_message_queue"
self.q = MessageQueue(self.client, self.stream_key)
self.message = {"name":"peter", "age":"36", "location":"UK"}
self.tearDown()
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream = "test_stream"
self.group = "test_group"
# objs
self.q = MessageQueue(self.client, self.stream)
self.g = Group(self.client, self.stream, self.group)
# create a empty stream key
self.msg_id = self.q.add_message({"k":"v"})
self.q.remove_message(self.msg_id)
def should_handle_chunks_of_data_stream_sliced_randomly():
stream = [bytes('7|U|46|17\n', 'UTF-8'),
b'6|U|46|68\n3|P|46|68\n4|F',
b'|4|68\n5|P|',
b'5|68\n1',
b'|F|46|49\n2|U|46|68\n']
queue = MessageQueue()
for chunk in stream:
queue.push_data_stream(chunk)
prev_seq = 0
for expected in [b'1|F|46|49', b'2|U|46|68', b'3|P|46|68',
b'4|F|4|68', b'5|P|5|68', b'6|U|46|68',
b'7|U|46|17']:
seq, data = queue.pop()
assert expected == data
assert seq >= prev_seq
assert not queue.queue
def should_handle_chunks_of_data_stream_sliced_randomly():
stream = [
bytes('7|U|46|17\n', 'UTF-8'), b'6|U|46|68\n3|P|46|68\n4|F',
b'|4|68\n5|P|', b'5|68\n1', b'|F|46|49\n2|U|46|68\n'
]
queue = MessageQueue()
for chunk in stream:
queue.push_data_stream(chunk)
prev_seq = 0
for expected in [
b'1|F|46|49', b'2|U|46|68', b'3|P|46|68', b'4|F|4|68', b'5|P|5|68',
b'6|U|46|68', b'7|U|46|17'
]:
seq, data = queue.pop()
assert expected == data
assert seq >= prev_seq
assert not queue.queue
class TestMessageQueue:
def setup_class(self):
self.client = Redis()
self.mq = MessageQueue(self.client, "email_queue")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
def test_len(self):
assert self.mq.len() == 3
def test_consume(self):
assert self.mq.get_message() == "*****@*****.**"
assert self.mq.get_message() == "*****@*****.**"
assert self.mq.get_message() == "*****@*****.**"
#!/usr/bin/python
import socket
from threading import Thread
from event_handler import EventHandler
from message_queue import MessageQueue
from user_network import UserNetwork
users = {}
queue = MessageQueue()
event_handler = EventHandler(users, UserNetwork())
def bind_server(port, queue_len=1):
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('localhost', port))
serversocket.listen(queue_len)
return serversocket
def event_listener(port=9090):
serversocket = bind_server(port)
data = None
clientsocket = None
while True:
try:
if not data:
if clientsocket:
print(
"Event stream has stopped. Press Ctrl-C to stop the server"
)
clientsocket.close()
class TestMessageQueue(unittest.TestCase):
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream_key = "test_message_queue"
self.q = MessageQueue(self.client, self.stream_key)
self.message = {"name":"peter", "age":"36", "location":"UK"}
self.tearDown()
def tearDown(self):
self.client.delete(self.stream_key)
def make_a_none_empty_queue(self, queue, size):
for i in range(size):
key = "key{0}".format(i)
value = "value{0}".format(i)
self.q.add_message({key:value,})
def test_add_message(self):
message_id = self.q.add_message(self.message)
self.assertIsNotNone(message_id)
def test_add_message_return_utf8_id(self):
message_id = self.q.add_message(self.message)
self.assertIsInstance(message_id, str)
def test_get_message_return_none_when_msg_not_exists(self):
self.assertIsNone(self.q.get_message("10086-0"))
def test_get_message_return_msg_when_its_exists(self):
message_id = self.q.add_message(self.message)
self.assertEqual(
self.message,
self.q.get_message(message_id)
)
def test_remove_message(self):
message_id = self.q.add_message(self.message)
self.q.remove_message(message_id)
self.assertIsNone(self.q.get_message(message_id))
def test_len_return_zero_when_queue_empty(self):
self.assertEqual(self.q.len(), 0)
def test_len_retrun_non_zero_when_queue_not_empty(self):
id = self.q.add_message(self.message)
self.assertEqual(self.q.len(), 1)
self.q.remove_message(id)
self.assertEqual(self.q.len(), 0)
def test_get_by_range_return_empty_list_when_queue_empty(self):
self.assertEqual(self.q.get_by_range("-", "+"), list())
def test_get_by_range_return_none_empty_list_when_queue_not_empty(self):
self.make_a_none_empty_queue(self.q, 10)
result = self.q.get_by_range("-", "+")
self.assertEqual(len(result), 10)
self.assertEqual(type(result), list)
for msg in result:
self.assertEqual(type(msg), dict)
def test_get_by_range_return_right_size(self):
self.make_a_none_empty_queue(self.q, 10)
result = self.q.get_by_range("-", "+", 5)
self.assertEqual(len(result), 5)
def test_get_by_range_return_empty_list_when_stream_empty(self):
self.assertEqual(
self.q.get_by_range("-", "+"),
list()
)
def test_iterate_return_right_size(self):
self.make_a_none_empty_queue(self.q, 10)
self.assertEqual(
len(self.q.iterate(0, 5)),
5
)
self.assertEqual(
len(self.q.iterate(0, 10)),
10
)
def test_iterate_return_right_message(self):
self.make_a_none_empty_queue(self.q, 10)
full_message_list = self.q.get_by_range("-", "+", 10)
self.assertEqual(
self.q.iterate(0, 5),
full_message_list[0:5]
)
self.assertEqual(
self.q.iterate(0, 10),
full_message_list
)
def test_iterate_return_empty_list_when_stream_empty(self):
self.assertEqual(
self.q.iterate(0),
list()
)
def should_block_for_next_item():
N = 5
queue = MessageQueue()
queue._push(N, 'data5')
consumed = []
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
queue._push(1, "data1")
consumer.join()
assert ['data1'] == consumed
queue._push(4, 'data4')
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
consumer.join(.3) #wait
assert consumer.is_alive() #Thread is blocked
queue._push(2, 'data2')
consumer.join()
assert ['data1', 'data2'] == consumed
queue._push(3, 'data3')
consumer = Thread(target=consume, args=[queue, consumed, 3])
consumer.start()
consumer.join()
assert consumed == ['data1', 'data2', 'data3', 'data4', 'data5']
class TestMessageQueue(unittest.TestCase):
def setUp(self):
self.client = Redis(decode_responses=True)
self.client.flushdb()
self.mq = MessageQueue(self.client, "mq")
self.msg = "hello world"
self.blocktime = 1
def test_add(self):
self.mq.add_message(self.msg)
self.assertEqual(
self.mq.len(),
1
)
def test_get_message_non_block(self):
self.mq.add_message(self.msg)
self.assertEqual(
self.mq.get_message(),
self.msg
)
def test_get_message_blocking(self):
before = time()
print("blocking, wait for {0} second(s)".format(self.blocktime))
self.assertIsNone(
self.mq.get_message(self.blocktime)
)
self.assertTrue(
time() - before >= self.blocktime
)
def test_len(self):
self.assertEqual(
self.mq.len(),
0
)
self.mq.add_message(self.msg)
self.assertEqual(
self.mq.len(),
1
)
async def main():
logging.info("-- Starting message queue... To exit press CTRL+C")
mq = MessageQueue(handler=mq_handler)
await mq.connect()
await mq.listen()
from message_queue import MessageQueue
import downloader as dwn
EXCHANGE = "crawler.vehicles"
QUEUE = "crawler_queue"
queue = MessageQueue(EXCHANGE, QUEUE)
## Starting consumer
queue.start_consumer(dwn.process_message)
## Expected message format
# {
# "vehicle_thumbnails": [
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/f.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/r.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/2.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/3.jpg?var=1507107710"
# ],
# "_values": {
# "vehicle_drive": "2WD",
# "vehicle_body": "Hatchback",
# "vehicle_title": "TOYOTA RAUM 2006/9 CBA-NCZ20",
# "vehicle_price_cif": "N",
# "vehicle_accessories": "",
# "vehicle_engine": "1,490cc",
# "internal_id": "SBT-MF3332",
# "vehicle_make": "TOYOTA RAUM 2006/9 CBA-NCZ20",
# "vehicle_mileage": "82,000",
# "vehicle_year": "2006/9",
# "vehicle_doors": "5",
def should_block_for_next_item():
N = 5
queue = MessageQueue()
queue._push(N, 'data5')
consumed = []
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
queue._push(1, "data1")
consumer.join()
assert ['data1'] == consumed
queue._push(4, 'data4')
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
consumer.join(.3) #wait
assert consumer.is_alive() #Thread is blocked
queue._push(2, 'data2')
consumer.join()
assert ['data1', 'data2'] == consumed
queue._push(3, 'data3')
consumer = Thread(target=consume, args=[queue, consumed, 3])
consumer.start()
consumer.join()
assert consumed == ['data1', 'data2', 'data3', 'data4', 'data5']
class TestGroup(unittest.TestCase):
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream = "test_stream"
self.group = "test_group"
# objs
self.q = MessageQueue(self.client, self.stream)
self.g = Group(self.client, self.stream, self.group)
# create a empty stream key
self.msg_id = self.q.add_message({"k":"v"})
self.q.remove_message(self.msg_id)
def tearDown(self):
self.client.delete(self.stream)
def create_group(self):
self.g.create(0)
def add_msg_to_stream(self):
self.q.add_message({"k":"v"})
def test_create(self):
self.create_group()
self.assertNotEqual(
self.g.info(),
[]
)
def test_read_message(self):
self.create_group()
self.assertEqual(
self.g.read_message("worker1", ">"),
[]
)
self.add_msg_to_stream()
self.assertNotEqual(
self.g.read_message("worker1", ">"),
[]
)
def test_ack_message(self):
self.create_group()
self.add_msg_to_stream()
msgs = self.g.read_message("worker1", ">")
msg_id = list(msgs[0].keys())[0]
self.g.ack_message(msg_id)
pending_count = self.g.info()['pending']
self.assertEqual(
pending_count,
0
)
def test_info(self):
self.assertEqual(
self.g.info(),
dict()
)
self.create_group()
self.assertIsNotNone(
self.g.info()
)
def test_consumer_info(self):
self.create_group()
self.assertEqual(
self.g.consumer_info(),
[]
)
self.add_msg_to_stream()
msgs = self.g.read_message("worker1", ">")
self.assertNotEqual(
self.g.consumer_info(),
[]
)
def test_delete_consumer(self):
self.create_group()
self.add_msg_to_stream()
msgs = self.g.read_message("worker1", ">")
self.g.delete_consumer("worker1")
self.assertEqual(
self.g.consumer_info(),
[]
)
def test_destroy_group(self):
self.create_group()
self.g.destroy()
self.assertEqual(
self.g.info(),
dict()
)
async def startup_tasks(app):
mq = MessageQueue()
await mq.connect()
app['mq'] = mq