def publish(message):
""" Publish message to the indexing queue """
EXCHANGE = "crawler.vehicles"
QUEUE = "indexer_queue"
q = MessageQueue(EXCHANGE, QUEUE)
ROUTING_KEY = "indexer.index"
q.publish(ROUTING_KEY, message)
def setup_class(self):
self.client = Redis()
self.mq = MessageQueue(self.client, "email_queue")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
self.mq.add_message("*****@*****.**")
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream_key = "test_message_queue"
self.q = MessageQueue(self.client, self.stream_key)
self.message = {"name":"peter", "age":"36", "location":"UK"}
self.tearDown()
def setUp(self):
self.client = Redis(decode_responses=True)
self.client.flushdb()
self.mq = MessageQueue(self.client, "mq")
self.msg = "hello world"
self.blocktime = 1
def setUp(self):
self.client = Redis(decode_responses=True)
self.stream = "test_stream"
self.group = "test_group"
# objs
self.q = MessageQueue(self.client, self.stream)
self.g = Group(self.client, self.stream, self.group)
# create a empty stream key
self.msg_id = self.q.add_message({"k":"v"})
self.q.remove_message(self.msg_id)
def should_handle_chunks_of_data_stream_sliced_randomly():
stream = [
bytes('7|U|46|17\n', 'UTF-8'), b'6|U|46|68\n3|P|46|68\n4|F',
b'|4|68\n5|P|', b'5|68\n1', b'|F|46|49\n2|U|46|68\n'
]
queue = MessageQueue()
for chunk in stream:
queue.push_data_stream(chunk)
prev_seq = 0
for expected in [
b'1|F|46|49', b'2|U|46|68', b'3|P|46|68', b'4|F|4|68', b'5|P|5|68',
b'6|U|46|68', b'7|U|46|17'
]:
seq, data = queue.pop()
assert expected == data
assert seq >= prev_seq
assert not queue.queue
def should_block_for_next_item():
N = 5
queue = MessageQueue()
queue._push(N, 'data5')
consumed = []
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
queue._push(1, "data1")
consumer.join()
assert ['data1'] == consumed
queue._push(4, 'data4')
consumer = Thread(target=consume, args=[queue, consumed])
consumer.start()
consumer.join(.3) #wait
assert consumer.is_alive() #Thread is blocked
queue._push(2, 'data2')
consumer.join()
assert ['data1', 'data2'] == consumed
queue._push(3, 'data3')
consumer = Thread(target=consume, args=[queue, consumed, 3])
consumer.start()
consumer.join()
assert consumed == ['data1', 'data2', 'data3', 'data4', 'data5']
async def main():
logging.info("-- Starting message queue... To exit press CTRL+C")
mq = MessageQueue(handler=mq_handler)
await mq.connect()
await mq.listen()
from message_queue import MessageQueue
import downloader as dwn
EXCHANGE = "crawler.vehicles"
QUEUE = "crawler_queue"
queue = MessageQueue(EXCHANGE, QUEUE)
## Starting consumer
queue.start_consumer(dwn.process_message)
## Expected message format
# {
# "vehicle_thumbnails": [
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/f.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/r.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/2.jpg?var=1507107710",
# "https://srv1.sbtjapan.com/photo/F0000/3000/MF3332/3.jpg?var=1507107710"
# ],
# "_values": {
# "vehicle_drive": "2WD",
# "vehicle_body": "Hatchback",
# "vehicle_title": "TOYOTA RAUM 2006/9 CBA-NCZ20",
# "vehicle_price_cif": "N",
# "vehicle_accessories": "",
# "vehicle_engine": "1,490cc",
# "internal_id": "SBT-MF3332",
# "vehicle_make": "TOYOTA RAUM 2006/9 CBA-NCZ20",
# "vehicle_mileage": "82,000",
# "vehicle_year": "2006/9",
# "vehicle_doors": "5",
async def startup_tasks(app):
mq = MessageQueue()
await mq.connect()
app['mq'] = mq
#!/usr/bin/python
import socket
from threading import Thread
from event_handler import EventHandler
from message_queue import MessageQueue
from user_network import UserNetwork
users = {}
queue = MessageQueue()
event_handler = EventHandler(users, UserNetwork())
def bind_server(port, queue_len=1):
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('localhost', port))
serversocket.listen(queue_len)
return serversocket
def event_listener(port=9090):
serversocket = bind_server(port)
data = None
clientsocket = None
while True:
try:
if not data:
if clientsocket:
print(
"Event stream has stopped. Press Ctrl-C to stop the server"
)
clientsocket.close()