class AsyncClient(object):
__metaclass__ = ClientMeta
def start(self, **kwargs):
self.connection = Connection(**kwargs)
self.channel = self.connection.channel()
self.loop = TaskLoop()
self.insert_task(self.read_frames, interval=0.01)
self.insert_task(self.__run)
self.loop.start()
def __run(self):
for declaration in itertools.chain(self._exchanges, self._queues, self._consumers):
declaration.client = weakref.ref(self)
declaration.declare()
# start 'auto' tasks
for name, attr in self.__class__.__dict__.iteritems():
if isinstance(attr, Task) and attr.auto:
getattr(self, name)()
for task, args, kwargs in self._tasks:
print 'scheduling %s from _tasks' % task
self.insert_task(task, *args, **kwargs)
# if there's a run function which isn't already a task, queue it
class_run = getattr(self.__class__, 'run', None)
self_run = getattr(self, 'run', None)
if callable(self_run) and not isinstance(class_run, Task):
self.run()
self.stop()
def stop(self):
self.connection.close()
def insert_task(self, task, *args, **kwargs):
loop = getattr(self, 'loop', None)
if isinstance(loop, TaskLoop):
loop.insert_task(task, *args, **kwargs)
else:
self._tasks.append((task, args, kwargs))
def read_frames(self):
if self.connection.close_info:
self.loop.stop()
else:
self.connection.read_frames()
@task
def basic_qos(self, **kwargs):
self.channel.basic.qos(**kwargs)
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange,
req_routing_key, req_msg_list, send_interval, report_queue,
state, codec):
assert_is_instance(send_interval, float)
assert_greater_equal(send_interval, 0.0)
conn, next_log_time, req_num = None, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
pch = conn.channel()
report_queue.put((name, 0, ProcessStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
req_num, total_sent = 0, 0
start_time = time.time()
next_send_time = start_time + send_interval
next_log_time = int(start_time) + 1
for app_msg in req_msg_list:
if 0 < send_interval:
dt = next_send_time - time.time()
if dt > 0.0:
time.sleep(dt)
pch.basic.publish(Message(app_msg), exchange, req_routing_key)
next_send_time += send_interval
req_num += 1
if time.time() >= next_log_time:
report_queue.put(
(name, next_log_time, DriverReqReport, (req_num, )))
total_sent += req_num
req_num = 0
next_log_time += 1
if StateRunning != state.value: break
except KeyboardInterrupt as e:
pass
finally:
total_sent += req_num
print name + ': end of loop, total %d msgs sent' % total_sent
report_queue.put((name, next_log_time, DriverReqReport, (req_num, )))
report_queue.put(
(name, next_log_time, DriverEndNotice, (total_sent, )))
if conn:
conn.close()
def __clear__(args):
bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
counter, timeout_time = 0, time.time()+1
try:
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(queue, durable=durable, auto_delete=False)
ch.queue.bind(queue, exchange, routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
while time.time() < timeout_time:
msg = ch.basic.get(queue)
if msg:
timeout_time = time.time() + 1
else:
time.sleep(0.01)
counter += 1
finally:
conn.close()
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange, req_routing_key,
req_msg_list, send_interval, report_queue, state, codec):
assert_is_instance(send_interval, float)
assert_greater_equal(send_interval, 0.0)
conn, next_log_time, req_num = None, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
pch = conn.channel()
report_queue.put((name, 0, ProcessStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
req_num, total_sent = 0, 0
start_time = time.time()
next_send_time = start_time + send_interval
next_log_time = int(start_time) + 1
for app_msg in req_msg_list:
if 0 < send_interval:
dt = next_send_time - time.time()
if dt > 0.0:
time.sleep(dt)
pch.basic.publish(Message(app_msg), exchange, req_routing_key)
next_send_time += send_interval
req_num += 1
if time.time() >= next_log_time:
report_queue.put((name, next_log_time, DriverReqReport, (req_num,)))
total_sent += req_num
req_num = 0
next_log_time += 1
if StateRunning != state.value: break
except KeyboardInterrupt as e:
pass
finally:
total_sent += req_num
print name + ': end of loop, total %d msgs sent'%total_sent
report_queue.put((name, next_log_time, DriverReqReport, (req_num,)))
report_queue.put((name, next_log_time, DriverEndNotice, (total_sent,)))
if conn:
conn.close()
def __clear__(args):
bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
counter, timeout_time = 0, time.time() + 1
try:
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(queue, durable=durable, auto_delete=False)
ch.queue.bind(queue, exchange, routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
while time.time() < timeout_time:
msg = ch.basic.get(queue)
if msg:
timeout_time = time.time() + 1
else:
time.sleep(0.01)
counter += 1
finally:
conn.close()
def _connect_to_broker(self):
''' Connect to broker and regisiter cleanup action to disconnect
:returns: connection instance
:rtype: `haigha.connection.Connection`
'''
sock_opts = {
(socket.IPPROTO_TCP, socket.TCP_NODELAY): 1,
}
connection = Connection(logger=_LOG,
debug=_OPTIONS.debug,
user=_OPTIONS.user,
password=_OPTIONS.password,
vhost=_OPTIONS.vhost,
host=_OPTIONS.host,
heartbeat=None,
sock_opts=sock_opts,
transport='socket')
self.addCleanup(lambda: connection.close(disconnect=True)
if not connection.closed else None)
return connection
def _connect_to_broker(self):
''' Connect to broker and regisiter cleanup action to disconnect
:returns: connection instance
:rtype: `haigha.connection.Connection`
'''
sock_opts = {
(socket.IPPROTO_TCP, socket.TCP_NODELAY) : 1,
}
connection = Connection(
logger=_LOG,
debug=_OPTIONS.debug,
user=_OPTIONS.user,
password=_OPTIONS.password,
vhost=_OPTIONS.vhost,
host=_OPTIONS.host,
heartbeat=None,
sock_opts=sock_opts,
transport='socket')
self.addCleanup(lambda: connection.close(disconnect=True)
if not connection.closed else None)
return connection
from haigha.connection import Connection
from haigha.message import Message
connection = Connection(user='******',
password='******',
vhost='/',
host='192.168.59.103',
heartbeat=None,
debug=True)
ch = connection.channel()
ch.exchange.declare('test_exchange', 'direct')
ch.queue.declare('test_queue', auto_delete=True)
ch.queue.bind('test_queue', 'test_exchange', 'test_key')
for i in xrange(30000):
ch.basic.publish(Message('body', application_headers={'hello': 'world'}),
'test_exchange', 'test_key')
connection.close()
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password,
exchange, rsp_queue, rsp_routing_key, report_que,
state, codec):
conn = None
next_log_time = int(time.time()) + 1
rsp_num, err_num, total_num = 0, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(rsp_queue, auto_delete=False)
ch.queue.bind(rsp_queue, exchange, rsp_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
#read all msgs in the consume queue before test
i = 0
msg = ch.basic.get(rsp_queue)
while msg:
msg = ch.basic.get(rsp_queue)
i += 1
print name + ': %d msgs read before test' % i
report_que.put((name, 0, ConsumerStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
next_log_time = int(time.time()) + 1
while StateRunning == state.value:
msg = ch.basic.get(rsp_queue)
if msg:
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = 0, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
else:
time.sleep(0.01)
if time.time() >= next_log_time:
report_que.put(
(name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num + err_num
rsp_num, err_num = 0, 0
next_log_time += 1
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1.0
while time.time() < timeout_time:
msg = ch.basic.get(rsp_queue)
if msg:
timeout_time = time.time() + 1.0
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = -1, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
conn.close()
report_que.put(
(name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num + err_num
print name + ': end of loop, total %d rsp' % total_num
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange,
consume_queue, req_routing_key, durable, rsp_routing_key,
mock_func, report_queue, state, codec):
conn, next_report_time = None, int(time.time()) + 1
req_num, rsp_num, err_num = 0, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(consume_queue, durable=durable, auto_delete=False)
ch.queue.bind(consume_queue, exchange, req_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
i = 0
while True:
msg = ch.basic.get()
if None == msg: break
i += 1
print 'Consumer of %s read %d msgs before test' % (name, i)
report_queue.put((name, 0, ProcessStarted, None))
while StateInit == state.value:
time.sleep(0.01)
assert_in(state.value, (StateRunning, StateStop))
while StateRunning == state.value:
msg = ch.basic.get()
if msg:
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
else:
time.sleep(0.01)
if time.time() >= next_report_time:
report_queue.put(
(pname, next_report_time, MockReport, (req_num, err_num)))
req_num, err_num = 0, 0
next_report_time += 1
#print pname + ' report_queue.put() counter=%d'%req_num
logging.debug(name + ': end of loop')
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1
while time.time() < timeout_time:
msg = ch.basic.get()
if msg:
timeout_time = time.time() + 1
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
report_queue.put(
(pname, next_report_time, MockReport, (req_num, err_num)))
conn.close()
logging.debug('End of ' + name)
class Monitoring():
"""This class manages the connection to the monitoring system"""
def __init__(self):
self.logger = logging.getLogger('radiovisserver.monitoring')
self.buffer = Queue.Queue()
self.enabled = self.isEnabled()
self.connection = None
self.server_id = config.monitoring['server_id']
@skipIfDisabled
def log(self, label, content, client='', metadata='', timestamp=None):
""" Log a single event """
if timestamp == None:
timestamp = time.time()
# self.buffer.put({ 'type': 'log', 'label': label, 'content': content, 'client': client, 'metadata': metadata, 'timestamp': timestamp, 'server': self.server_id })
@skipIfDisabled
def count(self, label, metadata='', timestamp=None):
""" Log a single event """
if timestamp == None:
timestamp = time.time()
self.buffer.put(
{'type': 'log', 'label': label, 'timestamp': timestamp, 'server': self.server_id, 'metadata': metadata})
@skipIfDisabled
def gauge(self, label, level, metadata='', timestamp=None):
""" Log an aggregated value """
if timestamp == None:
timestamp = time.time()
self.buffer.put(
{'type': 'gauge', 'label': label, 'level': level, 'timestamp': timestamp, 'server': self.server_id})
def run(self):
"""Thread with connection to send monitoring data"""
if not self.isEnabled():
self.logger.info("Monitoring disabled by config.py")
return
else:
self.logger.info("Monitoring enabled by config.py")
while True:
try:
time.sleep(1)
self.logger.debug(
"Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.monitoring['user'],
config.monitoring['host'],
config.monitoring['port'],
config.monitoring['vhost']))
self.connection = Connection(user=config.monitoring['user'],
password=config.monitoring['password'],
vhost=config.monitoring['vhost'],
host=config.monitoring['host'],
port=config.monitoring['port'],
debug=config.monitoring['debug'])
self.logger.debug("Creating the channel")
self.ch = self.connection.channel()
# Name will come from a callback
global queue_name
queue_name = None
queue_name = config.monitoring['queue']
self.logger.debug("Creating the queue")
self.ch.exchange.declare(config.monitoring['exchange'], 'fanout')
self.ch.queue.declare(queue=queue_name, durable=True, auto_delete=False, nowait=False)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" % (config.monitoring['exchange'],))
self.ch.queue.bind(queue_name, config.monitoring['exchange'], '')
self.logger.debug("Ready, waiting for monitoring data")
while True:
data = json.dumps(self.buffer.get(block=True, timeout=None))
headers = {}
headers['when'] = str(int(time.time()))
self.ch.basic.publish(Message(data, application_headers=headers), config.monitoring['exchange'], '')
except Exception as e:
self.logger.error("Error in run: %s" % (e,))
self.logger.error('Monitoring connection failed. Reconnecting in 30sec')
finally:
if self.connection:
self.logger.info('connection closed')
self.connection.close()
time.sleep(30)
def isEnabled(self):
return hasattr(config, 'monitoring') and config.monitoring['enabled']
import sys
from haigha.connection import Connection
from haigha.message import Message
sys.path.append("/home/ec2-user/source/Quant/component")
import rdm
r = rdm.RedisInfo()
connection = Connection(
host=r.rmq['hostname'], port=r.rmq['port'],
user=r.rmq['user'], password=r.rmq['pass'],
vhost='/', heartbeat=None, debug=True)
ch = connection.channel()
ch.exchange.declare('test_exchange', 'direct')
ch.queue.declare('test_queue', auto_delete=True)
ch.queue.bind('test_queue', 'test_exchange', 'test_key')
ch.basic.publish( Message('body', application_headers={'hello':'world'}), exchange='test_exchange', routing_key='test_key' )
mq = ch.basic.get('test_queue')
connection.close()
print mq
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
# Save the watchdog
self.watchdog = watchdog
def consumer(self, msg):
"""Called when a rabbitmq message arrive"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': #Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" %
(topic, body, bonusHeaders))
# Save the message as the last one
self.LAST_MESSAGES[topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders,
int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg, ))
except Exception as e:
self.logger.error("Error in consumer: %s", (e, ))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning(
"Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug(
"Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)"
% (config.RABBITMQ_USER, config.RABBITMQ_HOST,
config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER,
password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST,
host=config.RABBITMQ_HOST,
port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue, ))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True,
nowait=False,
cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=True,
nowait=False,
cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=False,
nowait=False,
cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" %
(config.RABBITMQ_EXCHANGE, ))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed')
and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e, ))
finally:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" %
(headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info(
"Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(
Message(message, application_headers=headers),
config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password, exchange,
rsp_queue, rsp_routing_key, report_que, state, codec):
conn = None
next_log_time = int(time.time()) + 1
rsp_num, err_num, total_num = 0, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(rsp_queue, auto_delete=False)
ch.queue.bind(rsp_queue, exchange, rsp_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
#read all msgs in the consume queue before test
i = 0
msg = ch.basic.get(rsp_queue)
while msg:
msg = ch.basic.get(rsp_queue)
i += 1
print name + ': %d msgs read before test'%i
report_que.put((name, 0, ConsumerStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
next_log_time = int(time.time()) + 1
while StateRunning == state.value:
msg = ch.basic.get(rsp_queue)
if msg:
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = 0, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
else:
time.sleep(0.01)
if time.time() >= next_log_time:
report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num+err_num
rsp_num, err_num = 0, 0
next_log_time += 1
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1.0
while time.time() < timeout_time:
msg = ch.basic.get(rsp_queue)
if msg:
timeout_time = time.time() + 1.0
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = -1, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
conn.close()
report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num+err_num
print name + ': end of loop, total %d rsp'%total_num
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange,
consume_queue, req_routing_key, durable,
rsp_routing_key, mock_func, report_queue, state, codec):
conn, next_report_time = None, int(time.time()) +1
req_num, rsp_num, err_num = 0, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(consume_queue, durable=durable, auto_delete=False)
ch.queue.bind(consume_queue, exchange, req_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
i = 0
while True:
msg = ch.basic.get()
if None == msg: break
i += 1
print 'Consumer of %s read %d msgs before test'%(name, i)
report_queue.put((name, 0, ProcessStarted, None))
while StateInit == state.value:
time.sleep(0.01)
assert_in(state.value, (StateRunning, StateStop))
while StateRunning == state.value:
msg = ch.basic.get()
if msg:
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
else:
time.sleep(0.01)
if time.time() >= next_report_time:
report_queue.put((pname, next_report_time, MockReport, (req_num, err_num)))
req_num, err_num = 0, 0
next_report_time += 1
#print pname + ' report_queue.put() counter=%d'%req_num
logging.debug(name + ': end of loop')
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1
while time.time() < timeout_time:
msg = ch.basic.get()
if msg:
timeout_time = time.time() + 1
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
report_queue.put((pname, next_report_time, MockReport, (req_num, err_num)))
conn.close()
logging.debug('End of ' + name)
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, monitoring=None, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# RadioDNS
self.radioDns = RadioDns()
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
self.monitoring = monitoring
# Save the watchdog
self.watchdog = watchdog
# The global gauge
# Initialize RadioDNS Caches
self.radioDns.update_channel_topics()
self.cox = None
def consumer(self, msg):
"""Called when a rabbitmq message arrives"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': # Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" % (topic, body, bonusHeaders))
# Save the message as the last one
converted_topic = self.radioDns.convert_fm_topic_to_gcc(topic)
self.LAST_MESSAGES[converted_topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
time.sleep(0) # Switch context
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders, int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg,))
except Exception as e:
self.logger.error("Error in consumer: %s", (e,))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning("Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug("Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (
config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue,))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE,))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed') and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e,))
finally:
if self.cox is not None:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" % (headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info("Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(Message(message, application_headers=headers), config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
self.update_stats()
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
self.update_stats()
def update_stats(self):
"""Update stats"""
# self.gauge.send(config.STATS_GAUGE_NB_CLIENTS, len(self.stompservers))
pass
