class FibonacciRpcClient(object):
def __init__(self):
self.connection = Connection(host='localhost',
heartbeat=None, debug=True)
self.channel = self.connection.channel()
result = self.channel.queue.declare(exclusive=True)
self.callback_queue = result[0]
print("callback_queue:", self.callback_queue)
self.channel.basic.consume(self.callback_queue,
self.on_response, no_ack=True)
def on_response(self, msg):
if msg.properties["correlation_id"] == self.corr_id:
self.response = msg.body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
msg = Message(str(n), reply_to=self.callback_queue,
correlation_id=self.corr_id)
self.channel.basic.publish(msg, '', 'rpc_queue')
while self.response is None:
self.connection.read_frames()
return int(self.response)
def __init__(self, **kwargs):
properties = {
"host": "localhost",
"port": 5672,
"user": "******",
"password": "******",
"vhost": "/",
"heartbeat": None,
"debug": False
}
self.kwargs = kwargs
for key, value in kwargs.iteritems():
if properties.has_key(key):
properties[key] = value
# Connection can be shared between multiple clients/workers
conn = self.kwargs.get("conn", None)
if conn is None:
conn = Connection(**properties)
self._conn = conn
self.channel = conn.channel()
class AsyncClient(object):
__metaclass__ = ClientMeta
def start(self, **kwargs):
self.connection = Connection(**kwargs)
self.channel = self.connection.channel()
self.loop = TaskLoop()
self.insert_task(self.read_frames, interval=0.01)
self.insert_task(self.__run)
self.loop.start()
def __run(self):
for declaration in itertools.chain(self._exchanges, self._queues, self._consumers):
declaration.client = weakref.ref(self)
declaration.declare()
# start 'auto' tasks
for name, attr in self.__class__.__dict__.iteritems():
if isinstance(attr, Task) and attr.auto:
getattr(self, name)()
for task, args, kwargs in self._tasks:
print 'scheduling %s from _tasks' % task
self.insert_task(task, *args, **kwargs)
# if there's a run function which isn't already a task, queue it
class_run = getattr(self.__class__, 'run', None)
self_run = getattr(self, 'run', None)
if callable(self_run) and not isinstance(class_run, Task):
self.run()
self.stop()
def stop(self):
self.connection.close()
def insert_task(self, task, *args, **kwargs):
loop = getattr(self, 'loop', None)
if isinstance(loop, TaskLoop):
loop.insert_task(task, *args, **kwargs)
else:
self._tasks.append((task, args, kwargs))
def read_frames(self):
if self.connection.close_info:
self.loop.stop()
else:
self.connection.read_frames()
@task
def basic_qos(self, **kwargs):
self.channel.basic.qos(**kwargs)
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange,
req_routing_key, req_msg_list, send_interval, report_queue,
state, codec):
assert_is_instance(send_interval, float)
assert_greater_equal(send_interval, 0.0)
conn, next_log_time, req_num = None, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
pch = conn.channel()
report_queue.put((name, 0, ProcessStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
req_num, total_sent = 0, 0
start_time = time.time()
next_send_time = start_time + send_interval
next_log_time = int(start_time) + 1
for app_msg in req_msg_list:
if 0 < send_interval:
dt = next_send_time - time.time()
if dt > 0.0:
time.sleep(dt)
pch.basic.publish(Message(app_msg), exchange, req_routing_key)
next_send_time += send_interval
req_num += 1
if time.time() >= next_log_time:
report_queue.put(
(name, next_log_time, DriverReqReport, (req_num, )))
total_sent += req_num
req_num = 0
next_log_time += 1
if StateRunning != state.value: break
except KeyboardInterrupt as e:
pass
finally:
total_sent += req_num
print name + ': end of loop, total %d msgs sent' % total_sent
report_queue.put((name, next_log_time, DriverReqReport, (req_num, )))
report_queue.put(
(name, next_log_time, DriverEndNotice, (total_sent, )))
if conn:
conn.close()
def __clear__(args):
bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
counter, timeout_time = 0, time.time()+1
try:
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(queue, durable=durable, auto_delete=False)
ch.queue.bind(queue, exchange, routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
while time.time() < timeout_time:
msg = ch.basic.get(queue)
if msg:
timeout_time = time.time() + 1
else:
time.sleep(0.01)
counter += 1
finally:
conn.close()
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange, req_routing_key,
req_msg_list, send_interval, report_queue, state, codec):
assert_is_instance(send_interval, float)
assert_greater_equal(send_interval, 0.0)
conn, next_log_time, req_num = None, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
pch = conn.channel()
report_queue.put((name, 0, ProcessStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
req_num, total_sent = 0, 0
start_time = time.time()
next_send_time = start_time + send_interval
next_log_time = int(start_time) + 1
for app_msg in req_msg_list:
if 0 < send_interval:
dt = next_send_time - time.time()
if dt > 0.0:
time.sleep(dt)
pch.basic.publish(Message(app_msg), exchange, req_routing_key)
next_send_time += send_interval
req_num += 1
if time.time() >= next_log_time:
report_queue.put((name, next_log_time, DriverReqReport, (req_num,)))
total_sent += req_num
req_num = 0
next_log_time += 1
if StateRunning != state.value: break
except KeyboardInterrupt as e:
pass
finally:
total_sent += req_num
print name + ': end of loop, total %d msgs sent'%total_sent
report_queue.put((name, next_log_time, DriverReqReport, (req_num,)))
report_queue.put((name, next_log_time, DriverEndNotice, (total_sent,)))
if conn:
conn.close()
def __clear__(args):
bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
counter, timeout_time = 0, time.time() + 1
try:
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(queue, durable=durable, auto_delete=False)
ch.queue.bind(queue, exchange, routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
while time.time() < timeout_time:
msg = ch.basic.get(queue)
if msg:
timeout_time = time.time() + 1
else:
time.sleep(0.01)
counter += 1
finally:
conn.close()
class FibonacciRpcClient(object):
def __init__(self):
self.connection = Connection(host='localhost', heartbeat=None, debug=True)
self.channel = self.connection.channel()
result = self.channel.queue.declare(exclusive=True)
self.callback_queue = result[0]
print("callback_queue:", self.callback_queue)
self.channel.basic.consume(self.callback_queue, self.on_response, no_ack=True)
def on_response(self, msg):
if msg.properties["correlation_id"] == self.corr_id:
self.response = msg.body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
msg = Message(str(n), reply_to=self.callback_queue, correlation_id=self.corr_id)
self.channel.basic.publish(msg, '', 'rpc_queue')
while self.response is None:
self.connection.read_frames()
return int(self.response)
data="".join(data)
s = sha1()
s.update("blob %u\0" % len(data))
s.update(data)
return s.hexdigest()
classifierTags = predict.CreateClassifier(SETTINGS_FILE_TAGS)
classifierNN = predict.CreateClassifier(SETTINGS_FILE_EMBEDDING)
print "Creating caffe client"
connection = Connection(
user='******', password='******',
vhost='/', host='52.86.70.177',
heartbeat=None, debug=True
)
channel = connection.channel()
channel.queue.declare(queue='rpc_queue', auto_delete=False)
def computeNN(imageURL):
filename=''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(10))+'.jpg'
filename='/home/ubuntu/caffe-cvprw15/examples/deepFashion/tmp/'+filename
if os.path.isfile(filename):
os.remove(filename)
urllib.urlretrieve(imageURL, filename)
imageHash=githash(filename)
dbRes=redisDb.fetchKey(imageHash+"_similar")
if dbRes:
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password,
exchange, rsp_queue, rsp_routing_key, report_que,
state, codec):
conn = None
next_log_time = int(time.time()) + 1
rsp_num, err_num, total_num = 0, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(rsp_queue, auto_delete=False)
ch.queue.bind(rsp_queue, exchange, rsp_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
#read all msgs in the consume queue before test
i = 0
msg = ch.basic.get(rsp_queue)
while msg:
msg = ch.basic.get(rsp_queue)
i += 1
print name + ': %d msgs read before test' % i
report_que.put((name, 0, ConsumerStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
next_log_time = int(time.time()) + 1
while StateRunning == state.value:
msg = ch.basic.get(rsp_queue)
if msg:
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = 0, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
else:
time.sleep(0.01)
if time.time() >= next_log_time:
report_que.put(
(name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num + err_num
rsp_num, err_num = 0, 0
next_log_time += 1
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1.0
while time.time() < timeout_time:
msg = ch.basic.get(rsp_queue)
if msg:
timeout_time = time.time() + 1.0
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = -1, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
conn.close()
report_que.put(
(name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num + err_num
print name + ': end of loop, total %d rsp' % total_num
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange,
consume_queue, req_routing_key, durable, rsp_routing_key,
mock_func, report_queue, state, codec):
conn, next_report_time = None, int(time.time()) + 1
req_num, rsp_num, err_num = 0, 0, 0
try:
conn = Connection(user=username,
password=password,
host=bus_ip,
port=bus_port,
vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(consume_queue, durable=durable, auto_delete=False)
ch.queue.bind(consume_queue, exchange, req_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
i = 0
while True:
msg = ch.basic.get()
if None == msg: break
i += 1
print 'Consumer of %s read %d msgs before test' % (name, i)
report_queue.put((name, 0, ProcessStarted, None))
while StateInit == state.value:
time.sleep(0.01)
assert_in(state.value, (StateRunning, StateStop))
while StateRunning == state.value:
msg = ch.basic.get()
if msg:
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
else:
time.sleep(0.01)
if time.time() >= next_report_time:
report_queue.put(
(pname, next_report_time, MockReport, (req_num, err_num)))
req_num, err_num = 0, 0
next_report_time += 1
#print pname + ' report_queue.put() counter=%d'%req_num
logging.debug(name + ': end of loop')
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1
while time.time() < timeout_time:
msg = ch.basic.get()
if msg:
timeout_time = time.time() + 1
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
report_queue.put(
(pname, next_report_time, MockReport, (req_num, err_num)))
conn.close()
logging.debug('End of ' + name)
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange,
consume_queue, req_routing_key, durable,
rsp_routing_key, mock_func, report_queue, state, codec):
conn, next_report_time = None, int(time.time()) +1
req_num, rsp_num, err_num = 0, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(consume_queue, durable=durable, auto_delete=False)
ch.queue.bind(consume_queue, exchange, req_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
i = 0
while True:
msg = ch.basic.get()
if None == msg: break
i += 1
print 'Consumer of %s read %d msgs before test'%(name, i)
report_queue.put((name, 0, ProcessStarted, None))
while StateInit == state.value:
time.sleep(0.01)
assert_in(state.value, (StateRunning, StateStop))
while StateRunning == state.value:
msg = ch.basic.get()
if msg:
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
else:
time.sleep(0.01)
if time.time() >= next_report_time:
report_queue.put((pname, next_report_time, MockReport, (req_num, err_num)))
req_num, err_num = 0, 0
next_report_time += 1
#print pname + ' report_queue.put() counter=%d'%req_num
logging.debug(name + ': end of loop')
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1
while time.time() < timeout_time:
msg = ch.basic.get()
if msg:
timeout_time = time.time() + 1
req_num += 1
if 'heartbeat' == msg.body:
session_id, protobuf_data = 0, body
else:
session_id, protobuf_data = self.codec.decode(msg.body)
rsp_data = mock_func(session_id, protobuf_data)
if rsp_data:
rsp_msg = codec.encode(session_id, rsp_data)
ch.basic.publish(rsp_msg, exchange, routing_key)
report_queue.put((pname, next_report_time, MockReport, (req_num, err_num)))
conn.close()
logging.debug('End of ' + name)
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
# Save the watchdog
self.watchdog = watchdog
def consumer(self, msg):
"""Called when a rabbitmq message arrive"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': #Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" %
(topic, body, bonusHeaders))
# Save the message as the last one
self.LAST_MESSAGES[topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders,
int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg, ))
except Exception as e:
self.logger.error("Error in consumer: %s", (e, ))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning(
"Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug(
"Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)"
% (config.RABBITMQ_USER, config.RABBITMQ_HOST,
config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER,
password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST,
host=config.RABBITMQ_HOST,
port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue, ))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True,
nowait=False,
cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=True,
nowait=False,
cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=False,
nowait=False,
cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" %
(config.RABBITMQ_EXCHANGE, ))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed')
and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e, ))
finally:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" %
(headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info(
"Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(
Message(message, application_headers=headers),
config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
class Connection(object):
def __init__(self, host, port, virtualhost, user, password, queue):
self.conn_id = hex(int(random.random() * 2**32))
self._conn_kwargs = {
"host": host,
"port": port,
"vhost": virtualhost,
"user": user,
"password": password,
"transport": "gevent",
"close_cb": self._on_disconnect,
"logger": logger,
}
self.queue = queue
self._connection = None
self._channel = None
self.lock = Semaphore()
self.broken = False
def log_debug(self, msg, *args, **kwargs):
# Ghetto log handler
logger.debug("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs)
def log_exception(self, msg, *args, **kwargs):
logger.exception("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs)
def disconnect(self):
self.log_debug("Disconnecting")
self.broken = True
self._connection.disconnect()
def _on_disconnect(self):
self.log_debug("Received disconnect")
self.broken = True
def _on_channel_closed(self, channel):
# Scrape the connection if our channel is closed.
self.log_debug("Received channel close")
self.disconnect()
def _open_connection(self):
self.log_debug("Opening RabbitMQ connection")
self._connection = HaighaConnection(**self._conn_kwargs)
self._start_read_loop()
def _open_channel(self):
# Open a channel and make sure we know if it gets closed
self.log_debug("Opening RabbitMQ channel")
self._channel = self._connection.channel()
self._channel.add_close_listener(self._on_channel_closed)
self._channel.queue.declare(self.queue, auto_delete=True)
def _ensure_open(self):
if self._channel is None:
self._open_connection()
self._open_channel()
@exec_or_break
def _read_frames(self):
self._connection.read_frames()
def _read_loop(self):
# The message pump needs to run for the connection to actually do something.
while not self.broken:
self._read_frames()
gevent.sleep() # Yield to other greenlets so they don't starve
def _start_read_loop(self):
self.log_debug("Starting connection loop")
gevent.spawn(self._read_loop) # Power our connection
@exec_or_break
def dispatch(self, key):
self._ensure_open()
self._channel.basic.publish(Message(key), "", self.queue)
@exec_or_break
def consume(self, consumer):
# Start consuming messages from the queue (they will be passed to `consumer`)
def cb():
self.log_debug("Registered as consumer")
def consumer_wrapper(message):
self.log_debug("Received a message: %s", message.body)
consumer(message)
self._ensure_open()
self._channel.basic.consume(queue=self.queue, consumer=consumer_wrapper, cb=cb)
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password, exchange,
rsp_queue, rsp_routing_key, report_que, state, codec):
conn = None
next_log_time = int(time.time()) + 1
rsp_num, err_num, total_num = 0, 0, 0
try:
conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost)
ch = conn.channel()
ch.exchange.declare(exchange, 'direct')
ch.queue.declare(rsp_queue, auto_delete=False)
ch.queue.bind(rsp_queue, exchange, rsp_routing_key)
ch.basic.qos(prefetch_size=0, prefetch_count=1000)
#read all msgs in the consume queue before test
i = 0
msg = ch.basic.get(rsp_queue)
while msg:
msg = ch.basic.get(rsp_queue)
i += 1
print name + ': %d msgs read before test'%i
report_que.put((name, 0, ConsumerStarted, None))
#waiting for tester set state to 1(means to start test)
while StateInit == state.value:
time.sleep(0.01)
eq_(StateRunning, state.value)
next_log_time = int(time.time()) + 1
while StateRunning == state.value:
msg = ch.basic.get(rsp_queue)
if msg:
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = 0, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
else:
time.sleep(0.01)
if time.time() >= next_log_time:
report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num+err_num
rsp_num, err_num = 0, 0
next_log_time += 1
except KeyboardInterrupt as e:
pass
finally:
if conn:
timeout_time = time.time() + 1.0
while time.time() < timeout_time:
msg = ch.basic.get(rsp_queue)
if msg:
timeout_time = time.time() + 1.0
s = ''.join(map(lambda b: chr(b), msg.body))
if 'heartbeat' == s:
session_id, protobuf_data = -1, s
rsp_num += 1
else:
session_id, protobuf_data = codec.decode(s)
if 0 == protobuf_data.RetCode:
rsp_num += 1
else:
err_num += 1
conn.close()
report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num)))
total_num += rsp_num+err_num
print name + ': end of loop, total %d rsp'%total_num
class AppClient(object):
def __init__(self, serverName, **options):
self.serverName = serverName
self._routes = {}
self.options = options
self.version = self.options.get("version", "0.1")
self.queueName = "{0}/{1}/request".format(self.serverName, self.version)
self.reply_queue_name = "{0}/{1}/reply".format(self.serverName, self.version)
self.exchange=self.options.get("exchange","main")
self.vhost = self.options.get("vhost", "/")
self.queueServer = self.options.get("queueServer","localhost")
self._pending_requests = {}
def route(self,operation):
def wrap(func):
self.add_route(operation, func)
return func
return wrap
def add_route(self,operation, func):
self._routes[operation]=func
def _process_reply(self, message):
'''
Receives message.body containing following attributes
{
context:
{
operation:<operation>,
params:<params>,
cid:<id for the request>
},
content:<content>
}
'''
logging.debug(" Reply: Getting msg: {0}".format(message))
body = str(message.body).encode('string_escape')
if not body:
pass
body=json.loads(body)
context = body["context"] or None
if not context:
logging.debug("QueueServer:Received invalid message: No context: {0}".format(message))
return
cid = context["cid"]
callback = self._pending_requests[cid]
if not callback:
logging.error("No callback registered for cid:",cid)
return
gevent.spawn(callback, body["content"])
gevent.sleep(0)
def send(self, message, options, callback):
context = options or {}
context["operation"]="get"
context["cid"]=1
self._pending_requests[context["cid"]]=callback
message = Message(json.dumps({
"context": context,
"content" : message
}),None, reply_to=self.reply_queue_name)
logging.debug("Sending message to {0}|{1}".format(self.exchange,self.queueName, message))
self._channel.basic.publish( message, self.exchange,self.queueName)
gevent.sleep(0)
def start(self):
self._connection = Connection(
transport='gevent',
user='******', password='******',
vhost=self.vhost, host=self.queueServer,
heartbeat=None, debug=True)
self._channel = self._connection.channel()
self._channel.add_close_listener(self._channel_closed_cb)
# Create and configure message exchange and queue
self._channel.exchange.declare(self.exchange, 'topic')
self._channel.queue.declare(self.queueName, auto_delete=False)
self._channel.queue.declare(self.reply_queue_name, auto_delete=False)
print "reply q name", self.reply_queue_name
self._channel.queue.bind(self.reply_queue_name, self.exchange, self.reply_queue_name)
# self._channel.basic.consume(queue=self.reply_queue_name,
# consumer=self._process_reply)
# Start message pump
self.running = True
gevent.spawn(self._message_pump_greenthread)
print "Started"
gevent.sleep(0)
def _message_pump_greenthread(self):
logging.debug("Entering Message Pump")
try:
while self.running:
# Pump
self._connection.read_frames()
# Yield to other greenlets so they don't starve
gevent.sleep()
except Exception as e:
print e
finally:
logging.debug("Leaving Message Pump, {0}".format(self.running))
return
def stop(self):
self.running = False
def _channel_closed_cb(self,*args):
print "Channel Closed"
class App(object):
def __init__(self, serverName, **options):
self.serverName = serverName
self._routes = {}
self.options = options
self.version = self.options.get("version", "0.1")
self.queueName = "{0}/{1}/request".format(self.serverName, self.version)
self.exchange=self.options.get("exchange","main")
self.vhost = self.options.get("vhost", "/")
self.queueServer = self.options.get("queueServer","localhost")
self._pending_requests = {}
def route(self,operation):
def wrap(func):
self.add_route(operation, func)
return func
return wrap
def add_route(self,operation, func):
self._routes[operation]=func
def get_route(self,operation):
return self._routes.get(operation,None)
def _process_message(self,message):
'''
Receives message.body containing following attributes
{
context:
{
operation:<operation>,
params:<params>,
cid:<id for the request>
},
content:<content>
}
'''
print " getting msg", message
body = str(message.body).encode('string_escape')
if not body:
pass
body=json.loads(body)
context = body["context"] or None
if not context:
logging.debug("QueueServer:Received invalid message: No context: {0}".format(message))
return
operation = context.get("operation",None)
print "\nOperation is ", operation
if not operation:
pass
func = self.get_route(operation);
if(not func):
pass
reply_to = message.properties.get("reply_to",None)
if not reply_to:
return
self._pending_requests[context["cid"]] = context
self._pending_requests[context["cid"]]["reply_to"]=reply_to
ret = func({
"context": context,
"content": body["content"]
})
# print "message properties ", message.properties
# self.send_reply(reply_to, ret)
def send_reply(self, message):
'''
message shoud be :
{
context:{
reply_to_cid:<unique id>
},
content:<content>
}
'''
reply_to_cid = message["reply_to_cid"] or None
if not reply_to_cid:
return
reply_context = self._pending_requests[reply_to_cid] or None
if not reply_context:
logging.error("No pending request for cid", reply_to_cid)
return
self.send_reply(reply_context["reply_to"], {
"context":reply_context,
"content":message["content"]
})
def _send_reply(self, reply_key, ret):
msg = Message(ret)
self._channel.basic.publish(msg, self.exchange, reply_key)
def _message_pump_greenthread(self):
print "Entering Message Pump"
try:
while self.running:
# Pump
self._connection.read_frames()
# Yield to other greenlets so they don't starve
gevent.sleep()
except Exception as e:
print e
finally:
print "Leaving Message Pump", self.running
#self._done_cb()
return
def start(self):
self._connection = Connection(
transport='gevent',
user='******', password='******',
vhost=self.vhost, host=self.queueServer,
heartbeat=None, debug=True)
self._channel = self._connection.channel()
self._channel.add_close_listener(self._channel_closed_cb)
# Create and configure message exchange and queue
print self.exchange
print self.queueName
self._channel.exchange.declare(self.exchange, 'topic')
self._channel.queue.declare(self.queueName, auto_delete=False)
self._channel.queue.bind(self.queueName, self.exchange, self.queueName)
self._channel.basic.consume(queue=self.queueName,
consumer=self._process_message)
# Start message pump
self.running = True
self._message_pump_greenlet = gevent.spawn(self._message_pump_greenthread)
print "Started"
def stop(self):
self.running = False
def _channel_closed_cb(*args):
print "Channel Closed", args
class Monitoring():
"""This class manages the connection to the monitoring system"""
def __init__(self):
self.logger = logging.getLogger('radiovisserver.monitoring')
self.buffer = Queue.Queue()
self.enabled = self.isEnabled()
self.connection = None
self.server_id = config.monitoring['server_id']
@skipIfDisabled
def log(self, label, content, client='', metadata='', timestamp=None):
""" Log a single event """
if timestamp == None:
timestamp = time.time()
# self.buffer.put({ 'type': 'log', 'label': label, 'content': content, 'client': client, 'metadata': metadata, 'timestamp': timestamp, 'server': self.server_id })
@skipIfDisabled
def count(self, label, metadata='', timestamp=None):
""" Log a single event """
if timestamp == None:
timestamp = time.time()
self.buffer.put(
{'type': 'log', 'label': label, 'timestamp': timestamp, 'server': self.server_id, 'metadata': metadata})
@skipIfDisabled
def gauge(self, label, level, metadata='', timestamp=None):
""" Log an aggregated value """
if timestamp == None:
timestamp = time.time()
self.buffer.put(
{'type': 'gauge', 'label': label, 'level': level, 'timestamp': timestamp, 'server': self.server_id})
def run(self):
"""Thread with connection to send monitoring data"""
if not self.isEnabled():
self.logger.info("Monitoring disabled by config.py")
return
else:
self.logger.info("Monitoring enabled by config.py")
while True:
try:
time.sleep(1)
self.logger.debug(
"Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.monitoring['user'],
config.monitoring['host'],
config.monitoring['port'],
config.monitoring['vhost']))
self.connection = Connection(user=config.monitoring['user'],
password=config.monitoring['password'],
vhost=config.monitoring['vhost'],
host=config.monitoring['host'],
port=config.monitoring['port'],
debug=config.monitoring['debug'])
self.logger.debug("Creating the channel")
self.ch = self.connection.channel()
# Name will come from a callback
global queue_name
queue_name = None
queue_name = config.monitoring['queue']
self.logger.debug("Creating the queue")
self.ch.exchange.declare(config.monitoring['exchange'], 'fanout')
self.ch.queue.declare(queue=queue_name, durable=True, auto_delete=False, nowait=False)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" % (config.monitoring['exchange'],))
self.ch.queue.bind(queue_name, config.monitoring['exchange'], '')
self.logger.debug("Ready, waiting for monitoring data")
while True:
data = json.dumps(self.buffer.get(block=True, timeout=None))
headers = {}
headers['when'] = str(int(time.time()))
self.ch.basic.publish(Message(data, application_headers=headers), config.monitoring['exchange'], '')
except Exception as e:
self.logger.error("Error in run: %s" % (e,))
self.logger.error('Monitoring connection failed. Reconnecting in 30sec')
finally:
if self.connection:
self.logger.info('connection closed')
self.connection.close()
time.sleep(30)
def isEnabled(self):
return hasattr(config, 'monitoring') and config.monitoring['enabled']
#!/usr/bin/env python
'''
demostrate how to write a rpc server
'''
import sys, os, uuid, time
sys.path.append(os.path.abspath(".."))
from haigha.connection import Connection
from haigha.message import Message
connection = Connection(host='localhost', heartbeat=None, debug=True)
channel = connection.channel()
channel.queue.declare(queue='rpc_queue', auto_delete=False)
def fib(n):
if n == 0:
return 0
elif n == 1:
return 1
else:
return fib(n - 1) + fib(n - 2)
def on_request(msg):
n = int(msg.body)
print " [.] fib(%s)" % (n, )
result = fib(n)
reply_to = msg.properties["reply_to"]
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, monitoring=None, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# RadioDNS
self.radioDns = RadioDns()
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
self.monitoring = monitoring
# Save the watchdog
self.watchdog = watchdog
# The global gauge
# Initialize RadioDNS Caches
self.radioDns.update_channel_topics()
self.cox = None
def consumer(self, msg):
"""Called when a rabbitmq message arrives"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': # Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" % (topic, body, bonusHeaders))
# Save the message as the last one
converted_topic = self.radioDns.convert_fm_topic_to_gcc(topic)
self.LAST_MESSAGES[converted_topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
time.sleep(0) # Switch context
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders, int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg,))
except Exception as e:
self.logger.error("Error in consumer: %s", (e,))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning("Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug("Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (
config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue,))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE,))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed') and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e,))
finally:
if self.cox is not None:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" % (headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info("Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(Message(message, application_headers=headers), config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
self.update_stats()
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
self.update_stats()
def update_stats(self):
"""Update stats"""
# self.gauge.send(config.STATS_GAUGE_NB_CLIENTS, len(self.stompservers))
pass
