class FibonacciRpcClient(object):
def __init__(self):
self.connection = Connection(host='localhost',
heartbeat=None, debug=True)
self.channel = self.connection.channel()
result = self.channel.queue.declare(exclusive=True)
self.callback_queue = result[0]
print("callback_queue:", self.callback_queue)
self.channel.basic.consume(self.callback_queue,
self.on_response, no_ack=True)
def on_response(self, msg):
if msg.properties["correlation_id"] == self.corr_id:
self.response = msg.body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
msg = Message(str(n), reply_to=self.callback_queue,
correlation_id=self.corr_id)
self.channel.basic.publish(msg, '', 'rpc_queue')
while self.response is None:
self.connection.read_frames()
return int(self.response)
class AsyncClient(object):
__metaclass__ = ClientMeta
def start(self, **kwargs):
self.connection = Connection(**kwargs)
self.channel = self.connection.channel()
self.loop = TaskLoop()
self.insert_task(self.read_frames, interval=0.01)
self.insert_task(self.__run)
self.loop.start()
def __run(self):
for declaration in itertools.chain(self._exchanges, self._queues, self._consumers):
declaration.client = weakref.ref(self)
declaration.declare()
# start 'auto' tasks
for name, attr in self.__class__.__dict__.iteritems():
if isinstance(attr, Task) and attr.auto:
getattr(self, name)()
for task, args, kwargs in self._tasks:
print 'scheduling %s from _tasks' % task
self.insert_task(task, *args, **kwargs)
# if there's a run function which isn't already a task, queue it
class_run = getattr(self.__class__, 'run', None)
self_run = getattr(self, 'run', None)
if callable(self_run) and not isinstance(class_run, Task):
self.run()
self.stop()
def stop(self):
self.connection.close()
def insert_task(self, task, *args, **kwargs):
loop = getattr(self, 'loop', None)
if isinstance(loop, TaskLoop):
loop.insert_task(task, *args, **kwargs)
else:
self._tasks.append((task, args, kwargs))
def read_frames(self):
if self.connection.close_info:
self.loop.stop()
else:
self.connection.read_frames()
@task
def basic_qos(self, **kwargs):
self.channel.basic.qos(**kwargs)
class FibonacciRpcClient(object):
def __init__(self):
self.connection = Connection(host='localhost', heartbeat=None, debug=True)
self.channel = self.connection.channel()
result = self.channel.queue.declare(exclusive=True)
self.callback_queue = result[0]
print("callback_queue:", self.callback_queue)
self.channel.basic.consume(self.callback_queue, self.on_response, no_ack=True)
def on_response(self, msg):
if msg.properties["correlation_id"] == self.corr_id:
self.response = msg.body
def call(self, n):
self.response = None
self.corr_id = str(uuid.uuid4())
msg = Message(str(n), reply_to=self.callback_queue, correlation_id=self.corr_id)
self.channel.basic.publish(msg, '', 'rpc_queue')
while self.response is None:
self.connection.read_frames()
return int(self.response)
def fib(n):
if n == 0:
return 0
elif n == 1:
return 1
else:
return fib(n - 1) + fib(n - 2)
def on_request(msg):
n = int(msg.body)
print " [.] fib(%s)" % (n, )
result = fib(n)
reply_to = msg.properties["reply_to"]
correlation_id = msg.properties["correlation_id"]
resp = Message(str(result), correlation_id=correlation_id)
channel.basic.publish(resp, '', reply_to)
delivery_info = msg.delivery_info
channel.basic.ack(delivery_info["delivery_tag"])
channel.basic.qos(prefetch_count=1)
channel.basic.consume('rpc_queue', on_request, no_ack=False)
print " [x] Awaiting RPC requests"
while not channel.closed:
connection.read_frames()
class AppClient(object):
def __init__(self, serverName, **options):
self.serverName = serverName
self._routes = {}
self.options = options
self.version = self.options.get("version", "0.1")
self.queueName = "{0}/{1}/request".format(self.serverName, self.version)
self.reply_queue_name = "{0}/{1}/reply".format(self.serverName, self.version)
self.exchange=self.options.get("exchange","main")
self.vhost = self.options.get("vhost", "/")
self.queueServer = self.options.get("queueServer","localhost")
self._pending_requests = {}
def route(self,operation):
def wrap(func):
self.add_route(operation, func)
return func
return wrap
def add_route(self,operation, func):
self._routes[operation]=func
def _process_reply(self, message):
'''
Receives message.body containing following attributes
{
context:
{
operation:<operation>,
params:<params>,
cid:<id for the request>
},
content:<content>
}
'''
logging.debug(" Reply: Getting msg: {0}".format(message))
body = str(message.body).encode('string_escape')
if not body:
pass
body=json.loads(body)
context = body["context"] or None
if not context:
logging.debug("QueueServer:Received invalid message: No context: {0}".format(message))
return
cid = context["cid"]
callback = self._pending_requests[cid]
if not callback:
logging.error("No callback registered for cid:",cid)
return
gevent.spawn(callback, body["content"])
gevent.sleep(0)
def send(self, message, options, callback):
context = options or {}
context["operation"]="get"
context["cid"]=1
self._pending_requests[context["cid"]]=callback
message = Message(json.dumps({
"context": context,
"content" : message
}),None, reply_to=self.reply_queue_name)
logging.debug("Sending message to {0}|{1}".format(self.exchange,self.queueName, message))
self._channel.basic.publish( message, self.exchange,self.queueName)
gevent.sleep(0)
def start(self):
self._connection = Connection(
transport='gevent',
user='******', password='******',
vhost=self.vhost, host=self.queueServer,
heartbeat=None, debug=True)
self._channel = self._connection.channel()
self._channel.add_close_listener(self._channel_closed_cb)
# Create and configure message exchange and queue
self._channel.exchange.declare(self.exchange, 'topic')
self._channel.queue.declare(self.queueName, auto_delete=False)
self._channel.queue.declare(self.reply_queue_name, auto_delete=False)
print "reply q name", self.reply_queue_name
self._channel.queue.bind(self.reply_queue_name, self.exchange, self.reply_queue_name)
# self._channel.basic.consume(queue=self.reply_queue_name,
# consumer=self._process_reply)
# Start message pump
self.running = True
gevent.spawn(self._message_pump_greenthread)
print "Started"
gevent.sleep(0)
def _message_pump_greenthread(self):
logging.debug("Entering Message Pump")
try:
while self.running:
# Pump
self._connection.read_frames()
# Yield to other greenlets so they don't starve
gevent.sleep()
except Exception as e:
print e
finally:
logging.debug("Leaving Message Pump, {0}".format(self.running))
return
def stop(self):
self.running = False
def _channel_closed_cb(self,*args):
print "Channel Closed"
tags=predict.InputImagePredict(filename,SETTINGS_FILE_TAGS,"tags",classifierTags)
redisDb.insertKey(imageHash+"_tags",tags)
# db.InsertData({'imageName':imageHash,'similarProducts':[],'crossProducts':[],'tags':tags},mode="tags")
return {'imageURL':imageURL,'result':tags}
def on_request(msg):
val = json.loads(str(msg.body))
print val
imageURL=val['url']
if val['type']=="similar":
result=computeNN(imageURL)
else:
result=computeTags(imageURL)
print result
reply_to = msg.properties["reply_to"]
correlation_id = msg.properties["correlation_id"]
resp = Message(json.dumps(result), correlation_id=correlation_id)
channel.basic.publish(resp,'',reply_to)
delivery_info = msg.delivery_info
channel.basic.ack(delivery_info["delivery_tag"])
channel.basic.qos(prefetch_count=2)
channel.basic.consume('rpc_queue', on_request, no_ack=False)
print " [x] Awaiting RPC requests"
while not channel.closed:
connection.read_frames()
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
# Save the watchdog
self.watchdog = watchdog
def consumer(self, msg):
"""Called when a rabbitmq message arrive"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': #Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" %
(topic, body, bonusHeaders))
# Save the message as the last one
self.LAST_MESSAGES[topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders,
int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg, ))
except Exception as e:
self.logger.error("Error in consumer: %s", (e, ))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning(
"Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug(
"Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)"
% (config.RABBITMQ_USER, config.RABBITMQ_HOST,
config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER,
password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST,
host=config.RABBITMQ_HOST,
port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue, ))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True,
nowait=False,
cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=True,
nowait=False,
cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE,
auto_delete=False,
nowait=False,
cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" %
(config.RABBITMQ_EXCHANGE, ))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed')
and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e, ))
finally:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" %
(headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info(
"Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(
Message(message, application_headers=headers),
config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
class Connection(object):
def __init__(self, host, port, virtualhost, user, password, queue):
self.conn_id = hex(int(random.random() * 2**32))
self._conn_kwargs = {
"host": host,
"port": port,
"vhost": virtualhost,
"user": user,
"password": password,
"transport": "gevent",
"close_cb": self._on_disconnect,
"logger": logger,
}
self.queue = queue
self._connection = None
self._channel = None
self.lock = Semaphore()
self.broken = False
def log_debug(self, msg, *args, **kwargs):
# Ghetto log handler
logger.debug("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs)
def log_exception(self, msg, *args, **kwargs):
logger.exception("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs)
def disconnect(self):
self.log_debug("Disconnecting")
self.broken = True
self._connection.disconnect()
def _on_disconnect(self):
self.log_debug("Received disconnect")
self.broken = True
def _on_channel_closed(self, channel):
# Scrape the connection if our channel is closed.
self.log_debug("Received channel close")
self.disconnect()
def _open_connection(self):
self.log_debug("Opening RabbitMQ connection")
self._connection = HaighaConnection(**self._conn_kwargs)
self._start_read_loop()
def _open_channel(self):
# Open a channel and make sure we know if it gets closed
self.log_debug("Opening RabbitMQ channel")
self._channel = self._connection.channel()
self._channel.add_close_listener(self._on_channel_closed)
self._channel.queue.declare(self.queue, auto_delete=True)
def _ensure_open(self):
if self._channel is None:
self._open_connection()
self._open_channel()
@exec_or_break
def _read_frames(self):
self._connection.read_frames()
def _read_loop(self):
# The message pump needs to run for the connection to actually do something.
while not self.broken:
self._read_frames()
gevent.sleep() # Yield to other greenlets so they don't starve
def _start_read_loop(self):
self.log_debug("Starting connection loop")
gevent.spawn(self._read_loop) # Power our connection
@exec_or_break
def dispatch(self, key):
self._ensure_open()
self._channel.basic.publish(Message(key), "", self.queue)
@exec_or_break
def consume(self, consumer):
# Start consuming messages from the queue (they will be passed to `consumer`)
def cb():
self.log_debug("Registered as consumer")
def consumer_wrapper(message):
self.log_debug("Received a message: %s", message.body)
consumer(message)
self._ensure_open()
self._channel.basic.consume(queue=self.queue, consumer=consumer_wrapper, cb=cb)
class RabbitConnexion():
"""Manage connexion to Rabbit"""
def __init__(self, LAST_MESSAGES, monitoring=None, watchdog=None):
self.logger = logging.getLogger('radiovisserver.rabbitmq')
# RadioDNS
self.radioDns = RadioDns()
# List of stompservers
self.stompservers = []
# Save LAST_MESSAGES
self.LAST_MESSAGES = LAST_MESSAGES
self.monitoring = monitoring
# Save the watchdog
self.watchdog = watchdog
# The global gauge
# Initialize RadioDNS Caches
self.radioDns.update_channel_topics()
self.cox = None
def consumer(self, msg):
"""Called when a rabbitmq message arrives"""
try:
headers = msg.properties['application_headers']
if 'topic' in headers:
body = msg.body
topic = headers['topic']
bonusHeaders = []
# Get the list of extras headers (eg: link, trigger-time)
for name in headers:
if name == 'topic': # Internal header
continue
bonusHeaders.append((name, headers[name]))
self.logger.info("Got message on topic %s: %s (headers: %s)" % (topic, body, bonusHeaders))
# Save the message as the last one
converted_topic = self.radioDns.convert_fm_topic_to_gcc(topic)
self.LAST_MESSAGES[converted_topic] = (body, bonusHeaders)
# Broadcast message to all clients
for c in self.stompservers:
time.sleep(0) # Switch context
c.new_message(topic, body, bonusHeaders)
# Inform the watchdog
if self.watchdog:
self.watchdog.new_message(topic, body, bonusHeaders, int(headers['when']))
else:
self.logger.warning("Got message without topic: %s" % (msg,))
except Exception as e:
self.logger.error("Error in consumer: %s", (e,))
def run(self):
"""Thread with connection to rabbitmq"""
if config.RABBITMQ_LOOPBACK:
self.logger.warning("Looopback mode: No connection, waiting for ever...")
while True:
time.sleep(1)
while True:
try:
time.sleep(1)
self.logger.debug("Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (
config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST))
self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD,
vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT,
debug=config.RABBITMQ_DEBUG)
self.logger.debug("Creating the channel")
self.ch = self.cox.channel()
# Name will come from a callback
global queue_name
queue_name = None
def queue_qb(queue, msg_count, consumer_count):
self.logger.debug("Created queue %s" % (queue,))
global queue_name
queue_name = queue
self.logger.debug("Creating the queue")
if not self.watchdog:
# 'Normal', tempory queue
self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb)
else:
# Persistant queue, if not in test mode
if len(sys.argv) > 1 and sys.argv[1] == '--test':
self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb)
else:
self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb)
for i in range(0, 10): # Max 10 seconds
if queue_name is None:
time.sleep(1)
if queue_name is None:
self.logger.warning("Queue creation timeout !")
raise Exception("Cannot create queue !")
self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE,))
self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '')
self.logger.debug("Binding the comsumer")
self.ch.basic.consume(queue_name, self.consumer)
self.logger.debug("Ready, waiting for events !")
while True:
if not hasattr(self.ch, 'channel') or (
hasattr(self.ch.channel, '_closed') and self.ch.channel._closed):
self.logger.warning("Channel is closed")
raise Exception("Connexion or channel closed !")
self.cox.read_frames()
except Exception as e:
self.logger.error("Error in run: %s" % (e,))
finally:
if self.cox is not None:
self.cox.close()
def send_message(self, headers, message):
"""Send a message to the queue"""
# Append current ts
headers['when'] = str(int(time.time()))
self.logger.info("Sending message (with headers %s) %s to %s" % (headers, message, config.RABBITMQ_EXCHANGE))
if config.RABBITMQ_LOOPBACK:
self.logger.info("Sending using loopback, calling function directly")
class FalseMsg():
def __init__(self, body, headers):
self.body = body
self.properties = {'application_headers': headers}
self.consumer(FalseMsg(message, headers))
else:
self.ch.basic.publish(Message(message, application_headers=headers), config.RABBITMQ_EXCHANGE, '')
def add_stomp_server(self, s):
"""Handle a new stomp server"""
self.stompservers.append(s)
self.update_stats()
def remove_stomp_server(self, s):
"""Stop handeling a stomp server"""
self.stompservers.remove(s)
self.update_stats()
def update_stats(self):
"""Update stats"""
# self.gauge.send(config.STATS_GAUGE_NB_CLIENTS, len(self.stompservers))
pass
