def __init__(self, **kwargs): properties = { "host": "localhost", "port": 5672, "user": "******", "password": "******", "vhost": "/", "heartbeat": None, "debug": False } self.kwargs = kwargs for key, value in kwargs.iteritems(): if properties.has_key(key): properties[key] = value # Connection can be shared between multiple clients/workers conn = self.kwargs.get("conn", None) if conn is None: conn = Connection(**properties) self._conn = conn self.channel = conn.channel()
class FibonacciRpcClient(object): def __init__(self): self.connection = Connection(host='localhost', heartbeat=None, debug=True) self.channel = self.connection.channel() result = self.channel.queue.declare(exclusive=True) self.callback_queue = result[0] print("callback_queue:", self.callback_queue) self.channel.basic.consume(self.callback_queue, self.on_response, no_ack=True) def on_response(self, msg): if msg.properties["correlation_id"] == self.corr_id: self.response = msg.body def call(self, n): self.response = None self.corr_id = str(uuid.uuid4()) msg = Message(str(n), reply_to=self.callback_queue, correlation_id=self.corr_id) self.channel.basic.publish(msg, '', 'rpc_queue') while self.response is None: self.connection.read_frames() return int(self.response)
def __init__(self): self.connection = Connection(host='localhost', heartbeat=None, debug=True) self.channel = self.connection.channel() result = self.channel.queue.declare(exclusive=True) self.callback_queue = result[0] print("callback_queue:", self.callback_queue) self.channel.basic.consume(self.callback_queue, self.on_response, no_ack=True)
class AsyncClient(object): __metaclass__ = ClientMeta def start(self, **kwargs): self.connection = Connection(**kwargs) self.channel = self.connection.channel() self.loop = TaskLoop() self.insert_task(self.read_frames, interval=0.01) self.insert_task(self.__run) self.loop.start() def __run(self): for declaration in itertools.chain(self._exchanges, self._queues, self._consumers): declaration.client = weakref.ref(self) declaration.declare() # start 'auto' tasks for name, attr in self.__class__.__dict__.iteritems(): if isinstance(attr, Task) and attr.auto: getattr(self, name)() for task, args, kwargs in self._tasks: print 'scheduling %s from _tasks' % task self.insert_task(task, *args, **kwargs) # if there's a run function which isn't already a task, queue it class_run = getattr(self.__class__, 'run', None) self_run = getattr(self, 'run', None) if callable(self_run) and not isinstance(class_run, Task): self.run() self.stop() def stop(self): self.connection.close() def insert_task(self, task, *args, **kwargs): loop = getattr(self, 'loop', None) if isinstance(loop, TaskLoop): loop.insert_task(task, *args, **kwargs) else: self._tasks.append((task, args, kwargs)) def read_frames(self): if self.connection.close_info: self.loop.stop() else: self.connection.read_frames() @task def basic_qos(self, **kwargs): self.channel.basic.qos(**kwargs)
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange, req_routing_key, req_msg_list, send_interval, report_queue, state, codec): assert_is_instance(send_interval, float) assert_greater_equal(send_interval, 0.0) conn, next_log_time, req_num = None, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) pch = conn.channel() report_queue.put((name, 0, ProcessStarted, None)) #waiting for tester set state to 1(means to start test) while StateInit == state.value: time.sleep(0.01) eq_(StateRunning, state.value) req_num, total_sent = 0, 0 start_time = time.time() next_send_time = start_time + send_interval next_log_time = int(start_time) + 1 for app_msg in req_msg_list: if 0 < send_interval: dt = next_send_time - time.time() if dt > 0.0: time.sleep(dt) pch.basic.publish(Message(app_msg), exchange, req_routing_key) next_send_time += send_interval req_num += 1 if time.time() >= next_log_time: report_queue.put( (name, next_log_time, DriverReqReport, (req_num, ))) total_sent += req_num req_num = 0 next_log_time += 1 if StateRunning != state.value: break except KeyboardInterrupt as e: pass finally: total_sent += req_num print name + ': end of loop, total %d msgs sent' % total_sent report_queue.put((name, next_log_time, DriverReqReport, (req_num, ))) report_queue.put( (name, next_log_time, DriverEndNotice, (total_sent, ))) if conn: conn.close()
def start(self, **kwargs): self.connection = Connection(**kwargs) self.channel = self.connection.channel() self.loop = TaskLoop() self.insert_task(self.read_frames, interval=0.01) self.insert_task(self.__run) self.loop.start()
def test_init_without_keyword_args(self): conn = Connection.__new__(Connection) strategy = mock() transport = mock() mock(connection, 'ConnectionChannel') expect(connection.ConnectionChannel).args(conn, 0, {}).returns('connection_channel') expect(socket_transport.SocketTransport).args(conn).returns(transport) expect(conn.connect).args('localhost', 5672) conn.__init__() assert_false(conn._debug) assert_equal(logging.root, conn._logger) assert_equal('guest', conn._user) assert_equal('guest', conn._password) assert_equal('localhost', conn._host) assert_equal(5672, conn._port) assert_equal('/', conn._vhost) assert_equal(5, conn._connect_timeout) assert_equal(None, conn._sock_opts) assert_equal(None, conn._sock) assert_equal(None, conn._heartbeat) assert_equal(None, conn._open_cb) assert_equal(None, conn._close_cb) assert_equal('AMQPLAIN', conn._login_method) assert_equal('en_US', conn._locale) assert_equal(None, conn._client_properties) assert_equal(conn._properties, { 'library': 'Haigha', 'library_version': __version__, }) assert_false(conn._closed) assert_false(conn._connected) assert_equal(conn._close_info, { 'reply_code': 0, 'reply_text': 'first connect', 'class_id': 0, 'method_id': 0 }) assert_equals({ 20: ChannelClass, 40: ExchangeClass, 50: QueueClass, 60: BasicClass, 90: TransactionClass }, conn._class_map) assert_equal({0: 'connection_channel'}, conn._channels) assert_equal('\x05LOGINS\x00\x00\x00\x05guest\x08PASSWORDS\x00\x00\x00\x05guest', conn._login_response) assert_equal(0, conn._channel_counter) assert_equal(65535, conn._channel_max) assert_equal(65535, conn._frame_max) assert_equal([], conn._output_frame_buffer) assert_equal(transport, conn._transport) transport.synchronous = True assert_false(conn._synchronous) assert_true(conn.synchronous) assert_true(conn._synchronous_connect)
def test_init_without_keyword_args(self): conn = Connection.__new__(Connection) strategy = mock() mock(connection, 'ConnectionChannel') expect(connection.ConnectionChannel).args( conn, 0, {}).returns('connection_channel') expect(socket_transport.SocketTransport).args(conn).returns( 'SocketTransport') expect(conn.connect).args('localhost', 5672) conn.__init__() self.assertFalse(conn._debug) self.assertEqual(logging.root, conn._logger) self.assertEqual('guest', conn._user) self.assertEqual('guest', conn._password) self.assertEqual('localhost', conn._host) self.assertEqual(5672, conn._port) self.assertEqual('/', conn._vhost) self.assertEqual(5, conn._connect_timeout) self.assertEqual(None, conn._sock_opts) self.assertEqual(None, conn._sock) self.assertEqual(None, conn._heartbeat) self.assertEqual(None, conn._open_cb) self.assertEqual(None, conn._close_cb) self.assertEqual('AMQPLAIN', conn._login_method) self.assertEqual('en_US', conn._locale) self.assertEqual(None, conn._client_properties) self.assertEqual(conn._properties, { 'library': 'Haigha', 'library_version': __version__, }) self.assertFalse(conn._closed) self.assertFalse(conn._connected) self.assertEqual( conn._close_info, { 'reply_code': 0, 'reply_text': 'first connect', 'class_id': 0, 'method_id': 0 }) assert_equals( { 20: ChannelClass, 40: ExchangeClass, 50: QueueClass, 60: BasicClass, 90: TransactionClass }, conn._class_map) self.assertEqual({0: 'connection_channel'}, conn._channels) self.assertEqual( '\x05LOGINS\x00\x00\x00\x05guest\x08PASSWORDS\x00\x00\x00\x05guest', conn._login_response) self.assertEqual(0, conn._channel_counter) self.assertEqual(65535, conn._channel_max) self.assertEqual(65535, conn._frame_max) self.assertEqual([], conn._output_frame_buffer) self.assertEqual('SocketTransport', conn._transport)
def __clear__(args): bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) counter, timeout_time = 0, time.time()+1 try: ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(queue, durable=durable, auto_delete=False) ch.queue.bind(queue, exchange, routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) while time.time() < timeout_time: msg = ch.basic.get(queue) if msg: timeout_time = time.time() + 1 else: time.sleep(0.01) counter += 1 finally: conn.close()
def test_init_with_event_transport(self): conn = Connection.__new__( Connection ) strategy = mock() mock( connection, 'ConnectionChannel' ) expect(connection.ConnectionChannel).args( conn, 0, {} ).returns( 'connection_channel' ) expect(event_transport.EventTransport).args( conn ).returns( 'EventTransport' ) expect(conn.connect).args( 'localhost', 5672 ) conn.__init__(transport='event')
def driver_main(name, bus_ip, bus_port, vhost, username, password, exchange, req_routing_key, req_msg_list, send_interval, report_queue, state, codec): assert_is_instance(send_interval, float) assert_greater_equal(send_interval, 0.0) conn, next_log_time, req_num = None, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) pch = conn.channel() report_queue.put((name, 0, ProcessStarted, None)) #waiting for tester set state to 1(means to start test) while StateInit == state.value: time.sleep(0.01) eq_(StateRunning, state.value) req_num, total_sent = 0, 0 start_time = time.time() next_send_time = start_time + send_interval next_log_time = int(start_time) + 1 for app_msg in req_msg_list: if 0 < send_interval: dt = next_send_time - time.time() if dt > 0.0: time.sleep(dt) pch.basic.publish(Message(app_msg), exchange, req_routing_key) next_send_time += send_interval req_num += 1 if time.time() >= next_log_time: report_queue.put((name, next_log_time, DriverReqReport, (req_num,))) total_sent += req_num req_num = 0 next_log_time += 1 if StateRunning != state.value: break except KeyboardInterrupt as e: pass finally: total_sent += req_num print name + ': end of loop, total %d msgs sent'%total_sent report_queue.put((name, next_log_time, DriverReqReport, (req_num,))) report_queue.put((name, next_log_time, DriverEndNotice, (total_sent,))) if conn: conn.close()
def test_init_with_event_transport(self): conn = Connection.__new__(Connection) strategy = mock() mock(connection, 'ConnectionChannel') expect(connection.ConnectionChannel).args( conn, 0, {}).returns('connection_channel') expect(event_transport.EventTransport).args(conn).returns( 'EventTransport') expect(conn.connect).args('localhost', 5672) conn.__init__(transport='event')
def _connect_to_broker(self): ''' Connect to broker and regisiter cleanup action to disconnect :returns: connection instance :rtype: `haigha.connection.Connection` ''' sock_opts = { (socket.IPPROTO_TCP, socket.TCP_NODELAY): 1, } connection = Connection(logger=_LOG, debug=_OPTIONS.debug, user=_OPTIONS.user, password=_OPTIONS.password, vhost=_OPTIONS.vhost, host=_OPTIONS.host, heartbeat=None, sock_opts=sock_opts, transport='socket') self.addCleanup(lambda: connection.close(disconnect=True) if not connection.closed else None) return connection
def test_init_without_keyword_args(self): conn = Connection.__new__(Connection) strategy = mock() mock(connection, 'ConnectionChannel') mock(connection, 'ConnectionStrategy') expect(connection.ConnectionChannel).args( conn, 0).returns('connection_channel') expect(connection.ConnectionStrategy).args( conn, 'localhost', reconnect_cb=None).returns(strategy) expect(strategy.connect) conn.__init__() self.assertFalse(conn._debug) self.assertEqual(logging.root, conn._logger) self.assertEqual('guest', conn._user) self.assertEqual('guest', conn._password) self.assertEqual('localhost', conn._host) self.assertEqual('/', conn._vhost) self.assertEqual(5, conn._connect_timeout) self.assertEqual(None, conn._sock_opts) self.assertEqual(None, conn._sock) self.assertEqual(None, conn._heartbeat) self.assertEqual(None, conn._reconnect_cb) self.assertEqual(None, conn._close_cb) self.assertEqual('AMQPLAIN', conn._login_method) self.assertEqual('en_US', conn._locale) self.assertEqual(None, conn._client_properties) self.assertEqual(conn._properties, { 'library': 'Haigha', 'library_version': VERSION, }) self.assertFalse(conn._closed) self.assertFalse(conn._connected) self.assertEqual( conn._close_info, { 'reply_code': 0, 'reply_text': 'first connect', 'class_id': 0, 'method_id': 0 }) self.assertEqual({0: 'connection_channel'}, conn._channels) self.assertEqual( '\x05LOGINS\x00\x00\x00\x05guest\x08PASSWORDS\x00\x00\x00\x05guest', conn._login_response) self.assertEqual(0, conn._channel_counter) self.assertEqual(65535, conn._channel_max) self.assertEqual(65535, conn._frame_max) self.assertEqual(strategy, conn._strategy) self.assertEqual([], conn._output_frame_buffer)
def _connect_to_broker(self): ''' Connect to broker and regisiter cleanup action to disconnect :returns: connection instance :rtype: `haigha.connection.Connection` ''' sock_opts = { (socket.IPPROTO_TCP, socket.TCP_NODELAY) : 1, } connection = Connection( logger=_LOG, debug=_OPTIONS.debug, user=_OPTIONS.user, password=_OPTIONS.password, vhost=_OPTIONS.vhost, host=_OPTIONS.host, heartbeat=None, sock_opts=sock_opts, transport='socket') self.addCleanup(lambda: connection.close(disconnect=True) if not connection.closed else None) return connection
def __clear__(args): bus_ip, bus_port, vhost, username, password, exchange, queue, routing_key, durable = args conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) counter, timeout_time = 0, time.time() + 1 try: ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(queue, durable=durable, auto_delete=False) ch.queue.bind(queue, exchange, routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) while time.time() < timeout_time: msg = ch.basic.get(queue) if msg: timeout_time = time.time() + 1 else: time.sleep(0.01) counter += 1 finally: conn.close()
def test_init_without_keyword_args(self): conn = Connection.__new__( Connection ) strategy = mock() mock( connection, 'ConnectionChannel' ) mock( connection, 'ConnectionStrategy' ) expect(connection.ConnectionChannel).args( conn, 0 ).returns( 'connection_channel' ) expect(connection.ConnectionStrategy).args( conn, 'localhost', reconnect_cb=None ).returns( strategy ) expect(strategy.connect) conn.__init__() self.assertFalse( conn._debug ) self.assertEqual( logging.root, conn._logger ) self.assertEqual( 'guest', conn._user ) self.assertEqual( 'guest', conn._password ) self.assertEqual( 'localhost', conn._host ) self.assertEqual( '/', conn._vhost ) self.assertEqual( 5, conn._connect_timeout ) self.assertEqual( None, conn._sock_opts ) self.assertEqual( None, conn._sock ) self.assertEqual( None, conn._heartbeat ) self.assertEqual( None, conn._reconnect_cb ) self.assertEqual( None, conn._close_cb ) self.assertEqual( 'AMQPLAIN', conn._login_method ) self.assertEqual( 'en_US', conn._locale ) self.assertEqual( None, conn._client_properties ) self.assertEqual( conn._properties, { 'library': 'Haigha', 'library_version': VERSION, } ) self.assertFalse( conn._closed ) self.assertFalse( conn._connected ) self.assertEqual( conn._close_info, { 'reply_code' : 0, 'reply_text' : 'first connect', 'class_id' : 0, 'method_id' : 0 } ) self.assertEqual( {0:'connection_channel'}, conn._channels ) self.assertEqual( '\x05LOGINS\x00\x00\x00\x05guest\x08PASSWORDS\x00\x00\x00\x05guest', conn._login_response ) self.assertEqual( 0, conn._channel_counter ) self.assertEqual( 65535, conn._channel_max ) self.assertEqual( 65535, conn._frame_max ) self.assertEqual( strategy, conn._strategy ) self.assertEqual( None, conn._output_buffer ) self.assertEqual( [], conn._output_frame_buffer )
def setUp(self): super(ConnectionTest,self).setUp() self.connection = Connection.__new__( Connection ) self.connection._debug = False self.connection._logger = self.mock() self.connection._user = '******' self.connection._password = '******' self.connection._host = 'localhost' self.connection._vhost = '/' self.connection._connect_timeout = 5 self.connection._sock_opts = None self.connection._sock = None # mock anything? self.connection._heartbeat = None self.connection._open_cb = self.mock() self.connection._close_cb = self.mock() self.connection._login_method = 'AMQPLAIN' self.connection._locale = 'en_US' self.connection._client_properties = None self.connection._properties = { 'library': 'Haigha', 'library_version': 'x.y.z', } self.connection._closed = False self.connection._connected = False self.connection._close_info = { 'reply_code' : 0, 'reply_text' : 'first connect', 'class_id' : 0, 'method_id' : 0 } self.connection._class_map = {} self.connection._channels = { 0 : self.mock() } self.connection._login_response = 'loginresponse' self.connection._channel_counter = 0 self.connection._channel_max = 65535 self.connection._frame_max = 65535 self.connection._frames_read = 0 self.connection._frames_written = 0 self.connection._strategy = self.mock() self.connection._output_frame_buffer = [] self.connection._transport = mock() self.connection._synchronous = False self.connection._synchronous_connect = False
def setUp(self): super(ConnectionTest, self).setUp() self.connection = Connection.__new__(Connection) self.connection._debug = False self.connection._logger = self.mock() self.connection._user = '******' self.connection._password = '******' self.connection._host = 'localhost' self.connection._vhost = '/' self.connection._connect_timeout = 5 self.connection._sock_opts = None self.connection._sock = None # mock anything? self.connection._heartbeat = None self.connection._open_cb = self.mock() self.connection._close_cb = self.mock() self.connection._login_method = 'AMQPLAIN' self.connection._locale = 'en_US' self.connection._client_properties = None self.connection._properties = { 'library': 'Haigha', 'library_version': 'x.y.z', } self.connection._closed = False self.connection._connected = False self.connection._close_info = { 'reply_code': 0, 'reply_text': 'first connect', 'class_id': 0, 'method_id': 0 } self.connection._class_map = {} self.connection._channels = { 0: self.mock() } self.connection._login_response = 'loginresponse' self.connection._channel_counter = 0 self.connection._channel_max = 65535 self.connection._frame_max = 65535 self.connection._frames_read = 0 self.connection._frames_written = 0 self.connection._strategy = self.mock() self.connection._output_frame_buffer = [] self.connection._transport = mock() self.connection._synchronous = False self.connection._synchronous_connect = False
def start(self): self._connection = Connection( transport='gevent', user='******', password='******', vhost=self.vhost, host=self.queueServer, heartbeat=None, debug=True) self._channel = self._connection.channel() self._channel.add_close_listener(self._channel_closed_cb) # Create and configure message exchange and queue print self.exchange print self.queueName self._channel.exchange.declare(self.exchange, 'topic') self._channel.queue.declare(self.queueName, auto_delete=False) self._channel.queue.bind(self.queueName, self.exchange, self.queueName) self._channel.basic.consume(queue=self.queueName, consumer=self._process_message) # Start message pump self.running = True self._message_pump_greenlet = gevent.spawn(self._message_pump_greenthread) print "Started"
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password, exchange, rsp_queue, rsp_routing_key, report_que, state, codec): conn = None next_log_time = int(time.time()) + 1 rsp_num, err_num, total_num = 0, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(rsp_queue, auto_delete=False) ch.queue.bind(rsp_queue, exchange, rsp_routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) #read all msgs in the consume queue before test i = 0 msg = ch.basic.get(rsp_queue) while msg: msg = ch.basic.get(rsp_queue) i += 1 print name + ': %d msgs read before test'%i report_que.put((name, 0, ConsumerStarted, None)) #waiting for tester set state to 1(means to start test) while StateInit == state.value: time.sleep(0.01) eq_(StateRunning, state.value) next_log_time = int(time.time()) + 1 while StateRunning == state.value: msg = ch.basic.get(rsp_queue) if msg: s = ''.join(map(lambda b: chr(b), msg.body)) if 'heartbeat' == s: session_id, protobuf_data = 0, s rsp_num += 1 else: session_id, protobuf_data = codec.decode(s) if 0 == protobuf_data.RetCode: rsp_num += 1 else: err_num += 1 else: time.sleep(0.01) if time.time() >= next_log_time: report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num))) total_num += rsp_num+err_num rsp_num, err_num = 0, 0 next_log_time += 1 except KeyboardInterrupt as e: pass finally: if conn: timeout_time = time.time() + 1.0 while time.time() < timeout_time: msg = ch.basic.get(rsp_queue) if msg: timeout_time = time.time() + 1.0 s = ''.join(map(lambda b: chr(b), msg.body)) if 'heartbeat' == s: session_id, protobuf_data = -1, s rsp_num += 1 else: session_id, protobuf_data = codec.decode(s) if 0 == protobuf_data.RetCode: rsp_num += 1 else: err_num += 1 conn.close() report_que.put((name, next_log_time, DriverRspReport, (rsp_num, err_num))) total_num += rsp_num+err_num print name + ': end of loop, total %d rsp'%total_num
import sys from haigha.connection import Connection from haigha.message import Message sys.path.append("/home/ec2-user/source/Quant/component") import rdm r = rdm.RedisInfo() connection = Connection(host=r.rmq['hostname'], port=r.rmq['port'], user=r.rmq['user'], password=r.rmq['pass'], vhost='/', heartbeat=None, debug=True) ch = connection.channel() ch.exchange.declare('test_exchange', 'direct') ch.queue.declare('test_queue', auto_delete=True) ch.queue.bind('test_queue', 'test_exchange', 'test_key') ch.basic.publish(Message('body', application_headers={'hello': 'world'}), exchange='test_exchange', routing_key='test_key') mq = ch.basic.get('test_queue') connection.close() print mq
def _open_connection(self): self.log_debug("Opening RabbitMQ connection") self._connection = HaighaConnection(**self._conn_kwargs) self._start_read_loop()
#!/usr/bin/env python ''' demostrate how to write a rpc server ''' import sys, os, uuid, time sys.path.append(os.path.abspath("..")) import time from haigha.connection import Connection from haigha.message import Message connection = Connection(user='******', password='******', vhost='/', host='52.86.70.177', heartbeat=None, debug=True) channel = connection.channel() channel.queue.declare(queue='rpc_queue', auto_delete=False) def fib(n): if n == 0: return 0 elif n == 1: return 1 else: return fib(n - 1) + fib(n - 2) def on_request(msg): print msg
def run(self): """Thread with connection to rabbitmq""" if config.RABBITMQ_LOOPBACK: self.logger.warning("Looopback mode: No connection, waiting for ever...") while True: time.sleep(1) while True: try: time.sleep(1) self.logger.debug("Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % ( config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST)) self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD, vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, debug=config.RABBITMQ_DEBUG) self.logger.debug("Creating the channel") self.ch = self.cox.channel() # Name will come from a callback global queue_name queue_name = None def queue_qb(queue, msg_count, consumer_count): self.logger.debug("Created queue %s" % (queue,)) global queue_name queue_name = queue self.logger.debug("Creating the queue") if not self.watchdog: # 'Normal', tempory queue self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb) else: # Persistant queue, if not in test mode if len(sys.argv) > 1 and sys.argv[1] == '--test': self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb) else: self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE,)) self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '') self.logger.debug("Binding the comsumer") self.ch.basic.consume(queue_name, self.consumer) self.logger.debug("Ready, waiting for events !") while True: if not hasattr(self.ch, 'channel') or ( hasattr(self.ch.channel, '_closed') and self.ch.channel._closed): self.logger.warning("Channel is closed") raise Exception("Connexion or channel closed !") self.cox.read_frames() except Exception as e: self.logger.error("Error in run: %s" % (e,)) finally: if self.cox is not None: self.cox.close()
class AppClient(object): def __init__(self, serverName, **options): self.serverName = serverName self._routes = {} self.options = options self.version = self.options.get("version", "0.1") self.queueName = "{0}/{1}/request".format(self.serverName, self.version) self.reply_queue_name = "{0}/{1}/reply".format(self.serverName, self.version) self.exchange=self.options.get("exchange","main") self.vhost = self.options.get("vhost", "/") self.queueServer = self.options.get("queueServer","localhost") self._pending_requests = {} def route(self,operation): def wrap(func): self.add_route(operation, func) return func return wrap def add_route(self,operation, func): self._routes[operation]=func def _process_reply(self, message): ''' Receives message.body containing following attributes { context: { operation:<operation>, params:<params>, cid:<id for the request> }, content:<content> } ''' logging.debug(" Reply: Getting msg: {0}".format(message)) body = str(message.body).encode('string_escape') if not body: pass body=json.loads(body) context = body["context"] or None if not context: logging.debug("QueueServer:Received invalid message: No context: {0}".format(message)) return cid = context["cid"] callback = self._pending_requests[cid] if not callback: logging.error("No callback registered for cid:",cid) return gevent.spawn(callback, body["content"]) gevent.sleep(0) def send(self, message, options, callback): context = options or {} context["operation"]="get" context["cid"]=1 self._pending_requests[context["cid"]]=callback message = Message(json.dumps({ "context": context, "content" : message }),None, reply_to=self.reply_queue_name) logging.debug("Sending message to {0}|{1}".format(self.exchange,self.queueName, message)) self._channel.basic.publish( message, self.exchange,self.queueName) gevent.sleep(0) def start(self): self._connection = Connection( transport='gevent', user='******', password='******', vhost=self.vhost, host=self.queueServer, heartbeat=None, debug=True) self._channel = self._connection.channel() self._channel.add_close_listener(self._channel_closed_cb) # Create and configure message exchange and queue self._channel.exchange.declare(self.exchange, 'topic') self._channel.queue.declare(self.queueName, auto_delete=False) self._channel.queue.declare(self.reply_queue_name, auto_delete=False) print "reply q name", self.reply_queue_name self._channel.queue.bind(self.reply_queue_name, self.exchange, self.reply_queue_name) # self._channel.basic.consume(queue=self.reply_queue_name, # consumer=self._process_reply) # Start message pump self.running = True gevent.spawn(self._message_pump_greenthread) print "Started" gevent.sleep(0) def _message_pump_greenthread(self): logging.debug("Entering Message Pump") try: while self.running: # Pump self._connection.read_frames() # Yield to other greenlets so they don't starve gevent.sleep() except Exception as e: print e finally: logging.debug("Leaving Message Pump, {0}".format(self.running)) return def stop(self): self.running = False def _channel_closed_cb(self,*args): print "Channel Closed"
def driver_cunsume_func(name, bus_ip, bus_port, vhost, username, password, exchange, rsp_queue, rsp_routing_key, report_que, state, codec): conn = None next_log_time = int(time.time()) + 1 rsp_num, err_num, total_num = 0, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(rsp_queue, auto_delete=False) ch.queue.bind(rsp_queue, exchange, rsp_routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) #read all msgs in the consume queue before test i = 0 msg = ch.basic.get(rsp_queue) while msg: msg = ch.basic.get(rsp_queue) i += 1 print name + ': %d msgs read before test' % i report_que.put((name, 0, ConsumerStarted, None)) #waiting for tester set state to 1(means to start test) while StateInit == state.value: time.sleep(0.01) eq_(StateRunning, state.value) next_log_time = int(time.time()) + 1 while StateRunning == state.value: msg = ch.basic.get(rsp_queue) if msg: s = ''.join(map(lambda b: chr(b), msg.body)) if 'heartbeat' == s: session_id, protobuf_data = 0, s rsp_num += 1 else: session_id, protobuf_data = codec.decode(s) if 0 == protobuf_data.RetCode: rsp_num += 1 else: err_num += 1 else: time.sleep(0.01) if time.time() >= next_log_time: report_que.put( (name, next_log_time, DriverRspReport, (rsp_num, err_num))) total_num += rsp_num + err_num rsp_num, err_num = 0, 0 next_log_time += 1 except KeyboardInterrupt as e: pass finally: if conn: timeout_time = time.time() + 1.0 while time.time() < timeout_time: msg = ch.basic.get(rsp_queue) if msg: timeout_time = time.time() + 1.0 s = ''.join(map(lambda b: chr(b), msg.body)) if 'heartbeat' == s: session_id, protobuf_data = -1, s rsp_num += 1 else: session_id, protobuf_data = codec.decode(s) if 0 == protobuf_data.RetCode: rsp_num += 1 else: err_num += 1 conn.close() report_que.put( (name, next_log_time, DriverRspReport, (rsp_num, err_num))) total_num += rsp_num + err_num print name + ': end of loop, total %d rsp' % total_num
class Connection(object): def __init__(self, host, port, virtualhost, user, password, queue): self.conn_id = hex(int(random.random() * 2**32)) self._conn_kwargs = { "host": host, "port": port, "vhost": virtualhost, "user": user, "password": password, "transport": "gevent", "close_cb": self._on_disconnect, "logger": logger, } self.queue = queue self._connection = None self._channel = None self.lock = Semaphore() self.broken = False def log_debug(self, msg, *args, **kwargs): # Ghetto log handler logger.debug("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs) def log_exception(self, msg, *args, **kwargs): logger.exception("[Conn {0}] {1}".format(self.conn_id, msg), *args, **kwargs) def disconnect(self): self.log_debug("Disconnecting") self.broken = True self._connection.disconnect() def _on_disconnect(self): self.log_debug("Received disconnect") self.broken = True def _on_channel_closed(self, channel): # Scrape the connection if our channel is closed. self.log_debug("Received channel close") self.disconnect() def _open_connection(self): self.log_debug("Opening RabbitMQ connection") self._connection = HaighaConnection(**self._conn_kwargs) self._start_read_loop() def _open_channel(self): # Open a channel and make sure we know if it gets closed self.log_debug("Opening RabbitMQ channel") self._channel = self._connection.channel() self._channel.add_close_listener(self._on_channel_closed) self._channel.queue.declare(self.queue, auto_delete=True) def _ensure_open(self): if self._channel is None: self._open_connection() self._open_channel() @exec_or_break def _read_frames(self): self._connection.read_frames() def _read_loop(self): # The message pump needs to run for the connection to actually do something. while not self.broken: self._read_frames() gevent.sleep() # Yield to other greenlets so they don't starve def _start_read_loop(self): self.log_debug("Starting connection loop") gevent.spawn(self._read_loop) # Power our connection @exec_or_break def dispatch(self, key): self._ensure_open() self._channel.basic.publish(Message(key), "", self.queue) @exec_or_break def consume(self, consumer): # Start consuming messages from the queue (they will be passed to `consumer`) def cb(): self.log_debug("Registered as consumer") def consumer_wrapper(message): self.log_debug("Received a message: %s", message.body) consumer(message) self._ensure_open() self._channel.basic.consume(queue=self.queue, consumer=consumer_wrapper, cb=cb)
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange, consume_queue, req_routing_key, durable, rsp_routing_key, mock_func, report_queue, state, codec): conn, next_report_time = None, int(time.time()) + 1 req_num, rsp_num, err_num = 0, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(consume_queue, durable=durable, auto_delete=False) ch.queue.bind(consume_queue, exchange, req_routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) i = 0 while True: msg = ch.basic.get() if None == msg: break i += 1 print 'Consumer of %s read %d msgs before test' % (name, i) report_queue.put((name, 0, ProcessStarted, None)) while StateInit == state.value: time.sleep(0.01) assert_in(state.value, (StateRunning, StateStop)) while StateRunning == state.value: msg = ch.basic.get() if msg: req_num += 1 if 'heartbeat' == msg.body: session_id, protobuf_data = 0, body else: session_id, protobuf_data = self.codec.decode(msg.body) rsp_data = mock_func(session_id, protobuf_data) if rsp_data: rsp_msg = codec.encode(session_id, rsp_data) ch.basic.publish(rsp_msg, exchange, routing_key) else: time.sleep(0.01) if time.time() >= next_report_time: report_queue.put( (pname, next_report_time, MockReport, (req_num, err_num))) req_num, err_num = 0, 0 next_report_time += 1 #print pname + ' report_queue.put() counter=%d'%req_num logging.debug(name + ': end of loop') except KeyboardInterrupt as e: pass finally: if conn: timeout_time = time.time() + 1 while time.time() < timeout_time: msg = ch.basic.get() if msg: timeout_time = time.time() + 1 req_num += 1 if 'heartbeat' == msg.body: session_id, protobuf_data = 0, body else: session_id, protobuf_data = self.codec.decode(msg.body) rsp_data = mock_func(session_id, protobuf_data) if rsp_data: rsp_msg = codec.encode(session_id, rsp_data) ch.basic.publish(rsp_msg, exchange, routing_key) report_queue.put( (pname, next_report_time, MockReport, (req_num, err_num))) conn.close() logging.debug('End of ' + name)
def run(self): """Thread with connection to send monitoring data""" if not self.isEnabled(): self.logger.info("Monitoring disabled by config.py") return else: self.logger.info("Monitoring enabled by config.py") while True: try: time.sleep(1) self.logger.debug( "Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.monitoring['user'], config.monitoring['host'], config.monitoring['port'], config.monitoring['vhost'])) self.connection = Connection(user=config.monitoring['user'], password=config.monitoring['password'], vhost=config.monitoring['vhost'], host=config.monitoring['host'], port=config.monitoring['port'], debug=config.monitoring['debug']) self.logger.debug("Creating the channel") self.ch = self.connection.channel() # Name will come from a callback global queue_name queue_name = None queue_name = config.monitoring['queue'] self.logger.debug("Creating the queue") self.ch.exchange.declare(config.monitoring['exchange'], 'fanout') self.ch.queue.declare(queue=queue_name, durable=True, auto_delete=False, nowait=False) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.monitoring['exchange'],)) self.ch.queue.bind(queue_name, config.monitoring['exchange'], '') self.logger.debug("Ready, waiting for monitoring data") while True: data = json.dumps(self.buffer.get(block=True, timeout=None)) headers = {} headers['when'] = str(int(time.time())) self.ch.basic.publish(Message(data, application_headers=headers), config.monitoring['exchange'], '') except Exception as e: self.logger.error("Error in run: %s" % (e,)) self.logger.error('Monitoring connection failed. Reconnecting in 30sec') finally: if self.connection: self.logger.info('connection closed') self.connection.close() time.sleep(30)
class Monitoring(): """This class manages the connection to the monitoring system""" def __init__(self): self.logger = logging.getLogger('radiovisserver.monitoring') self.buffer = Queue.Queue() self.enabled = self.isEnabled() self.connection = None self.server_id = config.monitoring['server_id'] @skipIfDisabled def log(self, label, content, client='', metadata='', timestamp=None): """ Log a single event """ if timestamp == None: timestamp = time.time() # self.buffer.put({ 'type': 'log', 'label': label, 'content': content, 'client': client, 'metadata': metadata, 'timestamp': timestamp, 'server': self.server_id }) @skipIfDisabled def count(self, label, metadata='', timestamp=None): """ Log a single event """ if timestamp == None: timestamp = time.time() self.buffer.put( {'type': 'log', 'label': label, 'timestamp': timestamp, 'server': self.server_id, 'metadata': metadata}) @skipIfDisabled def gauge(self, label, level, metadata='', timestamp=None): """ Log an aggregated value """ if timestamp == None: timestamp = time.time() self.buffer.put( {'type': 'gauge', 'label': label, 'level': level, 'timestamp': timestamp, 'server': self.server_id}) def run(self): """Thread with connection to send monitoring data""" if not self.isEnabled(): self.logger.info("Monitoring disabled by config.py") return else: self.logger.info("Monitoring enabled by config.py") while True: try: time.sleep(1) self.logger.debug( "Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.monitoring['user'], config.monitoring['host'], config.monitoring['port'], config.monitoring['vhost'])) self.connection = Connection(user=config.monitoring['user'], password=config.monitoring['password'], vhost=config.monitoring['vhost'], host=config.monitoring['host'], port=config.monitoring['port'], debug=config.monitoring['debug']) self.logger.debug("Creating the channel") self.ch = self.connection.channel() # Name will come from a callback global queue_name queue_name = None queue_name = config.monitoring['queue'] self.logger.debug("Creating the queue") self.ch.exchange.declare(config.monitoring['exchange'], 'fanout') self.ch.queue.declare(queue=queue_name, durable=True, auto_delete=False, nowait=False) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.monitoring['exchange'],)) self.ch.queue.bind(queue_name, config.monitoring['exchange'], '') self.logger.debug("Ready, waiting for monitoring data") while True: data = json.dumps(self.buffer.get(block=True, timeout=None)) headers = {} headers['when'] = str(int(time.time())) self.ch.basic.publish(Message(data, application_headers=headers), config.monitoring['exchange'], '') except Exception as e: self.logger.error("Error in run: %s" % (e,)) self.logger.error('Monitoring connection failed. Reconnecting in 30sec') finally: if self.connection: self.logger.info('connection closed') self.connection.close() time.sleep(30) def isEnabled(self): return hasattr(config, 'monitoring') and config.monitoring['enabled']
import time from haigha.connection import Connection from haigha.message import Message connection = Connection(user='******', password='******', vhost='/', host='127.0.0.1', heartbeat=None, debug=True) ch = connection.channel() ch.exchange.declare('test_exchange', 'direct') ch.queue.declare('test_queue', auto_delete=False) ch.queue.bind('test_queue', 'test_exchange', 'test_key') while True: message = Message('body', application_headers={'hello': 'world'}) ch.basic.publish(message, 'test_exchange', 'test_key') time.sleep(1)
from haigha.connection import Connection from haigha.message import Message connection = Connection(user='******', password='******', vhost='/', host='192.168.59.103', heartbeat=None, debug=True) ch = connection.channel() ch.exchange.declare('test_exchange', 'direct') ch.queue.declare('test_queue', auto_delete=True) ch.queue.bind('test_queue', 'test_exchange', 'test_key') for i in xrange(30000): ch.basic.publish(Message('body', application_headers={'hello': 'world'}), 'test_exchange', 'test_key') connection.close()
#!/usr/bin/env python ''' demostrate how to write a rpc server ''' import sys, os, uuid, time sys.path.append(os.path.abspath("..")) from haigha.connection import Connection from haigha.message import Message connection = Connection(host='localhost', heartbeat=None, debug=True) channel = connection.channel() channel.queue.declare(queue='rpc_queue', auto_delete=False) def fib(n): if n == 0: return 0 elif n == 1: return 1 else: return fib(n - 1) + fib(n - 2) def on_request(msg): n = int(msg.body) print " [.] fib(%s)" % (n, ) result = fib(n) reply_to = msg.properties["reply_to"]
def githash(filename): with open(filename,'r') as f: data=f.readlines() data="".join(data) s = sha1() s.update("blob %u\0" % len(data)) s.update(data) return s.hexdigest() classifierTags = predict.CreateClassifier(SETTINGS_FILE_TAGS) classifierNN = predict.CreateClassifier(SETTINGS_FILE_EMBEDDING) print "Creating caffe client" connection = Connection( user='******', password='******', vhost='/', host='52.86.70.177', heartbeat=None, debug=True ) channel = connection.channel() channel.queue.declare(queue='rpc_queue', auto_delete=False) def computeNN(imageURL): filename=''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(10))+'.jpg' filename='/home/ubuntu/caffe-cvprw15/examples/deepFashion/tmp/'+filename if os.path.isfile(filename): os.remove(filename) urllib.urlretrieve(imageURL, filename) imageHash=githash(filename)
def run(self): """Thread with connection to rabbitmq""" if config.RABBITMQ_LOOPBACK: self.logger.warning( "Looopback mode: No connection, waiting for ever...") while True: time.sleep(1) while True: try: time.sleep(1) self.logger.debug( "Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST)) self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD, vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, debug=config.RABBITMQ_DEBUG) self.logger.debug("Creating the channel") self.ch = self.cox.channel() # Name will come from a callback global queue_name queue_name = None def queue_qb(queue, msg_count, consumer_count): self.logger.debug("Created queue %s" % (queue, )) global queue_name queue_name = queue self.logger.debug("Creating the queue") if not self.watchdog: # 'Normal', tempory queue self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb) else: # Persistant queue, if not in test mode if len(sys.argv) > 1 and sys.argv[1] == '--test': self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb) else: self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE, )) self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '') self.logger.debug("Binding the comsumer") self.ch.basic.consume(queue_name, self.consumer) self.logger.debug("Ready, waiting for events !") while True: if not hasattr(self.ch, 'channel') or ( hasattr(self.ch.channel, '_closed') and self.ch.channel._closed): self.logger.warning("Channel is closed") raise Exception("Connexion or channel closed !") self.cox.read_frames() except Exception as e: self.logger.error("Error in run: %s" % (e, )) finally: self.cox.close()
def mock_main(name, bus_ip, bus_port, vhost, username, password, exchange, consume_queue, req_routing_key, durable, rsp_routing_key, mock_func, report_queue, state, codec): conn, next_report_time = None, int(time.time()) +1 req_num, rsp_num, err_num = 0, 0, 0 try: conn = Connection(user=username, password=password, host=bus_ip, port=bus_port, vhost=vhost) ch = conn.channel() ch.exchange.declare(exchange, 'direct') ch.queue.declare(consume_queue, durable=durable, auto_delete=False) ch.queue.bind(consume_queue, exchange, req_routing_key) ch.basic.qos(prefetch_size=0, prefetch_count=1000) i = 0 while True: msg = ch.basic.get() if None == msg: break i += 1 print 'Consumer of %s read %d msgs before test'%(name, i) report_queue.put((name, 0, ProcessStarted, None)) while StateInit == state.value: time.sleep(0.01) assert_in(state.value, (StateRunning, StateStop)) while StateRunning == state.value: msg = ch.basic.get() if msg: req_num += 1 if 'heartbeat' == msg.body: session_id, protobuf_data = 0, body else: session_id, protobuf_data = self.codec.decode(msg.body) rsp_data = mock_func(session_id, protobuf_data) if rsp_data: rsp_msg = codec.encode(session_id, rsp_data) ch.basic.publish(rsp_msg, exchange, routing_key) else: time.sleep(0.01) if time.time() >= next_report_time: report_queue.put((pname, next_report_time, MockReport, (req_num, err_num))) req_num, err_num = 0, 0 next_report_time += 1 #print pname + ' report_queue.put() counter=%d'%req_num logging.debug(name + ': end of loop') except KeyboardInterrupt as e: pass finally: if conn: timeout_time = time.time() + 1 while time.time() < timeout_time: msg = ch.basic.get() if msg: timeout_time = time.time() + 1 req_num += 1 if 'heartbeat' == msg.body: session_id, protobuf_data = 0, body else: session_id, protobuf_data = self.codec.decode(msg.body) rsp_data = mock_func(session_id, protobuf_data) if rsp_data: rsp_msg = codec.encode(session_id, rsp_data) ch.basic.publish(rsp_msg, exchange, routing_key) report_queue.put((pname, next_report_time, MockReport, (req_num, err_num))) conn.close() logging.debug('End of ' + name)
import time from haigha.connection import Connection from haigha.message import Message connection = Connection( user='******', password='******', vhost='/', host='33.33.33.10', heartbeat=None, debug=True) ch = connection.channel() ch.exchange.declare('test_exchange', 'direct') ch.queue.declare('test_queue', auto_delete=False) ch.queue.bind('test_queue', 'test_exchange', 'test_key') while True: message = Message('body', application_headers={'hello': 'world'}) ch.basic.publish(message, 'test_exchange', 'test_key') time.sleep(1)
class RabbitConnexion(): """Manage connexion to Rabbit""" def __init__(self, LAST_MESSAGES, monitoring=None, watchdog=None): self.logger = logging.getLogger('radiovisserver.rabbitmq') # RadioDNS self.radioDns = RadioDns() # List of stompservers self.stompservers = [] # Save LAST_MESSAGES self.LAST_MESSAGES = LAST_MESSAGES self.monitoring = monitoring # Save the watchdog self.watchdog = watchdog # The global gauge # Initialize RadioDNS Caches self.radioDns.update_channel_topics() self.cox = None def consumer(self, msg): """Called when a rabbitmq message arrives""" try: headers = msg.properties['application_headers'] if 'topic' in headers: body = msg.body topic = headers['topic'] bonusHeaders = [] # Get the list of extras headers (eg: link, trigger-time) for name in headers: if name == 'topic': # Internal header continue bonusHeaders.append((name, headers[name])) self.logger.info("Got message on topic %s: %s (headers: %s)" % (topic, body, bonusHeaders)) # Save the message as the last one converted_topic = self.radioDns.convert_fm_topic_to_gcc(topic) self.LAST_MESSAGES[converted_topic] = (body, bonusHeaders) # Broadcast message to all clients for c in self.stompservers: time.sleep(0) # Switch context c.new_message(topic, body, bonusHeaders) # Inform the watchdog if self.watchdog: self.watchdog.new_message(topic, body, bonusHeaders, int(headers['when'])) else: self.logger.warning("Got message without topic: %s" % (msg,)) except Exception as e: self.logger.error("Error in consumer: %s", (e,)) def run(self): """Thread with connection to rabbitmq""" if config.RABBITMQ_LOOPBACK: self.logger.warning("Looopback mode: No connection, waiting for ever...") while True: time.sleep(1) while True: try: time.sleep(1) self.logger.debug("Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % ( config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST)) self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD, vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, debug=config.RABBITMQ_DEBUG) self.logger.debug("Creating the channel") self.ch = self.cox.channel() # Name will come from a callback global queue_name queue_name = None def queue_qb(queue, msg_count, consumer_count): self.logger.debug("Created queue %s" % (queue,)) global queue_name queue_name = queue self.logger.debug("Creating the queue") if not self.watchdog: # 'Normal', tempory queue self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb) else: # Persistant queue, if not in test mode if len(sys.argv) > 1 and sys.argv[1] == '--test': self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb) else: self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE,)) self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '') self.logger.debug("Binding the comsumer") self.ch.basic.consume(queue_name, self.consumer) self.logger.debug("Ready, waiting for events !") while True: if not hasattr(self.ch, 'channel') or ( hasattr(self.ch.channel, '_closed') and self.ch.channel._closed): self.logger.warning("Channel is closed") raise Exception("Connexion or channel closed !") self.cox.read_frames() except Exception as e: self.logger.error("Error in run: %s" % (e,)) finally: if self.cox is not None: self.cox.close() def send_message(self, headers, message): """Send a message to the queue""" # Append current ts headers['when'] = str(int(time.time())) self.logger.info("Sending message (with headers %s) %s to %s" % (headers, message, config.RABBITMQ_EXCHANGE)) if config.RABBITMQ_LOOPBACK: self.logger.info("Sending using loopback, calling function directly") class FalseMsg(): def __init__(self, body, headers): self.body = body self.properties = {'application_headers': headers} self.consumer(FalseMsg(message, headers)) else: self.ch.basic.publish(Message(message, application_headers=headers), config.RABBITMQ_EXCHANGE, '') def add_stomp_server(self, s): """Handle a new stomp server""" self.stompservers.append(s) self.update_stats() def remove_stomp_server(self, s): """Stop handeling a stomp server""" self.stompservers.remove(s) self.update_stats() def update_stats(self): """Update stats""" # self.gauge.send(config.STATS_GAUGE_NB_CLIENTS, len(self.stompservers)) pass
import sys from haigha.connection import Connection from haigha.message import Message sys.path.append("/home/ec2-user/source/Quant/component") import rdm r = rdm.RedisInfo() connection = Connection( host=r.rmq['hostname'], port=r.rmq['port'], user=r.rmq['user'], password=r.rmq['pass'], vhost='/', heartbeat=None, debug=True) ch = connection.channel() ch.exchange.declare('test_exchange', 'direct') ch.queue.declare('test_queue', auto_delete=True) ch.queue.bind('test_queue', 'test_exchange', 'test_key') ch.basic.publish( Message('body', application_headers={'hello':'world'}), exchange='test_exchange', routing_key='test_key' ) mq = ch.basic.get('test_queue') connection.close() print mq
class App(object): def __init__(self, serverName, **options): self.serverName = serverName self._routes = {} self.options = options self.version = self.options.get("version", "0.1") self.queueName = "{0}/{1}/request".format(self.serverName, self.version) self.exchange=self.options.get("exchange","main") self.vhost = self.options.get("vhost", "/") self.queueServer = self.options.get("queueServer","localhost") self._pending_requests = {} def route(self,operation): def wrap(func): self.add_route(operation, func) return func return wrap def add_route(self,operation, func): self._routes[operation]=func def get_route(self,operation): return self._routes.get(operation,None) def _process_message(self,message): ''' Receives message.body containing following attributes { context: { operation:<operation>, params:<params>, cid:<id for the request> }, content:<content> } ''' print " getting msg", message body = str(message.body).encode('string_escape') if not body: pass body=json.loads(body) context = body["context"] or None if not context: logging.debug("QueueServer:Received invalid message: No context: {0}".format(message)) return operation = context.get("operation",None) print "\nOperation is ", operation if not operation: pass func = self.get_route(operation); if(not func): pass reply_to = message.properties.get("reply_to",None) if not reply_to: return self._pending_requests[context["cid"]] = context self._pending_requests[context["cid"]]["reply_to"]=reply_to ret = func({ "context": context, "content": body["content"] }) # print "message properties ", message.properties # self.send_reply(reply_to, ret) def send_reply(self, message): ''' message shoud be : { context:{ reply_to_cid:<unique id> }, content:<content> } ''' reply_to_cid = message["reply_to_cid"] or None if not reply_to_cid: return reply_context = self._pending_requests[reply_to_cid] or None if not reply_context: logging.error("No pending request for cid", reply_to_cid) return self.send_reply(reply_context["reply_to"], { "context":reply_context, "content":message["content"] }) def _send_reply(self, reply_key, ret): msg = Message(ret) self._channel.basic.publish(msg, self.exchange, reply_key) def _message_pump_greenthread(self): print "Entering Message Pump" try: while self.running: # Pump self._connection.read_frames() # Yield to other greenlets so they don't starve gevent.sleep() except Exception as e: print e finally: print "Leaving Message Pump", self.running #self._done_cb() return def start(self): self._connection = Connection( transport='gevent', user='******', password='******', vhost=self.vhost, host=self.queueServer, heartbeat=None, debug=True) self._channel = self._connection.channel() self._channel.add_close_listener(self._channel_closed_cb) # Create and configure message exchange and queue print self.exchange print self.queueName self._channel.exchange.declare(self.exchange, 'topic') self._channel.queue.declare(self.queueName, auto_delete=False) self._channel.queue.bind(self.queueName, self.exchange, self.queueName) self._channel.basic.consume(queue=self.queueName, consumer=self._process_message) # Start message pump self.running = True self._message_pump_greenlet = gevent.spawn(self._message_pump_greenthread) print "Started" def stop(self): self.running = False def _channel_closed_cb(*args): print "Channel Closed", args
class RabbitConnexion(): """Manage connexion to Rabbit""" def __init__(self, LAST_MESSAGES, watchdog=None): self.logger = logging.getLogger('radiovisserver.rabbitmq') # List of stompservers self.stompservers = [] # Save LAST_MESSAGES self.LAST_MESSAGES = LAST_MESSAGES # Save the watchdog self.watchdog = watchdog def consumer(self, msg): """Called when a rabbitmq message arrive""" try: headers = msg.properties['application_headers'] if 'topic' in headers: body = msg.body topic = headers['topic'] bonusHeaders = [] # Get the list of extras headers (eg: link, trigger-time) for name in headers: if name == 'topic': #Internal header continue bonusHeaders.append((name, headers[name])) self.logger.info("Got message on topic %s: %s (headers: %s)" % (topic, body, bonusHeaders)) # Save the message as the last one self.LAST_MESSAGES[topic] = (body, bonusHeaders) # Broadcast message to all clients for c in self.stompservers: c.new_message(topic, body, bonusHeaders) # Inform the watchdog if self.watchdog: self.watchdog.new_message(topic, body, bonusHeaders, int(headers['when'])) else: self.logger.warning("Got message without topic: %s" % (msg, )) except Exception as e: self.logger.error("Error in consumer: %s", (e, )) def run(self): """Thread with connection to rabbitmq""" if config.RABBITMQ_LOOPBACK: self.logger.warning( "Looopback mode: No connection, waiting for ever...") while True: time.sleep(1) while True: try: time.sleep(1) self.logger.debug( "Connecting to RabbitMQ (user=%s,host=%s,port=%s,vhost=%s)" % (config.RABBITMQ_USER, config.RABBITMQ_HOST, config.RABBITMQ_PORT, config.RABBITMQ_VHOST)) self.cox = Connection(user=config.RABBITMQ_USER, password=config.RABBITMQ_PASSWORD, vhost=config.RABBITMQ_VHOST, host=config.RABBITMQ_HOST, port=config.RABBITMQ_PORT, debug=config.RABBITMQ_DEBUG) self.logger.debug("Creating the channel") self.ch = self.cox.channel() # Name will come from a callback global queue_name queue_name = None def queue_qb(queue, msg_count, consumer_count): self.logger.debug("Created queue %s" % (queue, )) global queue_name queue_name = queue self.logger.debug("Creating the queue") if not self.watchdog: # 'Normal', tempory queue self.ch.queue.declare(auto_delete=True, nowait=False, cb=queue_qb) else: # Persistant queue, if not in test mode if len(sys.argv) > 1 and sys.argv[1] == '--test': self.ch.queue.declare(config.FB_QUEUE, auto_delete=True, nowait=False, cb=queue_qb) else: self.ch.queue.declare(config.FB_QUEUE, auto_delete=False, nowait=False, cb=queue_qb) for i in range(0, 10): # Max 10 seconds if queue_name is None: time.sleep(1) if queue_name is None: self.logger.warning("Queue creation timeout !") raise Exception("Cannot create queue !") self.logger.debug("Binding the exchange %s" % (config.RABBITMQ_EXCHANGE, )) self.ch.queue.bind(queue_name, config.RABBITMQ_EXCHANGE, '') self.logger.debug("Binding the comsumer") self.ch.basic.consume(queue_name, self.consumer) self.logger.debug("Ready, waiting for events !") while True: if not hasattr(self.ch, 'channel') or ( hasattr(self.ch.channel, '_closed') and self.ch.channel._closed): self.logger.warning("Channel is closed") raise Exception("Connexion or channel closed !") self.cox.read_frames() except Exception as e: self.logger.error("Error in run: %s" % (e, )) finally: self.cox.close() def send_message(self, headers, message): """Send a message to the queue""" # Append current ts headers['when'] = str(int(time.time())) self.logger.info("Sending message (with headers %s) %s to %s" % (headers, message, config.RABBITMQ_EXCHANGE)) if config.RABBITMQ_LOOPBACK: self.logger.info( "Sending using loopback, calling function directly") class FalseMsg(): def __init__(self, body, headers): self.body = body self.properties = {'application_headers': headers} self.consumer(FalseMsg(message, headers)) else: self.ch.basic.publish( Message(message, application_headers=headers), config.RABBITMQ_EXCHANGE, '') def add_stomp_server(self, s): """Handle a new stomp server""" self.stompservers.append(s) def remove_stomp_server(self, s): """Stop handeling a stomp server""" self.stompservers.remove(s)