class Producer(object): """ Defines the base Rabbit Producer class """ def __init__(self, config): self.connection_info = { 'hostname': config['host'], 'port': int(config['port']), 'userid': config['username'], 'password': config['password'], 'virtual_host': config['vhost'] } self.connection = BrokerConnection( transport=Transport, **self.connection_info ) self.connection.ensure_connection(self.connection_error) self.exchange = Exchange(name=config['exchange'], channel=self.connection) self.route = config.get('route') self._producer = messaging.Producer(self.connection, exchange=self.exchange, routing_key=self.route, auto_declare=False) def connection_error(self, exc, interval): """Defines action on connection error """ if interval > 10: raise IOError("Bad connection to rabbit") else: logging.error("rabbitmq connection error, retry in %s seconds. error: %s", interval, exc.strerror if hasattr(exc, 'strerror') else exc) def publish(self, payload, route=None): """Defines the publish action Args: payload: a message payload route: a message route """ if not route: route = self.route logging.info("Published %s to %s with a route of %s", json.dumps(payload), self.exchange, route) return self._producer.publish(payload, route=route)
class Consumer(object): """ Defines the base Rabbit Consumer class """ def __init__(self, config): self.connection_info = { 'hostname': config['host'], 'port': int(config['port']), 'userid': config['username'], 'password': config['password'], 'virtual_host': config['vhost'] } self.connection = BrokerConnection( transport=Transport, **self.connection_info ) self.connection.ensure_connection(self.connection_error) self.exchange = Exchange(name=config['exchange'], channel=self.connection) if isinstance(config['queue_name'], list): self.queues = [Queue(name, exchange=self.exchange) for name in config['queue_name']] else: self.queues = [Queue(config['queue_name'], exchange=self.exchange)] self.callbacks = [] self.add_callback(self.on_message) self.prefetch_count = 1 def consume(self): """Consumes all messages on the queues """ self.callbacks.append(self.ack_message) c = messaging.Consumer(self.connection, queues=self.queues, callbacks=self.callbacks, auto_declare=False) c.qos(prefetch_count=self.prefetch_count) with c: for _ in eventloop(self.connection, timeout=10, ignore_timeouts=True): pass def add_callback(self, callback): """Registers a new callback to be called upon message receipt Args: callback: a function that accepts body, message as args """ def _build_body(body): """Helper method to build body""" if isinstance(body, dict): return body try: json_data = json.loads(body) if "path" not in json_data: json_data["path"] = [None, None] return json_data except Exception as e: logging.error("Invalid JSON. Exception: %s", e) return {"invalid_json": str(body), 'path': [None, None]} self.callbacks.append(lambda body, message: callback(_build_body(body), message)) def add_queue(self, queue_name): """Adds a queue to the consumer Args: queue_name: name of queue to be bound to consumer """ self.queues.append(Queue(queue_name, exchange=self.exchange)) def connection_error(self, exc, interval): """Defines action on connection error """ if interval > 10: raise IOError("Bad connection to rabbit") else: logging.error("rabbitmq connection error, retry in %s seconds. error: %s", interval, exc.strerror if hasattr(exc, 'strerror') else exc) def on_message(self, body, _): """Defines action to take on receipt of message Args: body: a decoded message body message: the message object """ logging.info("Recieved message %s from one of the following:%s", json.dumps(body), self.queues) def ack_message(self, body, message): """Defines acking the message after processing Args: body: a decoded message body message: the message object """ logging.info("Acking message %s", json.dumps(body)) message.ack()
class test_Connection(unittest.TestCase): def setUp(self): self.conn = BrokerConnection(port=5672, transport=Transport) def test_establish_connection(self): conn = self.conn conn.connect() self.assertTrue(conn.connection.connected) self.assertEqual(conn.host, "localhost:5672") channel = conn.channel() self.assertTrue(channel.open) self.assertEqual(conn.drain_events(), "event") _connection = conn.connection conn.close() self.assertFalse(_connection.connected) self.assertIsInstance(conn.transport, Transport) def test__enter____exit__(self): conn = self.conn context = conn.__enter__() self.assertIs(context, conn) conn.connect() self.assertTrue(conn.connection.connected) conn.__exit__() self.assertIsNone(conn.connection) conn.close() # again def test_close_survives_connerror(self): class _CustomError(Exception): pass class MyTransport(Transport): connection_errors = (_CustomError, ) def close_connection(self, connection): raise _CustomError("foo") conn = BrokerConnection(transport=MyTransport) conn.connect() conn.close() self.assertTrue(conn._closed) def test_ensure_connection(self): self.assertTrue(self.conn.ensure_connection()) def test_SimpleQueue(self): conn = self.conn q = conn.SimpleQueue("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleQueue("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_SimpleBuffer(self): conn = self.conn q = conn.SimpleBuffer("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleBuffer("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test__repr__(self): self.assertTrue(repr(self.conn)) def test__reduce__(self): x = pickle.loads(pickle.dumps(self.conn)) self.assertDictEqual(x.info(), self.conn.info()) def test_channel_errors(self): class MyTransport(Transport): channel_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.channel_errors, (KeyError, ValueError)) def test_connection_errors(self): class MyTransport(Transport): connection_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.connection_errors, (KeyError, ValueError))
class test_Connection(unittest.TestCase): def setUp(self): self.conn = BrokerConnection(port=5672, transport=Transport) def test_establish_connection(self): conn = self.conn conn.connect() self.assertTrue(conn.connection.connected) self.assertEqual(conn.host, "localhost:5672") channel = conn.channel() self.assertTrue(channel.open) self.assertEqual(conn.drain_events(), "event") _connection = conn.connection conn.close() self.assertFalse(_connection.connected) self.assertIsInstance(conn.transport, Transport) def test__enter____exit__(self): conn = self.conn context = conn.__enter__() self.assertIs(context, conn) conn.connect() self.assertTrue(conn.connection.connected) conn.__exit__() self.assertIsNone(conn.connection) conn.close() # again def test_close_survives_connerror(self): class _CustomError(Exception): pass class MyTransport(Transport): connection_errors = (_CustomError, ) def close_connection(self, connection): raise _CustomError("foo") conn = BrokerConnection(transport=MyTransport) conn.connect() conn.close() self.assertTrue(conn._closed) def test_ensure_connection(self): self.assertTrue(self.conn.ensure_connection()) def test_SimpleQueue(self): conn = self.conn q = conn.SimpleQueue("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleQueue("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_SimpleBuffer(self): conn = self.conn q = conn.SimpleBuffer("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleBuffer("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test__repr__(self): self.assertTrue(repr(self.conn)) def test__reduce__(self): x = pickle.loads(pickle.dumps(self.conn)) self.assertDictEqual(x.info(), self.conn.info()) def test_channel_errors(self): class MyTransport(Transport): channel_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.channel_errors, (KeyError, ValueError)) def test_connection_errors(self): class MyTransport(Transport): connection_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.connection_errors, (KeyError, ValueError))
class test_Connection(TestCase): def setUp(self): self.conn = BrokerConnection(port=5672, transport=Transport) def test_establish_connection(self): conn = self.conn conn.connect() self.assertTrue(conn.connection.connected) self.assertEqual(conn.host, "localhost:5672") channel = conn.channel() self.assertTrue(channel.open) self.assertEqual(conn.drain_events(), "event") _connection = conn.connection conn.close() self.assertFalse(_connection.connected) self.assertIsInstance(conn.transport, Transport) def test__enter____exit__(self): conn = self.conn context = conn.__enter__() self.assertIs(context, conn) conn.connect() self.assertTrue(conn.connection.connected) conn.__exit__() self.assertIsNone(conn.connection) conn.close() # again def test_close_survives_connerror(self): class _CustomError(Exception): pass class MyTransport(Transport): connection_errors = (_CustomError, ) def close_connection(self, connection): raise _CustomError("foo") conn = BrokerConnection(transport=MyTransport) conn.connect() conn.close() self.assertTrue(conn._closed) def test_close_when_default_channel(self): conn = self.conn conn._default_channel = Mock() conn._close() conn._default_channel.close.assert_called_with() def test_close_when_default_channel_close_raises(self): class Conn(BrokerConnection): @property def connection_errors(self): return (KeyError, ) conn = Conn("memory://") conn._default_channel = Mock() conn._default_channel.close.side_effect = KeyError() conn._close() conn._default_channel.close.assert_called_with() def test_revive_when_default_channel(self): conn = self.conn defchan = conn._default_channel = Mock() conn.revive(Mock()) defchan.close.assert_called_with() self.assertIsNone(conn._default_channel) def test_ensure_connection(self): self.assertTrue(self.conn.ensure_connection()) def test_ensure_success(self): def publish(): return "foobar" ensured = self.conn.ensure(None, publish) self.assertEqual(ensured(), "foobar") def test_ensure_failure(self): class _CustomError(Exception): pass def publish(): raise _CustomError("bar") ensured = self.conn.ensure(None, publish) with self.assertRaises(_CustomError): ensured() def test_ensure_connection_failure(self): class _ConnectionError(Exception): pass def publish(): raise _ConnectionError("failed connection") self.conn.transport.connection_errors = (_ConnectionError,) ensured = self.conn.ensure(self.conn, publish) with self.assertRaises(_ConnectionError): ensured() def test_autoretry(self): myfun = Mock() myfun.__name__ = "test_autoretry" self.conn.transport.connection_errors = (KeyError, ) def on_call(*args, **kwargs): myfun.side_effect = None raise KeyError("foo") myfun.side_effect = on_call insured = self.conn.autoretry(myfun) insured() self.assertTrue(myfun.called) def test_SimpleQueue(self): conn = self.conn q = conn.SimpleQueue("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleQueue("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_SimpleBuffer(self): conn = self.conn q = conn.SimpleBuffer("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleBuffer("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_Producer(self): conn = self.conn self.assertIsInstance(conn.Producer(), Producer) self.assertIsInstance(conn.Producer(conn.default_channel), Producer) def test_Consumer(self): conn = self.conn self.assertIsInstance(conn.Consumer(queues=[]), Consumer) self.assertIsInstance(conn.Consumer(queues=[], channel=conn.default_channel), Consumer) def test__repr__(self): self.assertTrue(repr(self.conn)) def test__reduce__(self): x = pickle.loads(pickle.dumps(self.conn)) self.assertDictEqual(x.info(), self.conn.info()) def test_channel_errors(self): class MyTransport(Transport): channel_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.channel_errors, (KeyError, ValueError)) def test_connection_errors(self): class MyTransport(Transport): connection_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.connection_errors, (KeyError, ValueError))
class test_Connection(TestCase): def setUp(self): self.conn = BrokerConnection(port=5672, transport=Transport) def test_establish_connection(self): conn = self.conn conn.connect() self.assertTrue(conn.connection.connected) self.assertEqual(conn.host, "localhost:5672") channel = conn.channel() self.assertTrue(channel.open) self.assertEqual(conn.drain_events(), "event") _connection = conn.connection conn.close() self.assertFalse(_connection.connected) self.assertIsInstance(conn.transport, Transport) def test__enter____exit__(self): conn = self.conn context = conn.__enter__() self.assertIs(context, conn) conn.connect() self.assertTrue(conn.connection.connected) conn.__exit__() self.assertIsNone(conn.connection) conn.close() # again def test_close_survives_connerror(self): class _CustomError(Exception): pass class MyTransport(Transport): connection_errors = (_CustomError, ) def close_connection(self, connection): raise _CustomError("foo") conn = BrokerConnection(transport=MyTransport) conn.connect() conn.close() self.assertTrue(conn._closed) def test_close_when_default_channel(self): conn = self.conn conn._default_channel = Mock() conn._close() conn._default_channel.close.assert_called_with() def test_close_when_default_channel_close_raises(self): class Conn(BrokerConnection): @property def connection_errors(self): return (KeyError, ) conn = Conn("memory://") conn._default_channel = Mock() conn._default_channel.close.side_effect = KeyError() conn._close() conn._default_channel.close.assert_called_with() def test_revive_when_default_channel(self): conn = self.conn defchan = conn._default_channel = Mock() conn.revive(Mock()) defchan.close.assert_called_with() self.assertIsNone(conn._default_channel) def test_ensure_connection(self): self.assertTrue(self.conn.ensure_connection()) def test_ensure_success(self): def publish(): return "foobar" ensured = self.conn.ensure(None, publish) self.assertEqual(ensured(), "foobar") def test_ensure_failure(self): class _CustomError(Exception): pass def publish(): raise _CustomError("bar") ensured = self.conn.ensure(None, publish) with self.assertRaises(_CustomError): ensured() def test_ensure_connection_failure(self): class _ConnectionError(Exception): pass def publish(): raise _ConnectionError("failed connection") self.conn.transport.connection_errors = (_ConnectionError, ) ensured = self.conn.ensure(self.conn, publish) with self.assertRaises(_ConnectionError): ensured() def test_autoretry(self): myfun = Mock() myfun.__name__ = "test_autoretry" self.conn.transport.connection_errors = (KeyError, ) def on_call(*args, **kwargs): myfun.side_effect = None raise KeyError("foo") myfun.side_effect = on_call insured = self.conn.autoretry(myfun) insured() self.assertTrue(myfun.called) def test_SimpleQueue(self): conn = self.conn q = conn.SimpleQueue("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleQueue("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_SimpleBuffer(self): conn = self.conn q = conn.SimpleBuffer("foo") self.assertTrue(q.channel) self.assertTrue(q.channel_autoclose) chan = conn.channel() q2 = conn.SimpleBuffer("foo", channel=chan) self.assertIs(q2.channel, chan) self.assertFalse(q2.channel_autoclose) def test_Producer(self): conn = self.conn self.assertIsInstance(conn.Producer(), Producer) self.assertIsInstance(conn.Producer(conn.default_channel), Producer) def test_Consumer(self): conn = self.conn self.assertIsInstance(conn.Consumer(queues=[]), Consumer) self.assertIsInstance( conn.Consumer(queues=[], channel=conn.default_channel), Consumer) def test__repr__(self): self.assertTrue(repr(self.conn)) def test__reduce__(self): x = pickle.loads(pickle.dumps(self.conn)) self.assertDictEqual(x.info(), self.conn.info()) def test_channel_errors(self): class MyTransport(Transport): channel_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.channel_errors, (KeyError, ValueError)) def test_connection_errors(self): class MyTransport(Transport): connection_errors = (KeyError, ValueError) conn = BrokerConnection(transport=MyTransport) self.assertTupleEqual(conn.connection_errors, (KeyError, ValueError))