def run(): # pragma: no cover logging.basicConfig(format=settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=settings.LOGGING_LEVEL) logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG) logger.info("Starting SDX Collect", version=__version__) response_processor = ResponseProcessor() quarantine_publisher = QueuePublisher( urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE ) message_consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type="topic", rabbit_queue=settings.RABBIT_QUEUE, rabbit_urls=settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=response_processor.process ) try: message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
def main(): # Set tornado to listen to healthcheck endpoint app = make_app() server = tornado.httpserver.HTTPServer(app) server.bind(os.getenv('PORT', '8080')) server.start(1) rp = ResponseProcessor("outbound") quarantine_publisher = QueuePublisher( [Config.RABBIT_URL], os.getenv('RABBIT_QUARANTINE_QUEUE', 'QUARANTINE_TEST'), ) message_consumer = MessageConsumer( durable_queue=True, exchange=os.getenv('RABBIT_EXCHANGE', 'test'), exchange_type=os.getenv('EXCHANGE_TYPE', 'topic'), rabbit_queue=os.getenv('RABBIT_QUEUE', 'test'), rabbit_urls=[Config.RABBIT_URL], quarantine_publisher=quarantine_publisher, process=rp.process, check_tx_id=False, ) message_consumer.run() return 0
def test_callable(self): """The final argument of the message consumer is a callable function for it to run. Not providing it with something callable raises an AttributeError""" with self.assertRaises(AttributeError): self.consumer = MessageConsumer(True, 'test', 'topic', 'test', [self.amqp_url], self.quarantine_publisher, self.body)
def __init__(self): """Initialise a Bridge object.""" self._eq_queue_hosts = [ settings.SDX_GATEWAY_EQ_RABBITMQ_HOST, settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2 ] self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST self._eq_queue_urls = [ 'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user, self._eq_queue_password, self._eq_queue_hosts[0], self._eq_queue_port), 'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user, self._eq_queue_password, self._eq_queue_hosts[1], self._eq_queue_port), ] self._sdx_queue_url = [ 'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user, self._sdx_queue_password, self._sdx_queue_host, self._sdx_queue_port) ] self.publisher = QueuePublisher( self._sdx_queue_url, settings.COLLECT_QUEUE, ) self.quarantine_publisher = QueuePublisher( urls=self._sdx_queue_url, queue=settings.QUARANTINE_QUEUE, ) self.consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type="topic", rabbit_queue=settings.EQ_QUEUE, rabbit_urls=self._eq_queue_urls, quarantine_publisher=self.quarantine_publisher, process=self.process, )
def setUp(self): self.amqp_url = 'amqp://*****:*****@0.0.0.0:5672' self.quarantine_publisher = QueuePublisher([self.amqp_url], 'test_quarantine') self.consumer = MessageConsumer(True, 'test', 'topic', 'test', [self.amqp_url], self.quarantine_publisher, lambda x, y: True) self.props = DotDict({'headers': {'tx_id': 'test'}}) self.props_no_tx_id = DotDict({'headers': {}}) self.props_no_headers = DotDict({}) self.props_no_x_delivery_count = DotDict( {'headers': { 'tx_id': 'test' }}) self.basic_deliver = DotDict({'delivery_tag': 'test'}) self.body = json.loads('"{test message}"')
def run(): logging.basicConfig(format=settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=settings.LOGGING_LEVEL) logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG) logger.info("Starting SDX Receipt RRM", version=__version__) response_processor = ResponseProcessor(logger) quarantine_publisher = QueuePublisher( urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE) message_consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type="topic", rabbit_queue=settings.RABBIT_QUEUE, rabbit_urls=settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=response_processor.process) try: logger.info("Starting consumer") if settings.SDX_RECEIPT_RRM_SECRET is None: logger.error("No SDX_RECEIPT_RRM_SECRET env var supplied") sys.exit(1) message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
def run(): logging.basicConfig(format=settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=settings.LOGGING_LEVEL) logging.getLogger('sdc.rabbit').setLevel(logging.INFO) # These structlog settings allow bound fields to persist between classes structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict)) logger = structlog.getLogger() logger.info('Starting SDX Downstream', version=__version__) message_processor = MessageProcessor() quarantine_publisher = QueuePublisher( urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE ) message_consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type='topic', rabbit_queue=settings.RABBIT_QUEUE, rabbit_urls=settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=message_processor.process ) try: message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
def run(): # pragma: no cover logging.basicConfig(format=settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=settings.LOGGING_LEVEL) logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG) logger.info("Starting SDX Collect", version=__version__) response_processor = ResponseProcessor() quarantine_publisher = QueuePublisher( urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE) message_consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type="topic", rabbit_queue=settings.RABBIT_QUEUE, rabbit_urls=settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=response_processor.process) try: message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
def run(): logging.basicConfig(format=app.settings.LOGGING_FORMAT, datefmt="%Y-%m-%dT%H:%M:%S", level=app.settings.LOGGING_LEVEL) logging.getLogger('sdc.rabbit').setLevel(logging.INFO) message_processor = MessageProcessor() quarantine_publisher = QueuePublisher( urls=app.settings.RABBIT_URLS, queue=app.settings.RABBIT_QUARANTINE_QUEUE ) message_consumer = MessageConsumer( durable_queue=False, exchange=app.settings.RABBIT_EXCHANGE, exchange_type='topic', rabbit_queue=app.settings.RABBIT_QUEUE, rabbit_urls=app.settings.RABBIT_URLS, quarantine_publisher=quarantine_publisher, process=message_processor.process ) try: message_consumer.run() except KeyboardInterrupt: message_consumer.stop()
class TestSdxConsumer(unittest.TestCase): logger = logging.getLogger(__name__) def setUp(self): self.amqp_url = 'amqp://*****:*****@0.0.0.0:5672' self.quarantine_publisher = QueuePublisher([self.amqp_url], 'test_quarantine') self.consumer = MessageConsumer(True, 'test', 'topic', 'test', [self.amqp_url], self.quarantine_publisher, lambda x, y: True) self.props = DotDict({'headers': {'tx_id': 'test'}}) self.props_no_tx_id = DotDict({'headers': {}}) self.props_no_headers = DotDict({}) self.props_no_x_delivery_count = DotDict( {'headers': { 'tx_id': 'test' }}) self.basic_deliver = DotDict({'delivery_tag': 'test'}) self.body = json.loads('"{test message}"') def test_callable(self): """The final argument of the message consumer is a callable function for it to run. Not providing it with something callable raises an AttributeError""" with self.assertRaises(AttributeError): self.consumer = MessageConsumer(True, 'test', 'topic', 'test', [self.amqp_url], self.quarantine_publisher, self.body) def test_queue_attributes(self): """Checks the values of the consumers private variables are as expected""" self.assertEqual(self.consumer._count, 1) self.assertEqual(self.consumer._exchange, 'test') self.assertEqual(self.consumer._exchange_type, 'topic') self.assertEqual(self.consumer._queue, 'test') self.assertEqual(self.consumer._rabbit_urls, ['amqp://*****:*****@0.0.0.0:5672']) self.assertEqual(self.consumer._durable_queue, True) self.assertEqual(self.consumer.quarantine_publisher._queue, 'test_quarantine') self.assertEqual(self.consumer.quarantine_publisher._urls, ['amqp://*****:*****@0.0.0.0:5672']) def test_tx_id(self): self.assertEqual('test', self.consumer.tx_id(self.props)) with self.assertRaises(KeyError): self.consumer.tx_id(self.props_no_tx_id) with self.assertRaises(TypeError): self.consumer.tx_id(self.props_no_headers) def test_on_message_txid_key_error_returns_none(self): mock_method = 'sdc.rabbit.AsyncConsumer.reject_message' with mock.patch(mock_method) as barMock: barMock.return_value = None self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props_no_tx_id, 'test') def test_on_message_headers_type_error_returns_none(self): mock_method = 'sdc.rabbit.AsyncConsumer.reject_message' with mock.patch(mock_method) as barMock: barMock.return_value = None self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props_no_headers, 'test'.encode()) def test_on_message_logger(self): mock_method = 'sdc.rabbit.AsyncConsumer.acknowledge_message' with mock.patch(mock_method) as bar_mock: bar_mock.return_value = None self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) def test_on_message_quarantinable_error(self): mock_method = 'sdc.rabbit.AsyncConsumer.reject_message' with mock.patch(mock_method): self.consumer.process = lambda x: (_ for _ in ()).throw(QuarantinableError()) with self.assertLogs(level='ERROR') as cm: self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) self.assertIn("Quarantinable error occured", cm[0][1].message) def test_on_message_publish_message_error(self): mock_reject_message = 'sdc.rabbit.AsyncConsumer.reject_message' mock_publish_message = 'sdc.rabbit.QueuePublisher.publish_message' with mock.patch(mock_reject_message): self.consumer.process = lambda x: (_ for _ in ()).throw(QuarantinableError()) with mock.patch(mock_publish_message) as publish_mock: publish_mock.side_effect = PublishMessageError with self.assertLogs(level='ERROR') as cm: self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) expected_msg = "Unable to publish message to quarantine queue. Rejecting message and requeuing." self.assertIn(expected_msg, cm[0][1].message) def test_on_message_bad_message_error(self): mock_method = 'sdc.rabbit.AsyncConsumer.reject_message' def bad_message_error(x, y): raise BadMessageError with mock.patch(mock_method): self.consumer.process = bad_message_error with self.assertLogs(level='ERROR') as cm: self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) self.assertIn("Quarantinable error occured", cm[0][0].message) def test_on_message_retryable_message_error(self): """If a retryable error is thrown then a log line with a nack will be logged out""" mock_method = 'sdc.rabbit.AsyncConsumer.nack_message' def retryable_error(x, y): raise RetryableError with mock.patch(mock_method): self.consumer.process = retryable_error with self.assertLogs(level='ERROR') as cm: self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) self.assertIn("Failed to process", cm[0][0].message) self.assertIn("action=nack", cm[0][0].message) def test_on_message_generic_exception_caught_and_nacked(self): """If an unexpected exception is thrown (one we haven't caught specifically), then we log out that something unexpected has happened and nack it""" mock_method = 'sdc.rabbit.AsyncConsumer.nack_message' def exception_error(x, y): raise Exception with mock.patch(mock_method) as mocker: self.consumer.process = exception_error with self.assertLogs(level='ERROR') as cm: self.consumer.on_message(self.consumer._channel, self.basic_deliver, self.props, self.body.encode('UTF-8')) mocker.assert_called_with(self.basic_deliver.delivery_tag, tx_id='test') self.assertIn("Unexpected exception occurred, failed to process", cm[0][0].message) self.assertIn("action=nack", cm[0][0].message)
class Bridge: """A Bridge object that takes from one queue and publishes to another.""" def __init__(self): """Initialise a Bridge object.""" self._eq_queue_hosts = [ settings.SDX_GATEWAY_EQ_RABBITMQ_HOST, settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2 ] self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST self._eq_queue_urls = [ 'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user, self._eq_queue_password, self._eq_queue_hosts[0], self._eq_queue_port), 'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user, self._eq_queue_password, self._eq_queue_hosts[1], self._eq_queue_port), ] self._sdx_queue_url = [ 'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user, self._sdx_queue_password, self._sdx_queue_host, self._sdx_queue_port) ] self.publisher = QueuePublisher( self._sdx_queue_url, settings.COLLECT_QUEUE, ) self.quarantine_publisher = QueuePublisher( urls=self._sdx_queue_url, queue=settings.QUARANTINE_QUEUE, ) self.consumer = MessageConsumer( durable_queue=True, exchange=settings.RABBIT_EXCHANGE, exchange_type="topic", rabbit_queue=settings.EQ_QUEUE, rabbit_urls=self._eq_queue_urls, quarantine_publisher=self.quarantine_publisher, process=self.process, ) def process(self, message, tx_id=None): try: self.publisher.publish_message(message, headers={'tx_id': tx_id}) except PublishMessageError: logger.exception('Unsuccessful publish.', tx_id=tx_id) raise RetryableError except: logger.exception( 'Unknown exception occurred during publish. Retrying.', tx_id=tx_id) raise RetryableError def run(self): """Run this object's MessageConsumer. Stops on KeyboardInterrupt.""" logger.info("Starting consumer") self.consumer.run() def stop(self): logger.info("Stopping consumer") self.consumer.stop()