def run(): # pragma: no cover
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG)
logger.info("Starting SDX Collect", version=__version__)
response_processor = ResponseProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS,
queue=settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=response_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def main():
# Set tornado to listen to healthcheck endpoint
app = make_app()
server = tornado.httpserver.HTTPServer(app)
server.bind(os.getenv('PORT', '8080'))
server.start(1)
rp = ResponseProcessor("outbound")
quarantine_publisher = QueuePublisher(
[Config.RABBIT_URL],
os.getenv('RABBIT_QUARANTINE_QUEUE', 'QUARANTINE_TEST'),
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=os.getenv('RABBIT_EXCHANGE', 'test'),
exchange_type=os.getenv('EXCHANGE_TYPE', 'topic'),
rabbit_queue=os.getenv('RABBIT_QUEUE', 'test'),
rabbit_urls=[Config.RABBIT_URL],
quarantine_publisher=quarantine_publisher,
process=rp.process,
check_tx_id=False,
)
message_consumer.run()
return 0
def test_callable(self):
"""The final argument of the message consumer is a callable function for it to run.
Not providing it with something callable raises an AttributeError"""
with self.assertRaises(AttributeError):
self.consumer = MessageConsumer(True, 'test', 'topic', 'test',
[self.amqp_url],
self.quarantine_publisher,
self.body)
def __init__(self):
"""Initialise a Bridge object."""
self._eq_queue_hosts = [
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST,
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2
]
self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT
self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER
self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD
self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT
self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER
self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD
self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST
self._eq_queue_urls = [
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[0],
self._eq_queue_port),
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[1],
self._eq_queue_port),
]
self._sdx_queue_url = [
'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user,
self._sdx_queue_password,
self._sdx_queue_host,
self._sdx_queue_port)
]
self.publisher = QueuePublisher(
self._sdx_queue_url,
settings.COLLECT_QUEUE,
)
self.quarantine_publisher = QueuePublisher(
urls=self._sdx_queue_url,
queue=settings.QUARANTINE_QUEUE,
)
self.consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.EQ_QUEUE,
rabbit_urls=self._eq_queue_urls,
quarantine_publisher=self.quarantine_publisher,
process=self.process,
)
def setUp(self):
self.amqp_url = 'amqp://*****:*****@0.0.0.0:5672'
self.quarantine_publisher = QueuePublisher([self.amqp_url],
'test_quarantine')
self.consumer = MessageConsumer(True, 'test', 'topic', 'test',
[self.amqp_url],
self.quarantine_publisher,
lambda x, y: True)
self.props = DotDict({'headers': {'tx_id': 'test'}})
self.props_no_tx_id = DotDict({'headers': {}})
self.props_no_headers = DotDict({})
self.props_no_x_delivery_count = DotDict(
{'headers': {
'tx_id': 'test'
}})
self.basic_deliver = DotDict({'delivery_tag': 'test'})
self.body = json.loads('"{test message}"')
def run():
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG)
logger.info("Starting SDX Receipt RRM", version=__version__)
response_processor = ResponseProcessor(logger)
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=response_processor.process)
try:
logger.info("Starting consumer")
if settings.SDX_RECEIPT_RRM_SECRET is None:
logger.error("No SDX_RECEIPT_RRM_SECRET env var supplied")
sys.exit(1)
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def run():
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
# These structlog settings allow bound fields to persist between classes
structlog.configure(logger_factory=LoggerFactory(), context_class=wrap_dict(dict))
logger = structlog.getLogger()
logger.info('Starting SDX Downstream', version=__version__)
message_processor = MessageProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS,
queue=settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type='topic',
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=message_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def run(): # pragma: no cover
logging.basicConfig(format=settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=settings.LOGGING_LEVEL)
logging.getLogger("sdc.rabbit").setLevel(logging.DEBUG)
logger.info("Starting SDX Collect", version=__version__)
response_processor = ResponseProcessor()
quarantine_publisher = QueuePublisher(
urls=settings.RABBIT_URLS, queue=settings.RABBIT_QUARANTINE_QUEUE)
message_consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.RABBIT_QUEUE,
rabbit_urls=settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=response_processor.process)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
def run():
logging.basicConfig(format=app.settings.LOGGING_FORMAT,
datefmt="%Y-%m-%dT%H:%M:%S",
level=app.settings.LOGGING_LEVEL)
logging.getLogger('sdc.rabbit').setLevel(logging.INFO)
message_processor = MessageProcessor()
quarantine_publisher = QueuePublisher(
urls=app.settings.RABBIT_URLS,
queue=app.settings.RABBIT_QUARANTINE_QUEUE
)
message_consumer = MessageConsumer(
durable_queue=False,
exchange=app.settings.RABBIT_EXCHANGE,
exchange_type='topic',
rabbit_queue=app.settings.RABBIT_QUEUE,
rabbit_urls=app.settings.RABBIT_URLS,
quarantine_publisher=quarantine_publisher,
process=message_processor.process
)
try:
message_consumer.run()
except KeyboardInterrupt:
message_consumer.stop()
class TestSdxConsumer(unittest.TestCase):
logger = logging.getLogger(__name__)
def setUp(self):
self.amqp_url = 'amqp://*****:*****@0.0.0.0:5672'
self.quarantine_publisher = QueuePublisher([self.amqp_url],
'test_quarantine')
self.consumer = MessageConsumer(True, 'test', 'topic', 'test',
[self.amqp_url],
self.quarantine_publisher,
lambda x, y: True)
self.props = DotDict({'headers': {'tx_id': 'test'}})
self.props_no_tx_id = DotDict({'headers': {}})
self.props_no_headers = DotDict({})
self.props_no_x_delivery_count = DotDict(
{'headers': {
'tx_id': 'test'
}})
self.basic_deliver = DotDict({'delivery_tag': 'test'})
self.body = json.loads('"{test message}"')
def test_callable(self):
"""The final argument of the message consumer is a callable function for it to run.
Not providing it with something callable raises an AttributeError"""
with self.assertRaises(AttributeError):
self.consumer = MessageConsumer(True, 'test', 'topic', 'test',
[self.amqp_url],
self.quarantine_publisher,
self.body)
def test_queue_attributes(self):
"""Checks the values of the consumers private variables are as expected"""
self.assertEqual(self.consumer._count, 1)
self.assertEqual(self.consumer._exchange, 'test')
self.assertEqual(self.consumer._exchange_type, 'topic')
self.assertEqual(self.consumer._queue, 'test')
self.assertEqual(self.consumer._rabbit_urls,
['amqp://*****:*****@0.0.0.0:5672'])
self.assertEqual(self.consumer._durable_queue, True)
self.assertEqual(self.consumer.quarantine_publisher._queue,
'test_quarantine')
self.assertEqual(self.consumer.quarantine_publisher._urls,
['amqp://*****:*****@0.0.0.0:5672'])
def test_tx_id(self):
self.assertEqual('test', self.consumer.tx_id(self.props))
with self.assertRaises(KeyError):
self.consumer.tx_id(self.props_no_tx_id)
with self.assertRaises(TypeError):
self.consumer.tx_id(self.props_no_headers)
def test_on_message_txid_key_error_returns_none(self):
mock_method = 'sdc.rabbit.AsyncConsumer.reject_message'
with mock.patch(mock_method) as barMock:
barMock.return_value = None
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props_no_tx_id,
'test')
def test_on_message_headers_type_error_returns_none(self):
mock_method = 'sdc.rabbit.AsyncConsumer.reject_message'
with mock.patch(mock_method) as barMock:
barMock.return_value = None
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props_no_headers,
'test'.encode())
def test_on_message_logger(self):
mock_method = 'sdc.rabbit.AsyncConsumer.acknowledge_message'
with mock.patch(mock_method) as bar_mock:
bar_mock.return_value = None
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
def test_on_message_quarantinable_error(self):
mock_method = 'sdc.rabbit.AsyncConsumer.reject_message'
with mock.patch(mock_method):
self.consumer.process = lambda x: (_ for _ in
()).throw(QuarantinableError())
with self.assertLogs(level='ERROR') as cm:
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
self.assertIn("Quarantinable error occured", cm[0][1].message)
def test_on_message_publish_message_error(self):
mock_reject_message = 'sdc.rabbit.AsyncConsumer.reject_message'
mock_publish_message = 'sdc.rabbit.QueuePublisher.publish_message'
with mock.patch(mock_reject_message):
self.consumer.process = lambda x: (_ for _ in
()).throw(QuarantinableError())
with mock.patch(mock_publish_message) as publish_mock:
publish_mock.side_effect = PublishMessageError
with self.assertLogs(level='ERROR') as cm:
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
expected_msg = "Unable to publish message to quarantine queue. Rejecting message and requeuing."
self.assertIn(expected_msg, cm[0][1].message)
def test_on_message_bad_message_error(self):
mock_method = 'sdc.rabbit.AsyncConsumer.reject_message'
def bad_message_error(x, y):
raise BadMessageError
with mock.patch(mock_method):
self.consumer.process = bad_message_error
with self.assertLogs(level='ERROR') as cm:
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
self.assertIn("Quarantinable error occured", cm[0][0].message)
def test_on_message_retryable_message_error(self):
"""If a retryable error is thrown then a log line with a nack will
be logged out"""
mock_method = 'sdc.rabbit.AsyncConsumer.nack_message'
def retryable_error(x, y):
raise RetryableError
with mock.patch(mock_method):
self.consumer.process = retryable_error
with self.assertLogs(level='ERROR') as cm:
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
self.assertIn("Failed to process", cm[0][0].message)
self.assertIn("action=nack", cm[0][0].message)
def test_on_message_generic_exception_caught_and_nacked(self):
"""If an unexpected exception is thrown (one we haven't caught specifically),
then we log out that something unexpected has happened and nack it"""
mock_method = 'sdc.rabbit.AsyncConsumer.nack_message'
def exception_error(x, y):
raise Exception
with mock.patch(mock_method) as mocker:
self.consumer.process = exception_error
with self.assertLogs(level='ERROR') as cm:
self.consumer.on_message(self.consumer._channel,
self.basic_deliver, self.props,
self.body.encode('UTF-8'))
mocker.assert_called_with(self.basic_deliver.delivery_tag,
tx_id='test')
self.assertIn("Unexpected exception occurred, failed to process",
cm[0][0].message)
self.assertIn("action=nack", cm[0][0].message)
class Bridge:
"""A Bridge object that takes from one queue and publishes to another."""
def __init__(self):
"""Initialise a Bridge object."""
self._eq_queue_hosts = [
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST,
settings.SDX_GATEWAY_EQ_RABBITMQ_HOST2
]
self._eq_queue_port = settings.SDX_GATEWAY_EQ_RABBIT_PORT
self._eq_queue_user = settings.SDX_GATEWAY_EQ_RABBITMQ_USER
self._eq_queue_password = settings.SDX_GATEWAY_EQ_RABBITMQ_PASSWORD
self._sdx_queue_port = settings.SDX_GATEWAY_SDX_RABBITMQ_PORT
self._sdx_queue_user = settings.SDX_GATEWAY_SDX_RABBITMQ_USER
self._sdx_queue_password = settings.SDX_GATEWAY_SDX_RABBITMQ_PASSWORD
self._sdx_queue_host = settings.SDX_GATEWAY_SDX_RABBITMQ_HOST
self._eq_queue_urls = [
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[0],
self._eq_queue_port),
'amqp://{}:{}@{}:{}/%2f'.format(self._eq_queue_user,
self._eq_queue_password,
self._eq_queue_hosts[1],
self._eq_queue_port),
]
self._sdx_queue_url = [
'amqp://{}:{}@{}:{}/%2f'.format(self._sdx_queue_user,
self._sdx_queue_password,
self._sdx_queue_host,
self._sdx_queue_port)
]
self.publisher = QueuePublisher(
self._sdx_queue_url,
settings.COLLECT_QUEUE,
)
self.quarantine_publisher = QueuePublisher(
urls=self._sdx_queue_url,
queue=settings.QUARANTINE_QUEUE,
)
self.consumer = MessageConsumer(
durable_queue=True,
exchange=settings.RABBIT_EXCHANGE,
exchange_type="topic",
rabbit_queue=settings.EQ_QUEUE,
rabbit_urls=self._eq_queue_urls,
quarantine_publisher=self.quarantine_publisher,
process=self.process,
)
def process(self, message, tx_id=None):
try:
self.publisher.publish_message(message, headers={'tx_id': tx_id})
except PublishMessageError:
logger.exception('Unsuccessful publish.', tx_id=tx_id)
raise RetryableError
except:
logger.exception(
'Unknown exception occurred during publish. Retrying.',
tx_id=tx_id)
raise RetryableError
def run(self):
"""Run this object's MessageConsumer. Stops on KeyboardInterrupt."""
logger.info("Starting consumer")
self.consumer.run()
def stop(self):
logger.info("Stopping consumer")
self.consumer.stop()