def setUp(self):
self._mock_settings = {
'QUEUE_ARN': 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1',
'KEY_ID': 'XXXXXXXXXXXXXXXX',
'KEY': 'YYYYYYYYYYYYYYYYYYYYYYYY',
'WAIT_TIME': 10,
'VISIBILITY_TIMEOUT': 10,
'MESSAGE_GATHER_SIZE': 10,
'POLL_COUNT': 10,
}
self.sqs = SQSQueue(self._mock_settings)
self.sqs._queue = MockQueue()
def __init__(self, processor=None, exception=None, sqs_settings=None):
"""
:param processor: A sub-class object of MessageBodyProcessor
"""
if not processor:
raise GatherException('missing event processor')
self._processor = processor
self._settings = sqs_settings if (
sqs_settings) else self._processor.get_queue_settings()
self._queue = SQSQueue(self._settings)
def test_queue_errors(self):
missing_settings = {}
with self.assertRaises(SQSException) as context:
sqs = SQSQueue(missing_settings)
self.assertIn(
'Invalid SQS configuration', str(context.exception))
invalid_settings = {
'QUEUE_ARN': 'abc',
'KEY_ID': '',
'KEY': '',
}
with self.assertRaises(SQSException) as context:
sqs = SQSQueue(invalid_settings)
self.assertIn('Invalid ARN', str(context.exception))
class Gather(object):
"""
Class to gather event messages from AWS SQS queue,
validate and process their content
"""
def __init__(self, sqs_settings=None, processor=None, exception=None):
if not processor:
raise GatherException('missing event processor')
self._processor = processor
self._exception = exception if exception \
else processor.EXCEPTION_CLASS
self._settings = sqs_settings if sqs_settings \
else settings.AWS_SQS.get(processor.SETTINGS_NAME)
self._topicArn = self._settings.get('TOPIC_ARN')
self._queue = SQSQueue(settings=self._settings)
self._log = getLogger(__name__)
def gather_events(self):
to_fetch = self._settings.get('MESSAGE_GATHER_SIZE')
while to_fetch > 0:
n = min([to_fetch, 10])
msgs = self._queue.get_messages(
num_messages=n,
visibility_timeout=self._settings.get('VISIBILITY_TIMEOUT'))
for msg in msgs:
try:
sqs_msg = json.loads(msg.get_body())
if sqs_msg['TopicArn'] == self._topicArn:
raw_message = SNS(sqs_msg)
if self._settings.get('VALIDATE_SNS_SIGNATURE', True):
raw_message.validate()
if sqs_msg['Type'] == 'Notification':
settings = self._settings.get(
'PAYLOAD_SETTINGS', {})
message = raw_message.extract()
self._processor(settings, message).process()
elif sqs_msg['Type'] == 'SubscriptionConfirmation':
self._log.debug(
'SubscribeURL: ' + sqs_msg['SubscribeURL'])
else:
self._log.warning(
'Unrecognized TopicARN : ' + sqs_msg['TopicArn'])
except ValueError as err:
raise GatherException('JSON : %s' % err)
except self._exception, err:
raise GatherException("MESSAGE: %s" % err)
except SNSException, err:
raise GatherException("SNS: %s" % err)
except Exception, err:
self._log.exception("Gather Error")
raise GatherException("ERROR: %s" % err)
else:
class TestSQSQueue(TestCase):
def setUp(self):
self._mock_settings = {
'QUEUE_ARN': 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1',
'KEY_ID': 'XXXXXXXXXXXXXXXX',
'KEY': 'YYYYYYYYYYYYYYYYYYYYYYYY',
'WAIT_TIME': 10,
'VISIBILITY_TIMEOUT': 10,
'MESSAGE_GATHER_SIZE': 10,
'POLL_COUNT': 10,
}
self.sqs = SQSQueue(self._mock_settings)
self.sqs._queue = MockQueue()
def test_queue(self):
self.assertEquals(
self.sqs.arn, 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1')
self.assertEquals(self.sqs.key_id, 'XXXXXXXXXXXXXXXX')
self.assertEquals(self.sqs.key, 'YYYYYYYYYYYYYYYYYYYYYYYY')
self.assertEquals(self.sqs.region, 'xx-mock-999')
self.assertEquals(self.sqs.account_id, '000000000000')
self.assertEquals(self.sqs.queue_name, 'ww-wwww-1')
# Overridden __getattr__ should still raise an AttributeError
self.assertRaises(AttributeError, lambda: self.sqs._does_not_exist)
def test_get_messages(self):
messages = self.sqs.get_messages()
self.assertEquals(len(messages), 100)
self.sqs._settings['MESSAGE_GATHER_SIZE'] = 9
messages = self.sqs.get_messages()
self.assertEquals(len(messages), 90)
self.sqs._settings['POLL_COUNT'] = 1
messages = self.sqs.get_messages()
self.assertEquals(len(messages), 9)
def test_empty_queue(self):
self.sqs._queue = MockEmptyQueue()
messages = self.sqs.get_messages()
self.assertEquals(len(messages), 0)
def __init__(self, sqs_settings=None, processor=None, exception=None):
if not processor:
raise GatherException('missing event processor')
self._processor = processor
self._exception = exception if exception \
else processor.EXCEPTION_CLASS
self._settings = sqs_settings if sqs_settings \
else settings.AWS_SQS.get(processor.SETTINGS_NAME)
self._topicArn = self._settings.get('TOPIC_ARN')
self._queue = SQSQueue(settings=self._settings)
self._log = getLogger(__name__)
class Gather(object):
"""
Class to gather event messages from AWS SQS queue,
validate and process their content
"""
def __init__(self, processor=None, exception=None, sqs_settings=None):
"""
:param processor: A sub-class object of MessageBodyProcessor
"""
if not processor:
raise GatherException('missing event processor')
self._processor = processor
self._settings = sqs_settings if (
sqs_settings) else self._processor.get_queue_settings()
self._queue = SQSQueue(self._settings)
# if Exception, abort!
def gather_events(self):
for msg in self._queue.get_messages():
try:
# validate the message and hand it off for processing
message = Message(json.loads(msg.body), self._settings)
if message.validate():
self._processor.process(message.extract())
except (CryptoException, ProcessorException) as err:
# log message specific error, abort if unknown error
logger.error('{}: {}'.format(
err,
traceback.format_exc().splitlines()))
else:
# inform the queue that this message has been processed
msg.delete()
