def setUp(self): self._mock_settings = { 'QUEUE_ARN': 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1', 'KEY_ID': 'XXXXXXXXXXXXXXXX', 'KEY': 'YYYYYYYYYYYYYYYYYYYYYYYY', 'WAIT_TIME': 10, 'VISIBILITY_TIMEOUT': 10, 'MESSAGE_GATHER_SIZE': 10, 'POLL_COUNT': 10, } self.sqs = SQSQueue(self._mock_settings) self.sqs._queue = MockQueue()
def __init__(self, processor=None, exception=None, sqs_settings=None): """ :param processor: A sub-class object of MessageBodyProcessor """ if not processor: raise GatherException('missing event processor') self._processor = processor self._settings = sqs_settings if ( sqs_settings) else self._processor.get_queue_settings() self._queue = SQSQueue(self._settings)
def test_queue_errors(self): missing_settings = {} with self.assertRaises(SQSException) as context: sqs = SQSQueue(missing_settings) self.assertIn( 'Invalid SQS configuration', str(context.exception)) invalid_settings = { 'QUEUE_ARN': 'abc', 'KEY_ID': '', 'KEY': '', } with self.assertRaises(SQSException) as context: sqs = SQSQueue(invalid_settings) self.assertIn('Invalid ARN', str(context.exception))
class Gather(object): """ Class to gather event messages from AWS SQS queue, validate and process their content """ def __init__(self, sqs_settings=None, processor=None, exception=None): if not processor: raise GatherException('missing event processor') self._processor = processor self._exception = exception if exception \ else processor.EXCEPTION_CLASS self._settings = sqs_settings if sqs_settings \ else settings.AWS_SQS.get(processor.SETTINGS_NAME) self._topicArn = self._settings.get('TOPIC_ARN') self._queue = SQSQueue(settings=self._settings) self._log = getLogger(__name__) def gather_events(self): to_fetch = self._settings.get('MESSAGE_GATHER_SIZE') while to_fetch > 0: n = min([to_fetch, 10]) msgs = self._queue.get_messages( num_messages=n, visibility_timeout=self._settings.get('VISIBILITY_TIMEOUT')) for msg in msgs: try: sqs_msg = json.loads(msg.get_body()) if sqs_msg['TopicArn'] == self._topicArn: raw_message = SNS(sqs_msg) if self._settings.get('VALIDATE_SNS_SIGNATURE', True): raw_message.validate() if sqs_msg['Type'] == 'Notification': settings = self._settings.get( 'PAYLOAD_SETTINGS', {}) message = raw_message.extract() self._processor(settings, message).process() elif sqs_msg['Type'] == 'SubscriptionConfirmation': self._log.debug( 'SubscribeURL: ' + sqs_msg['SubscribeURL']) else: self._log.warning( 'Unrecognized TopicARN : ' + sqs_msg['TopicArn']) except ValueError as err: raise GatherException('JSON : %s' % err) except self._exception, err: raise GatherException("MESSAGE: %s" % err) except SNSException, err: raise GatherException("SNS: %s" % err) except Exception, err: self._log.exception("Gather Error") raise GatherException("ERROR: %s" % err) else:
class TestSQSQueue(TestCase): def setUp(self): self._mock_settings = { 'QUEUE_ARN': 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1', 'KEY_ID': 'XXXXXXXXXXXXXXXX', 'KEY': 'YYYYYYYYYYYYYYYYYYYYYYYY', 'WAIT_TIME': 10, 'VISIBILITY_TIMEOUT': 10, 'MESSAGE_GATHER_SIZE': 10, 'POLL_COUNT': 10, } self.sqs = SQSQueue(self._mock_settings) self.sqs._queue = MockQueue() def test_queue(self): self.assertEquals( self.sqs.arn, 'arn:aws:sqs:xx-mock-999:000000000000:ww-wwww-1') self.assertEquals(self.sqs.key_id, 'XXXXXXXXXXXXXXXX') self.assertEquals(self.sqs.key, 'YYYYYYYYYYYYYYYYYYYYYYYY') self.assertEquals(self.sqs.region, 'xx-mock-999') self.assertEquals(self.sqs.account_id, '000000000000') self.assertEquals(self.sqs.queue_name, 'ww-wwww-1') # Overridden __getattr__ should still raise an AttributeError self.assertRaises(AttributeError, lambda: self.sqs._does_not_exist) def test_get_messages(self): messages = self.sqs.get_messages() self.assertEquals(len(messages), 100) self.sqs._settings['MESSAGE_GATHER_SIZE'] = 9 messages = self.sqs.get_messages() self.assertEquals(len(messages), 90) self.sqs._settings['POLL_COUNT'] = 1 messages = self.sqs.get_messages() self.assertEquals(len(messages), 9) def test_empty_queue(self): self.sqs._queue = MockEmptyQueue() messages = self.sqs.get_messages() self.assertEquals(len(messages), 0)
def __init__(self, sqs_settings=None, processor=None, exception=None): if not processor: raise GatherException('missing event processor') self._processor = processor self._exception = exception if exception \ else processor.EXCEPTION_CLASS self._settings = sqs_settings if sqs_settings \ else settings.AWS_SQS.get(processor.SETTINGS_NAME) self._topicArn = self._settings.get('TOPIC_ARN') self._queue = SQSQueue(settings=self._settings) self._log = getLogger(__name__)
class Gather(object): """ Class to gather event messages from AWS SQS queue, validate and process their content """ def __init__(self, processor=None, exception=None, sqs_settings=None): """ :param processor: A sub-class object of MessageBodyProcessor """ if not processor: raise GatherException('missing event processor') self._processor = processor self._settings = sqs_settings if ( sqs_settings) else self._processor.get_queue_settings() self._queue = SQSQueue(self._settings) # if Exception, abort! def gather_events(self): for msg in self._queue.get_messages(): try: # validate the message and hand it off for processing message = Message(json.loads(msg.body), self._settings) if message.validate(): self._processor.process(message.extract()) except (CryptoException, ProcessorException) as err: # log message specific error, abort if unknown error logger.error('{}: {}'.format( err, traceback.format_exc().splitlines())) else: # inform the queue that this message has been processed msg.delete()