def test_empty_worker(self): sqs = SQSConnection(region=get_sqs_region('us-east-1')) self.assertEqual(len(sqs.get_all_queues()), 0) with spawn_worker(Worker()): # Worker is empty, not registering any queues self.assertEqual(len(sqs.get_all_queues()), 0)
def test_worker_consumes_queue(self): sqs = SQSConnection(region=get_sqs_region('us-east-1')) self.assertEqual(len(sqs.get_all_queues()), 0) queue = sqs.create_queue('test_events') queue.write(make_message({'test': '1'})) self.assertEqual(queue.count(), 1) worker = Worker() worker.ctrl.wait_time_seconds = 0.1 worker.idle_time_seconds = 0.1 worker.add_consumer('test_events', dummy_consumer) with spawn_worker(worker): time.sleep(.2) self.assertEqual(queue.count(), 0) queue.write(make_message({'test': '2'})) self.assertEqual(queue.count(), 1) time.sleep(.2) self.assertEqual(queue.count(), 0)
class SqsApi(object): def __init__(self): self.conn = SQSConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY) def create_queue(self, name="MuscleOpsQ"): return self.conn.create_queue(name) def list_queues(self): return self.conn.get_all_queues() def write_message(self, q, msg="If you're sleepy and you know it; clap your hands!" ): m = RawMessage() m.set_body(msg) q.write(m) def read_message(self, q, msg="If you're sleepy and you know it; clap your hands!"): rs = q.get_messages() m = rs[0].get_body() if len(rs) else msg return m
def main(): try: opts, args = getopt.getopt( sys.argv[1:], 'hcq:o:t:r:', ['help', 'clear', 'queue=', 'output=', 'timeout=', 'region=']) except: usage() sys.exit(2) queue_name = '' output_file = '' timeout = 30 region = '' clear = False for o, a in opts: if o in ('-h', '--help'): usage() sys.exit() if o in ('-q', '--queue'): queue_name = a if o in ('-o', '--output'): output_file = a if o in ('-c', '--clear'): clear = True if o in ('-t', '--timeout'): timeout = int(a) if o in ('-r', '--region'): region = a if region: c = boto.sqs.connect_to_region(region) if c is None: print 'Invalid region (%s)' % region sys.exit(1) else: c = SQSConnection() if queue_name: try: rs = [c.create_queue(queue_name)] except SQSError as e: print 'An Error Occurred:' print '%s: %s' % (e.status, e.reason) print e.body sys.exit() else: try: rs = c.get_all_queues() except SQSError as e: print 'An Error Occurred:' print '%s: %s' % (e.status, e.reason) print e.body sys.exit() for q in rs: if clear: n = q.clear() print 'clearing %d messages from %s' % (n, q.id) elif output_file: q.dump(output_file) else: print q.id, q.count(vtimeout=timeout)
def test_worker_creates_queue(self): sqs = SQSConnection(region=get_sqs_region('us-east-1')) self.assertEqual(len(sqs.get_all_queues()), 0) worker = Worker() worker.ctrl.wait_time_seconds = 0.1 worker.idle_time_seconds = 0.1 worker.add_consumer('test_events', dummy_consumer) with spawn_worker(worker): time.sleep(.2) all_queues = sqs.get_all_queues() self.assertEqual(len(all_queues), 1) self.assertEqual( all_queues[0].name, 'test_events' )
class QueueReader(base.QueueReader): """Reads messages from work queues, respecting priority""" def __init__(self, conf): super(QueueReader, self).__init__(conf) self._conn = SQSConnection(conf['access_key'], conf['secret_access_key']) def get_all_queues(self): """Returns all queues with global prefix""" for queue in sorted(self._conn.get_all_queues(self._global_prefix), key=lambda x: x.name): yield MessageQueue(self._conf, queue.name) def get_queues(self): """Returns read_queues in priority order""" for prefix in self._read_queues: for queue in sorted(self._conn.get_all_queues(prefix), key=lambda x: x.name): if queue.name.endswith('_failure'): continue yield MessageQueue(self._conf, queue.name)
class QueueReader(base.QueueReader): """Reads messages from work queues, respecting priority""" def __init__(self, conf): super(QueueReader, self).__init__(conf) self._conn = SQSConnection( conf['access_key'], conf['secret_access_key']) def get_all_queues(self): """Returns all queues with global prefix""" for queue in sorted(self._conn.get_all_queues( self._global_prefix), key=lambda x: x.name): yield MessageQueue(self._conf, queue.name) def get_queues(self): """Returns read_queues in priority order""" for prefix in self._read_queues: for queue in sorted( self._conn.get_all_queues(prefix), key=lambda x: x.name): if queue.name.endswith('_failure'): continue yield MessageQueue(self._conf, queue.name)
class SQS(object): def __init__(self, config="config.ini"): if isinstance(config, basestring): config = credentials.ConfigFileCredentials(config) elif not isinstance(config, credentials.Credentials): raise TypeError("Unsupported config parameter type") aws_access_key_id, aws_secret_access_key, aws_queue = config.get_data() try: self.conn = SQSConnection(aws_access_key_id, aws_secret_access_key) self.set_queue(aws_queue) except: print 'Error connection' def get_all_queues(self): return self.conn.get_all_queues() def get_queue_attributes(self): return self.conn.get_queue_attributes(self.queue, attribute='All') def create_queue(self, queue, timeout): return self.conn.create_queue(queue, timeout) def set_queue(self, queue): self.queue = self.conn.get_queue(queue) return True def get_messages(self, limit=10): return self.queue.get_messages(limit) def count(self): #print "Count: %s" % self.queue.count() return self.queue.count() def write(self, data): m = Message() m.set_body(json.dumps(data)) return self.queue.write(m) def delete(self, id): #print "Eliminando %s" % id self.queue.delete_message(id) def clear(self): return self.queue.clear() def delete_queue(self): return self.conn.delete_queue(self.queue)
def test_queue_deletion_affects_full_queues(self): conn = SQSConnection() initial_count = len(conn.get_all_queues()) empty = conn.create_queue('empty%d' % int(time.time())) full = conn.create_queue('full%d' % int(time.time())) time.sleep(60) # Make sure they're both around. self.assertEqual(len(conn.get_all_queues()), initial_count + 2) # Put a message in the full queue. m1 = Message() m1.set_body('This is a test message.') full.write(m1) self.assertEqual(full.count(), 1) self.assertTrue(conn.delete_queue(empty)) # Here's the regression for the docs. SQS will delete a queue with # messages in it, no ``force_deletion`` needed. self.assertTrue(conn.delete_queue(full)) # Wait long enough for SQS to finally remove the queues. time.sleep(90) self.assertEqual(len(conn.get_all_queues()), initial_count)
def test_queue_deletion_affects_full_queues(self): conn = SQSConnection() initial_count = len(conn.get_all_queues()) empty = conn.create_queue("empty%d" % int(time.time())) full = conn.create_queue("full%d" % int(time.time())) time.sleep(60) # Make sure they're both around. self.assertEqual(len(conn.get_all_queues()), initial_count + 2) # Put a message in the full queue. m1 = Message() m1.set_body("This is a test message.") full.write(m1) self.assertEqual(full.count(), 1) self.assertTrue(conn.delete_queue(empty)) # Here's the regression for the docs. SQS will delete a queue with # messages in it, no ``force_deletion`` needed. self.assertTrue(conn.delete_queue(full)) # Wait long enough for SQS to finally remove the queues. time.sleep(90) self.assertEqual(len(conn.get_all_queues()), initial_count)
def dashboard(request): """ Graph SQS send statistics over time. """ cache_key = 'vhash:django_sqs_stats' cached_view = cache.get(cache_key) if cached_view: return cached_view region_name = getattr(settings, 'SQS_REGION', 'us-east-1') endpoint_name = getattr(settings, 'SQS_ENDPOINT', 'queue.amazonaws.com') sqs_conn = SQSConnection( settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, ) if region_name and endpoint_name: region = SQSRegionInfo(sqs_conn, region_name, endpoint_name) sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=region) queues = sqs_conn.get_all_queues() qas = {} for queue in queues: qas[queue] = sqs_conn.get_queue_attributes(queue) parse_attributes(qas) extra_context = { 'title': 'SQS Statistics', 'queues': queues, 'access_key': sqs_conn.gs_access_key_id, } response = render_to_response('django_sqs/queue_stats.html', extra_context, context_instance=RequestContext(request)) cache.set(cache_key, response, 60 * 1) # Cache for 1 minute return response
def dashboard(request): """ Graph SQS send statistics over time. """ cache_key = 'vhash:django_sqs_stats' cached_view = cache.get(cache_key) if cached_view: return cached_view region_name = getattr(settings, 'SQS_REGION', 'us-east-1') endpoint_name = getattr(settings, 'SQS_ENDPOINT', 'queue.amazonaws.com') sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY,) if region_name and endpoint_name: region = SQSRegionInfo(sqs_conn, region_name, endpoint_name) sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=region) queues = sqs_conn.get_all_queues() qas = {} for queue in queues: qas[queue] = sqs_conn.get_queue_attributes(queue) parse_attributes(qas) extra_context = { 'title': 'SQS Statistics', 'queues': queues, 'access_key': sqs_conn.gs_access_key_id, } response = render_to_response( 'django_sqs/queue_stats.html', extra_context, context_instance=RequestContext(request)) cache.set(cache_key, response, 60 * 1) # Cache for 1 minute return response
def timeout(self): from boto.sqs.message import Message if not self._queue and self._connected: try: from boto.sqs.connection import SQSConnection sqs_connection = SQSConnection(self._access_key, self._secret_access_key, **self._connection_kwargs) queues = sqs_connection.get_all_queues(prefix=self._queue_name) if len(queues) == 0: raise Exception('Queue %s does not exist' % self._queue_name) self._queue = queues[0] except Exception as e: LOGGER.error('Could not connect to logging queue %s' % self._queue_name, exc_info=e) self._connected = False while len(self._records_to_emit) and self._connected: record = self._records_to_emit.pop() self._queue.write(Message(body=record))
class SqsApi( object ): def __init__( self ): self.conn = SQSConnection( aws_access_key_id = AWS_ACCESS_KEY, aws_secret_access_key = AWS_SECRET_KEY ) def create_queue( self, name = "MuscleOpsQ" ): return self.conn.create_queue( name ) def list_queues( self ): return self.conn.get_all_queues() def write_message( self, q, msg = "If you're sleepy and you know it; clap your hands!" ): m = RawMessage() m.set_body( msg ) q.write( m ) def read_message( self, q, msg = "If you're sleepy and you know it; clap your hands!" ): rs = q.get_messages() m = rs[ 0 ].get_body() if len( rs ) else msg return m
def test_1_basic(self): print '--- running SQSConnection tests ---' c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue('bad*queue*name') self.fail('queue name should have been bad') except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) time.sleep(60) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues+1 assert queue.count_slow() == 0 # check the visibility timeout t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') a = q.get_attributes('ApproximateNumberOfMessages') assert a.has_key('ApproximateNumberOfMessages') assert not a.has_key('VisibilityTimeout') a = q.get_attributes('VisibilityTimeout') assert not a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') # now change the visibility timeout timeout = 45 queue.set_timeout(timeout) time.sleep(60) t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now add a message message_body = 'This is a test\n' message = queue.new_message(message_body) queue.write(message) time.sleep(60) assert queue.count_slow() == 1 time.sleep(90) # now read the message from the queue with a 10 second timeout message = queue.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue.read() assert message == None # now wait 30 seconds and try again time.sleep(30) message = queue.read() assert message # now delete the message queue.delete_message(message) time.sleep(30) assert queue.count_slow() == 0 # try a batch write num_msgs = 10 msgs = [(i, 'This is message %d' % i, 0) for i in range(num_msgs)] queue.write_batch(msgs) # try to delete all of the messages using batch delete deleted = 0 while deleted < num_msgs: time.sleep(5) msgs = queue.get_messages(num_msgs) if msgs: br = queue.delete_message_batch(msgs) deleted += len(br.results) # create another queue so we can test force deletion # we will also test MHMessage with this queue queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) queue.set_message_class(MHMessage) time.sleep(30) # now add a couple of messages message = queue.new_message() message['foo'] = 'bar' queue.write(message) message_body = {'fie' : 'baz', 'foo' : 'bar'} message = queue.new_message(body=message_body) queue.write(message) time.sleep(30) m = queue.read() assert m['foo'] == 'bar' # now delete that queue and messages c.delete_queue(queue, True) print '--- tests completed ---'
def test_1_basic(self): print "--- running SQSConnection tests ---" c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue("bad*queue*name") self.fail("queue name should have been bad") except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = "test%d" % int(time.time()) timeout = 60 queue_1 = c.create_queue(queue_name, timeout) self.addCleanup(c.delete_queue, queue_1, True) time.sleep(60) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues + 1 assert queue_1.count_slow() == 0 # check the visibility timeout t = queue_1.get_timeout() assert t == timeout, "%d != %d" % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert "ApproximateNumberOfMessages" in a assert "VisibilityTimeout" in a a = q.get_attributes("ApproximateNumberOfMessages") assert "ApproximateNumberOfMessages" in a assert "VisibilityTimeout" not in a a = q.get_attributes("VisibilityTimeout") assert "ApproximateNumberOfMessages" not in a assert "VisibilityTimeout" in a # now change the visibility timeout timeout = 45 queue_1.set_timeout(timeout) time.sleep(60) t = queue_1.get_timeout() assert t == timeout, "%d != %d" % (t, timeout) # now add a message message_body = "This is a test\n" message = queue_1.new_message(message_body) queue_1.write(message) time.sleep(60) assert queue_1.count_slow() == 1 time.sleep(90) # now read the message from the queue with a 10 second timeout message = queue_1.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue_1.read() assert message == None # now wait 30 seconds and try again time.sleep(30) message = queue_1.read() assert message # now delete the message queue_1.delete_message(message) time.sleep(30) assert queue_1.count_slow() == 0 # try a batch write num_msgs = 10 msgs = [(i, "This is message %d" % i, 0) for i in range(num_msgs)] queue_1.write_batch(msgs) # try to delete all of the messages using batch delete deleted = 0 while deleted < num_msgs: time.sleep(5) msgs = queue_1.get_messages(num_msgs) if msgs: br = queue_1.delete_message_batch(msgs) deleted += len(br.results) # create another queue so we can test force deletion # we will also test MHMessage with this queue queue_name = "test%d" % int(time.time()) timeout = 60 queue_2 = c.create_queue(queue_name, timeout) self.addCleanup(c.delete_queue, queue_2, True) queue_2.set_message_class(MHMessage) time.sleep(30) # now add a couple of messages message = queue_2.new_message() message["foo"] = "bar" queue_2.write(message) message_body = {"fie": "baz", "foo": "bar"} message = queue_2.new_message(body=message_body) queue_2.write(message) time.sleep(30) m = queue_2.read() assert m["foo"] == "bar" print "--- tests completed ---"
class sqsUtil(object): '''Methods to use aws sqs''' def __init__(self,aws_access_key,aws_secret_key,region=None): '''Default constructor. Inputs: aws_access_key = access key provided by aws aws_secret_key = secret key associated with access key ''' self.__access_key = aws_access_key self.__secret_key = aws_secret_key self.__region = region self.__conn = None def open(self): '''Creates a new sqs connection with given access and secret key.''' self.__conn = SQSConnection(aws_access_key_id = self.__access_key, aws_secret_access_key = self.__secret_key) def createQueue(self,name,visibilityTimeout=30): '''Creates a new queue. Input: name = name to associate with queue visibilityTimeout = how long, in seconds, to hide a message before it can be read again. Returns: True if queue was successfully created. ''' if not self.__checkConn(): return False self.__conn.create_queue(name,visibilityTimeout) return True def getVisibilityTimeout(self,name): '''Get visibility time out of a queue. Inputs: name = name of queue ''' if not self.__checkConn(): return False def getQueue(self,name): '''Get a queue. Inputs: name = name of queue Returns: A queue object or None if queue object not found ''' if not self.__checkConn(): return None return self.__conn.get_queue(name) def getAllQueues(self,prefix=""): '''Get a list of all queues created. Inputs: prefix = restricts return to names that start with prefix Returns: A ResultSet object of queues. Calling id on one of these queues will get the id of the queue. ''' if not self.__checkConn(): return None return self.__conn.get_all_queues(prefix) def writeToQueue(self,name,message,messageClass=None): '''Write a message to a queue. Inputs: name = name of queue to write to message = message to write into queue messageClass = a custom message class to use to write into queue Returns: False if message was not written, true otherwise ''' if not self.__checkConn(): return False q = self.getQueue(name) if q is None: return False if messageClass is not None: q.set_message_class(messageClass) else: messageClass = Message m = messageClass() m.set_body(message) q.write(m) return True def readFromQueue(self,name,num=1,visibilityTimeout=None,delete=False): '''Read a message from a queue. Inputs: name = name of queue to read from num = number of messages to read from queue visibilityTimeout = setting this will change the time until the next reader can read the messages received delete = true will delete all messages read from the queue Returns: ResultSet object of messages read. Calling get_body() on one of these messages will return the message it contains. Returns None if queue was not successfully read. ''' if not self.__checkConn(): return None q = self.getQueue(name) if q is None: return None if visibilityTimeout: results = q.get_messages(num_messages=num, visibility_timeout=visibilityTimeout) else: results = q.get_messages(num_messages=num) if delete: for m in results: q.delete_message(m) return results def readFromQueueToFile(self,name,fileName,separator="\n----------\n"): '''Dump all the messages from a queue into a local file. Inputs: name = name of queue fileName = name of file to dump to separator = what to place between messages in the file Returns: True if queue was successfully dumped. ''' if not self.__checkConn(): return False q = self.getQueue(name) if q is None: return False #why does this boto function have an underscore??? q.dump_(fileName,sep=separator) return True def deleteQueue(self,name,clear=False): '''Delete a queue. name = name of queue clear = if True, will clear the queue before deleting it. This method will not delete the queue if it is not empty. Returns: True if queue was successfully deleted. A queue will not be deleted if it still contains messages. ''' if not self.__checkConn(): return False q = self.getQueue(name) if q is None: return False if clear: q.clear() return self.__conn.delete_queue(q) def __checkConn(self): '''Helper function to make sure SQS connection is open.''' if not self.__conn: print("[ERROR] must call open() before using this function.") return False return True
def test_1_basic(self): print '--- running SQSConnection tests ---' c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue('bad_queue_name') except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) time.sleep(60) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues + 1 assert queue.count_slow() == 0 # check the visibility timeout t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') a = q.get_attributes('ApproximateNumberOfMessages') assert a.has_key('ApproximateNumberOfMessages') assert not a.has_key('VisibilityTimeout') a = q.get_attributes('VisibilityTimeout') assert not a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') # now change the visibility timeout timeout = 45 queue.set_timeout(timeout) time.sleep(60) t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now add a message message_body = 'This is a test\n' message = queue.new_message(message_body) queue.write(message) time.sleep(30) assert queue.count_slow() == 1 time.sleep(30) # now read the message from the queue with a 10 second timeout message = queue.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue.read() assert message == None # now wait 30 seconds and try again time.sleep(30) message = queue.read() assert message if c.APIVersion == '2007-05-01': # now terminate the visibility timeout for this message message.change_visibility(0) # now see if we can read it in the queue message = queue.read() assert message # now delete the message queue.delete_message(message) time.sleep(30) assert queue.count_slow() == 0 # create another queue so we can test force deletion # we will also test MHMessage with this queue queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) queue.set_message_class(MHMessage) time.sleep(30) # now add a couple of messages message = queue.new_message() message['foo'] = 'bar' queue.write(message) message_body = {'fie': 'baz', 'foo': 'bar'} message = queue.new_message(body=message_body) queue.write(message) time.sleep(30) m = queue.read() assert m['foo'] == 'bar' # now delete that queue and messages c.delete_queue(queue, True) print '--- tests completed ---'
class Controller(object): """Controller to export various functionalities regarding AWS in general.""" def __init__(self): credentials = { 'aws_access_key_id': settings.ZING_AWS_ACCESS_KEY_ID, 'aws_secret_access_key': settings.ZING_AWS_SECRET_ACCESS_KEY } region = settings.ZING_AWS_REGION self.sqs = SQSConnection(region=get_sqs_region(region), **credentials) self.vpc = VPCConnection(**credentials) self.ec2 = boto.connect_ec2(**credentials) self._queue_cache = {} # Populate the queue cache. self.get_queues() # Time to be used for keep-alive queue polling. This effectively # reduces the amount of poll requests, cost and improves efficiency. self.wait_time_seconds = settings.ZING_SQS_WAIT_TIME_SECONDS def normalize_queue_name(self, name, prefix=None, table=CHARS_REPLACE_TABLE): """Normalize an queue name into a legal SQS queue name. .. note:: Normalize all queue access for consistency with initial ``Controller._queue_cache`` population. Reference: http://aws.amazon.com/sqs/faqs/#Are_there_name_restrictions """ if prefix is not None: name = '{}.{}'.format(prefix, name) return unicode(name).translate(table) def get_queue(self, queue_name, prefix=None, visibility_timeout=None): """Get a :cls:`~boto.sqs.queue.Queue`object. If a queue does not exist it will be created. .. note:: The message class is set to :cls:`zing.message.EventMessage`. """ queue_name = self.normalize_queue_name(queue_name, prefix) try: queue = self._queue_cache[queue_name] except KeyError: queue = self.sqs.get_queue(queue_name) if queue is None: queue = self.sqs.create_queue( queue_name, visibility_timeout, ) self._queue_cache[queue_name] = queue queue.set_message_class(EventMessage) return queue def get_queues(self): """Returns all queues known to the controller. Updates the queue cache! :returns: A dictionary of (name, queue_object). """ # SQS blows up when you try to create a new queue if one already # exists with a different visibility_timeout, so this prepopulates # the queue_cache to protect us from recreating queues that # are known to already exist. # NOTE: Does *not* filter by prefix but only by SQS account. queues = self.sqs.get_all_queues() for queue in queues: self._queue_cache[queue.name] = queue return self._queue_cache def get_messages(self, queue_name, prefix=None, limit=1): """Get ``limit`` messages from ``queue_name``. .. warning:: This method blocks for ``wait_time_seconds``! """ queue = self.get_queue(queue_name, prefix) result = queue.get_messages(limit, wait_time_seconds=self.wait_time_seconds) return result def ack_message(self, message, queue_name, prefix=None): """Acknowledge ``message`` on ``queue_name``. This method deletes the message from the queue. """ queue = self.get_queue(queue_name, prefix) queue.delete_message(message) def send_message(self, queue_name, topic, subject, prefix=None, **options): message = { 'topic': topic, 'subject': subject, 'options': options } queue = self.get_queue(queue_name, prefix) queue.write(make_message(message))
def test_1_basic(self): print '--- running SQSConnection tests ---' c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue('bad_queue_name') except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) time.sleep(30) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues+1 assert queue.count_slow() == 0 # check the visibility timeout t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') a = q.get_attributes('ApproximateNumberOfMessages') assert a.has_key('ApproximateNumberOfMessages') assert not a.has_key('VisibilityTimeout') a = q.get_attributes('VisibilityTimeout') assert not a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') # now change the visibility timeout timeout = 45 queue.set_timeout(timeout) time.sleep(30) t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now add a message message_body = 'This is a test\n' message = queue.new_message(message_body) queue.write(message) time.sleep(30) assert queue.count_slow() == 1 time.sleep(30) # now read the message from the queue with a 10 second timeout message = queue.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue.read() assert message == None # now wait 10 seconds and try again time.sleep(10) message = queue.read() assert message if c.APIVersion == '2007-05-01': # now terminate the visibility timeout for this message message.change_visibility(0) # now see if we can read it in the queue message = queue.read() assert message # now delete the message queue.delete_message(message) time.sleep(30) assert queue.count_slow() == 0 # create another queue so we can test force deletion # we will also test MHMessage with this queue queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) queue.set_message_class(MHMessage) time.sleep(30) # now add a couple of messages message = queue.new_message() message['foo'] = 'bar' queue.write(message) message_body = {'fie' : 'baz', 'foo' : 'bar'} message = queue.new_message(body=message_body) queue.write(message) time.sleep(30) m = queue.read() assert m['foo'] == 'bar' # now delete that queue and messages c.delete_queue(queue, True) print '--- tests completed ---'
def test_1_basic(self): print '--- running SQSConnection tests ---' c = SQSConnection() rs = c.get_all_queues() num_queues = 0 for q in rs: num_queues += 1 # try illegal name try: queue = c.create_queue('bad_queue_name') except SQSError: pass # now create one that should work and should be unique (i.e. a new one) queue_name = 'test%d' % int(time.time()) timeout = 60 queue = c.create_queue(queue_name, timeout) time.sleep(10) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues+1 assert queue.count_slow() == 0 # check the visibility timeout t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now try to get queue attributes a = q.get_attributes() assert a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') a = q.get_attributes('ApproximateNumberOfMessages') assert a.has_key('ApproximateNumberOfMessages') assert not a.has_key('VisibilityTimeout') a = q.get_attributes('VisibilityTimeout') assert not a.has_key('ApproximateNumberOfMessages') assert a.has_key('VisibilityTimeout') # now change the visibility timeout timeout = 45 queue.set_timeout(timeout) t = queue.get_timeout() assert t == timeout, '%d != %d' % (t, timeout) # now add a message message_body = 'This is a test\n' message = queue.new_message(message_body) queue.write(message) time.sleep(5) assert queue.count_slow() == 1 time.sleep(10) # now read the message from the queue with a 10 second timeout message = queue.read(visibility_timeout=10) assert message assert message.get_body() == message_body # now immediately try another read, shouldn't find anything message = queue.read() assert message == None # now wait 10 seconds and try again time.sleep(10) message = queue.read() assert message # now delete the message queue.delete_message(message) time.sleep(5) assert queue.count_slow() == 0 # now delete that queue c.delete_queue(queue) rs = c.get_all_queues() i = 0 for q in rs: i += 1 assert i == num_queues print '--- tests completed ---'
aws_key = '' aws_secret = '' # The alerting thresholds can be overridden via command line arguments limits = [MIN_MESSAGES, MAX_MESSAGES] for i in [2,3]: if len(sys.argv) >= i: try: limits[i-2] = int(sys.argv[i-1]) except ValueError: print 'status err argument "%s" is not a valid integer' % sys.argv[i-1] sys.exit(1) try: conn = SQSConnection(aws_key, aws_secret) queues = conn.get_all_queues() error_queues = [] total = 0 for queue in queues: count = queue.count() total += count if count < limits[0] or count > limits[1]: error_queues.append(queue.name) print 'metric %s int %d' % (queue.name, count) if len(error_queues) == 0: print 'status ok %d messages in all queues' % total else: s = '/'.join(error_queues) print 'status err %s contains an unexpected number of messages' % s
def _create_queue(self, name): sqs = SQSConnection(region=get_sqs_region('us-east-1')) self.assertEqual(len(sqs.get_all_queues()), 0) queue = sqs.create_queue(name) return queue