def test_empty_worker(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
with spawn_worker(Worker()):
# Worker is empty, not registering any queues
self.assertEqual(len(sqs.get_all_queues()), 0)
def test_worker_consumes_queue(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
queue = sqs.create_queue('test_events')
queue.write(make_message({'test': '1'}))
self.assertEqual(queue.count(), 1)
worker = Worker()
worker.ctrl.wait_time_seconds = 0.1
worker.idle_time_seconds = 0.1
worker.add_consumer('test_events', dummy_consumer)
with spawn_worker(worker):
time.sleep(.2)
self.assertEqual(queue.count(), 0)
queue.write(make_message({'test': '2'}))
self.assertEqual(queue.count(), 1)
time.sleep(.2)
self.assertEqual(queue.count(), 0)
class SqsApi(object):
def __init__(self):
self.conn = SQSConnection(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
def create_queue(self, name="MuscleOpsQ"):
return self.conn.create_queue(name)
def list_queues(self):
return self.conn.get_all_queues()
def write_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"
):
m = RawMessage()
m.set_body(msg)
q.write(m)
def read_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"):
rs = q.get_messages()
m = rs[0].get_body() if len(rs) else msg
return m
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:], 'hcq:o:t:r:',
['help', 'clear', 'queue=', 'output=', 'timeout=', 'region='])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
region = ''
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
if o in ('-r', '--region'):
region = a
if region:
c = boto.sqs.connect_to_region(region)
if c is None:
print 'Invalid region (%s)' % region
sys.exit(1)
else:
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
else:
try:
rs = c.get_all_queues()
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
for q in rs:
if clear:
n = q.clear()
print 'clearing %d messages from %s' % (n, q.id)
elif output_file:
q.dump(output_file)
else:
print q.id, q.count(vtimeout=timeout)
def test_worker_creates_queue(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
worker = Worker()
worker.ctrl.wait_time_seconds = 0.1
worker.idle_time_seconds = 0.1
worker.add_consumer('test_events', dummy_consumer)
with spawn_worker(worker):
time.sleep(.2)
all_queues = sqs.get_all_queues()
self.assertEqual(len(all_queues), 1)
self.assertEqual(
all_queues[0].name,
'test_events'
)
class QueueReader(base.QueueReader):
"""Reads messages from work queues, respecting priority"""
def __init__(self, conf):
super(QueueReader, self).__init__(conf)
self._conn = SQSConnection(conf['access_key'],
conf['secret_access_key'])
def get_all_queues(self):
"""Returns all queues with global prefix"""
for queue in sorted(self._conn.get_all_queues(self._global_prefix),
key=lambda x: x.name):
yield MessageQueue(self._conf, queue.name)
def get_queues(self):
"""Returns read_queues in priority order"""
for prefix in self._read_queues:
for queue in sorted(self._conn.get_all_queues(prefix),
key=lambda x: x.name):
if queue.name.endswith('_failure'):
continue
yield MessageQueue(self._conf, queue.name)
class QueueReader(base.QueueReader):
"""Reads messages from work queues, respecting priority"""
def __init__(self, conf):
super(QueueReader, self).__init__(conf)
self._conn = SQSConnection(
conf['access_key'], conf['secret_access_key'])
def get_all_queues(self):
"""Returns all queues with global prefix"""
for queue in sorted(self._conn.get_all_queues(
self._global_prefix), key=lambda x: x.name):
yield MessageQueue(self._conf, queue.name)
def get_queues(self):
"""Returns read_queues in priority order"""
for prefix in self._read_queues:
for queue in sorted(
self._conn.get_all_queues(prefix), key=lambda x: x.name):
if queue.name.endswith('_failure'):
continue
yield MessageQueue(self._conf, queue.name)
class SQS(object):
def __init__(self, config="config.ini"):
if isinstance(config, basestring):
config = credentials.ConfigFileCredentials(config)
elif not isinstance(config, credentials.Credentials):
raise TypeError("Unsupported config parameter type")
aws_access_key_id, aws_secret_access_key, aws_queue = config.get_data()
try:
self.conn = SQSConnection(aws_access_key_id, aws_secret_access_key)
self.set_queue(aws_queue)
except:
print 'Error connection'
def get_all_queues(self):
return self.conn.get_all_queues()
def get_queue_attributes(self):
return self.conn.get_queue_attributes(self.queue, attribute='All')
def create_queue(self, queue, timeout):
return self.conn.create_queue(queue, timeout)
def set_queue(self, queue):
self.queue = self.conn.get_queue(queue)
return True
def get_messages(self, limit=10):
return self.queue.get_messages(limit)
def count(self):
#print "Count: %s" % self.queue.count()
return self.queue.count()
def write(self, data):
m = Message()
m.set_body(json.dumps(data))
return self.queue.write(m)
def delete(self, id):
#print "Eliminando %s" % id
self.queue.delete_message(id)
def clear(self):
return self.queue.clear()
def delete_queue(self):
return self.conn.delete_queue(self.queue)
def test_queue_deletion_affects_full_queues(self):
conn = SQSConnection()
initial_count = len(conn.get_all_queues())
empty = conn.create_queue('empty%d' % int(time.time()))
full = conn.create_queue('full%d' % int(time.time()))
time.sleep(60)
# Make sure they're both around.
self.assertEqual(len(conn.get_all_queues()), initial_count + 2)
# Put a message in the full queue.
m1 = Message()
m1.set_body('This is a test message.')
full.write(m1)
self.assertEqual(full.count(), 1)
self.assertTrue(conn.delete_queue(empty))
# Here's the regression for the docs. SQS will delete a queue with
# messages in it, no ``force_deletion`` needed.
self.assertTrue(conn.delete_queue(full))
# Wait long enough for SQS to finally remove the queues.
time.sleep(90)
self.assertEqual(len(conn.get_all_queues()), initial_count)
def test_queue_deletion_affects_full_queues(self):
conn = SQSConnection()
initial_count = len(conn.get_all_queues())
empty = conn.create_queue("empty%d" % int(time.time()))
full = conn.create_queue("full%d" % int(time.time()))
time.sleep(60)
# Make sure they're both around.
self.assertEqual(len(conn.get_all_queues()), initial_count + 2)
# Put a message in the full queue.
m1 = Message()
m1.set_body("This is a test message.")
full.write(m1)
self.assertEqual(full.count(), 1)
self.assertTrue(conn.delete_queue(empty))
# Here's the regression for the docs. SQS will delete a queue with
# messages in it, no ``force_deletion`` needed.
self.assertTrue(conn.delete_queue(full))
# Wait long enough for SQS to finally remove the queues.
time.sleep(90)
self.assertEqual(len(conn.get_all_queues()), initial_count)
def dashboard(request):
"""
Graph SQS send statistics over time.
"""
cache_key = 'vhash:django_sqs_stats'
cached_view = cache.get(cache_key)
if cached_view:
return cached_view
region_name = getattr(settings, 'SQS_REGION', 'us-east-1')
endpoint_name = getattr(settings, 'SQS_ENDPOINT', 'queue.amazonaws.com')
sqs_conn = SQSConnection(
settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY,
)
if region_name and endpoint_name:
region = SQSRegionInfo(sqs_conn, region_name, endpoint_name)
sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY,
region=region)
queues = sqs_conn.get_all_queues()
qas = {}
for queue in queues:
qas[queue] = sqs_conn.get_queue_attributes(queue)
parse_attributes(qas)
extra_context = {
'title': 'SQS Statistics',
'queues': queues,
'access_key': sqs_conn.gs_access_key_id,
}
response = render_to_response('django_sqs/queue_stats.html',
extra_context,
context_instance=RequestContext(request))
cache.set(cache_key, response, 60 * 1) # Cache for 1 minute
return response
def dashboard(request):
"""
Graph SQS send statistics over time.
"""
cache_key = 'vhash:django_sqs_stats'
cached_view = cache.get(cache_key)
if cached_view:
return cached_view
region_name = getattr(settings, 'SQS_REGION', 'us-east-1')
endpoint_name = getattr(settings, 'SQS_ENDPOINT', 'queue.amazonaws.com')
sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY,)
if region_name and endpoint_name:
region = SQSRegionInfo(sqs_conn, region_name, endpoint_name)
sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=region)
queues = sqs_conn.get_all_queues()
qas = {}
for queue in queues:
qas[queue] = sqs_conn.get_queue_attributes(queue)
parse_attributes(qas)
extra_context = {
'title': 'SQS Statistics',
'queues': queues,
'access_key': sqs_conn.gs_access_key_id,
}
response = render_to_response(
'django_sqs/queue_stats.html',
extra_context,
context_instance=RequestContext(request))
cache.set(cache_key, response, 60 * 1) # Cache for 1 minute
return response
def timeout(self):
from boto.sqs.message import Message
if not self._queue and self._connected:
try:
from boto.sqs.connection import SQSConnection
sqs_connection = SQSConnection(self._access_key,
self._secret_access_key,
**self._connection_kwargs)
queues = sqs_connection.get_all_queues(prefix=self._queue_name)
if len(queues) == 0:
raise Exception('Queue %s does not exist' %
self._queue_name)
self._queue = queues[0]
except Exception as e:
LOGGER.error('Could not connect to logging queue %s' %
self._queue_name,
exc_info=e)
self._connected = False
while len(self._records_to_emit) and self._connected:
record = self._records_to_emit.pop()
self._queue.write(Message(body=record))
class SqsApi( object ):
def __init__( self ):
self.conn = SQSConnection(
aws_access_key_id = AWS_ACCESS_KEY,
aws_secret_access_key = AWS_SECRET_KEY
)
def create_queue( self, name = "MuscleOpsQ" ):
return self.conn.create_queue( name )
def list_queues( self ):
return self.conn.get_all_queues()
def write_message( self, q, msg = "If you're sleepy and you know it; clap your hands!" ):
m = RawMessage()
m.set_body( msg )
q.write( m )
def read_message( self, q, msg = "If you're sleepy and you know it; clap your hands!" ):
rs = q.get_messages()
m = rs[ 0 ].get_body() if len( rs ) else msg
return m
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad*queue*name')
self.fail('queue name should have been bad')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues+1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(60)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(60)
assert queue.count_slow() == 1
time.sleep(90)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# try a batch write
num_msgs = 10
msgs = [(i, 'This is message %d' % i, 0) for i in range(num_msgs)]
queue.write_batch(msgs)
# try to delete all of the messages using batch delete
deleted = 0
while deleted < num_msgs:
time.sleep(5)
msgs = queue.get_messages(num_msgs)
if msgs:
br = queue.delete_message_batch(msgs)
deleted += len(br.results)
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie' : 'baz', 'foo' : 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
def test_1_basic(self):
print "--- running SQSConnection tests ---"
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue("bad*queue*name")
self.fail("queue name should have been bad")
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = "test%d" % int(time.time())
timeout = 60
queue_1 = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue_1, True)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues + 1
assert queue_1.count_slow() == 0
# check the visibility timeout
t = queue_1.get_timeout()
assert t == timeout, "%d != %d" % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert "ApproximateNumberOfMessages" in a
assert "VisibilityTimeout" in a
a = q.get_attributes("ApproximateNumberOfMessages")
assert "ApproximateNumberOfMessages" in a
assert "VisibilityTimeout" not in a
a = q.get_attributes("VisibilityTimeout")
assert "ApproximateNumberOfMessages" not in a
assert "VisibilityTimeout" in a
# now change the visibility timeout
timeout = 45
queue_1.set_timeout(timeout)
time.sleep(60)
t = queue_1.get_timeout()
assert t == timeout, "%d != %d" % (t, timeout)
# now add a message
message_body = "This is a test\n"
message = queue_1.new_message(message_body)
queue_1.write(message)
time.sleep(60)
assert queue_1.count_slow() == 1
time.sleep(90)
# now read the message from the queue with a 10 second timeout
message = queue_1.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue_1.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue_1.read()
assert message
# now delete the message
queue_1.delete_message(message)
time.sleep(30)
assert queue_1.count_slow() == 0
# try a batch write
num_msgs = 10
msgs = [(i, "This is message %d" % i, 0) for i in range(num_msgs)]
queue_1.write_batch(msgs)
# try to delete all of the messages using batch delete
deleted = 0
while deleted < num_msgs:
time.sleep(5)
msgs = queue_1.get_messages(num_msgs)
if msgs:
br = queue_1.delete_message_batch(msgs)
deleted += len(br.results)
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = "test%d" % int(time.time())
timeout = 60
queue_2 = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue_2, True)
queue_2.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue_2.new_message()
message["foo"] = "bar"
queue_2.write(message)
message_body = {"fie": "baz", "foo": "bar"}
message = queue_2.new_message(body=message_body)
queue_2.write(message)
time.sleep(30)
m = queue_2.read()
assert m["foo"] == "bar"
print "--- tests completed ---"
class sqsUtil(object):
'''Methods to use aws sqs'''
def __init__(self,aws_access_key,aws_secret_key,region=None):
'''Default constructor.
Inputs:
aws_access_key = access key provided by aws
aws_secret_key = secret key associated with access key
'''
self.__access_key = aws_access_key
self.__secret_key = aws_secret_key
self.__region = region
self.__conn = None
def open(self):
'''Creates a new sqs connection with given access and secret key.'''
self.__conn = SQSConnection(aws_access_key_id = self.__access_key,
aws_secret_access_key = self.__secret_key)
def createQueue(self,name,visibilityTimeout=30):
'''Creates a new queue.
Input:
name = name to associate with queue
visibilityTimeout = how long, in seconds, to hide a message
before it can be read again.
Returns:
True if queue was successfully created.
'''
if not self.__checkConn(): return False
self.__conn.create_queue(name,visibilityTimeout)
return True
def getVisibilityTimeout(self,name):
'''Get visibility time out of a queue.
Inputs:
name = name of queue
'''
if not self.__checkConn(): return False
def getQueue(self,name):
'''Get a queue.
Inputs:
name = name of queue
Returns:
A queue object or None if queue object not found
'''
if not self.__checkConn(): return None
return self.__conn.get_queue(name)
def getAllQueues(self,prefix=""):
'''Get a list of all queues created.
Inputs:
prefix = restricts return to names that start with prefix
Returns:
A ResultSet object of queues. Calling id on one of these queues
will get the id of the queue.
'''
if not self.__checkConn(): return None
return self.__conn.get_all_queues(prefix)
def writeToQueue(self,name,message,messageClass=None):
'''Write a message to a queue.
Inputs:
name = name of queue to write to
message = message to write into queue
messageClass = a custom message class to use to write into queue
Returns:
False if message was not written, true otherwise
'''
if not self.__checkConn(): return False
q = self.getQueue(name)
if q is None: return False
if messageClass is not None:
q.set_message_class(messageClass)
else:
messageClass = Message
m = messageClass()
m.set_body(message)
q.write(m)
return True
def readFromQueue(self,name,num=1,visibilityTimeout=None,delete=False):
'''Read a message from a queue.
Inputs:
name = name of queue to read from
num = number of messages to read from queue
visibilityTimeout = setting this will change the time until the
next reader can read the messages received
delete = true will delete all messages read from the queue
Returns:
ResultSet object of messages read. Calling get_body() on one of
these messages will return the message it contains. Returns
None if queue was not successfully read.
'''
if not self.__checkConn(): return None
q = self.getQueue(name)
if q is None: return None
if visibilityTimeout:
results = q.get_messages(num_messages=num,
visibility_timeout=visibilityTimeout)
else:
results = q.get_messages(num_messages=num)
if delete:
for m in results:
q.delete_message(m)
return results
def readFromQueueToFile(self,name,fileName,separator="\n----------\n"):
'''Dump all the messages from a queue into a local file.
Inputs:
name = name of queue
fileName = name of file to dump to
separator = what to place between messages in the file
Returns:
True if queue was successfully dumped.
'''
if not self.__checkConn(): return False
q = self.getQueue(name)
if q is None: return False
#why does this boto function have an underscore???
q.dump_(fileName,sep=separator)
return True
def deleteQueue(self,name,clear=False):
'''Delete a queue.
name = name of queue
clear = if True, will clear the queue before deleting it. This method
will not delete the queue if it is not empty.
Returns:
True if queue was successfully deleted. A queue will not be deleted
if it still contains messages.
'''
if not self.__checkConn(): return False
q = self.getQueue(name)
if q is None: return False
if clear:
q.clear()
return self.__conn.delete_queue(q)
def __checkConn(self):
'''Helper function to make sure SQS connection is open.'''
if not self.__conn:
print("[ERROR] must call open() before using this function.")
return False
return True
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad_queue_name')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues + 1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(60)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(30)
assert queue.count_slow() == 1
time.sleep(30)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue.read()
assert message
if c.APIVersion == '2007-05-01':
# now terminate the visibility timeout for this message
message.change_visibility(0)
# now see if we can read it in the queue
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie': 'baz', 'foo': 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
class Controller(object):
"""Controller to export various functionalities regarding AWS in general."""
def __init__(self):
credentials = {
'aws_access_key_id': settings.ZING_AWS_ACCESS_KEY_ID,
'aws_secret_access_key': settings.ZING_AWS_SECRET_ACCESS_KEY
}
region = settings.ZING_AWS_REGION
self.sqs = SQSConnection(region=get_sqs_region(region), **credentials)
self.vpc = VPCConnection(**credentials)
self.ec2 = boto.connect_ec2(**credentials)
self._queue_cache = {}
# Populate the queue cache.
self.get_queues()
# Time to be used for keep-alive queue polling. This effectively
# reduces the amount of poll requests, cost and improves efficiency.
self.wait_time_seconds = settings.ZING_SQS_WAIT_TIME_SECONDS
def normalize_queue_name(self, name, prefix=None, table=CHARS_REPLACE_TABLE):
"""Normalize an queue name into a legal SQS queue name.
.. note::
Normalize all queue access for consistency with initial
``Controller._queue_cache`` population.
Reference: http://aws.amazon.com/sqs/faqs/#Are_there_name_restrictions
"""
if prefix is not None:
name = '{}.{}'.format(prefix, name)
return unicode(name).translate(table)
def get_queue(self, queue_name, prefix=None, visibility_timeout=None):
"""Get a :cls:`~boto.sqs.queue.Queue`object.
If a queue does not exist it will be created.
.. note::
The message class is set to :cls:`zing.message.EventMessage`.
"""
queue_name = self.normalize_queue_name(queue_name, prefix)
try:
queue = self._queue_cache[queue_name]
except KeyError:
queue = self.sqs.get_queue(queue_name)
if queue is None:
queue = self.sqs.create_queue(
queue_name,
visibility_timeout,
)
self._queue_cache[queue_name] = queue
queue.set_message_class(EventMessage)
return queue
def get_queues(self):
"""Returns all queues known to the controller.
Updates the queue cache!
:returns: A dictionary of (name, queue_object).
"""
# SQS blows up when you try to create a new queue if one already
# exists with a different visibility_timeout, so this prepopulates
# the queue_cache to protect us from recreating queues that
# are known to already exist.
# NOTE: Does *not* filter by prefix but only by SQS account.
queues = self.sqs.get_all_queues()
for queue in queues:
self._queue_cache[queue.name] = queue
return self._queue_cache
def get_messages(self, queue_name, prefix=None, limit=1):
"""Get ``limit`` messages from ``queue_name``.
.. warning::
This method blocks for ``wait_time_seconds``!
"""
queue = self.get_queue(queue_name, prefix)
result = queue.get_messages(limit, wait_time_seconds=self.wait_time_seconds)
return result
def ack_message(self, message, queue_name, prefix=None):
"""Acknowledge ``message`` on ``queue_name``.
This method deletes the message from the queue.
"""
queue = self.get_queue(queue_name, prefix)
queue.delete_message(message)
def send_message(self, queue_name, topic, subject, prefix=None, **options):
message = {
'topic': topic,
'subject': subject,
'options': options
}
queue = self.get_queue(queue_name, prefix)
queue.write(make_message(message))
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad_queue_name')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(30)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues+1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(30)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(30)
assert queue.count_slow() == 1
time.sleep(30)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 10 seconds and try again
time.sleep(10)
message = queue.read()
assert message
if c.APIVersion == '2007-05-01':
# now terminate the visibility timeout for this message
message.change_visibility(0)
# now see if we can read it in the queue
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie' : 'baz', 'foo' : 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad_queue_name')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(10)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues+1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(5)
assert queue.count_slow() == 1
time.sleep(10)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 10 seconds and try again
time.sleep(10)
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(5)
assert queue.count_slow() == 0
# now delete that queue
c.delete_queue(queue)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues
print '--- tests completed ---'
aws_key = ''
aws_secret = ''
# The alerting thresholds can be overridden via command line arguments
limits = [MIN_MESSAGES, MAX_MESSAGES]
for i in [2,3]:
if len(sys.argv) >= i:
try:
limits[i-2] = int(sys.argv[i-1])
except ValueError:
print 'status err argument "%s" is not a valid integer' % sys.argv[i-1]
sys.exit(1)
try:
conn = SQSConnection(aws_key, aws_secret)
queues = conn.get_all_queues()
error_queues = []
total = 0
for queue in queues:
count = queue.count()
total += count
if count < limits[0] or count > limits[1]:
error_queues.append(queue.name)
print 'metric %s int %d' % (queue.name, count)
if len(error_queues) == 0:
print 'status ok %d messages in all queues' % total
else:
s = '/'.join(error_queues)
print 'status err %s contains an unexpected number of messages' % s
def _create_queue(self, name):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
queue = sqs.create_queue(name)
return queue
