def test_create_dead_letter_queue(sqs_stubber, unique_queue_name):
"""
Test that creating a queue with an associated dead-letter queue results in
the source queue being listed in the dead-letter queue's source queue list.
A dead-letter queue is any queue that is designated as a dead-letter target
by another queue's redrive policy.
"""
dl_queue_name = unique_queue_name + '_my_lost_messages'
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': dl_queue_name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + dl_queue_name})
dl_queue = queue_wrapper.create_queue(dl_queue_name)
sqs_stubber.add_response('get_queue_attributes',
expected_params={
'AttributeNames': ['All'],
'QueueUrl': 'url' + dl_queue_name
},
service_response={
'Attributes': {
'QueueArn': 'arn:' + unique_queue_name
}
})
# The redrive policy must be in JSON format. Note that its attribute names
# start with lowercase letters.
attributes = {
'RedrivePolicy':
json.dumps({
'deadLetterTargetArn': dl_queue.attributes['QueueArn'],
'maxReceiveCount': str(5)
})
}
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': unique_queue_name,
'Attributes': attributes
},
service_response={'QueueUrl': 'url' + unique_queue_name})
sqs_stubber.add_response(
'list_dead_letter_source_queues',
expected_params={'QueueUrl': 'url' + dl_queue_name},
service_response={'queueUrls': ['url' + unique_queue_name]})
queue = queue_wrapper.create_queue(unique_queue_name, attributes)
sources = list(dl_queue.dead_letter_source_queues.all())
assert queue in sources
if sqs_stubber.needs_cleanup:
queue_wrapper.remove_queue(queue)
queue_wrapper.remove_queue(dl_queue)
def test_get_queues_prefix(sqs_stubber, unique_queue_name):
"""
Test that creating some queues with a unique prefix, then retrieving a list of
queues that have that unique prefix, returns a list that contains exactly the
newly created queues.
"""
created_queues = []
for ind in range(1, 4):
name = unique_queue_name + str(ind)
sqs_stubber.add_response('create_queue',
expected_params={
'QueueName': name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + name})
created_queues.append(queue_wrapper.create_queue(name))
sqs_stubber.add_response(
'list_queues',
expected_params={'QueueNamePrefix': unique_queue_name},
service_response={'QueueUrls': [q.url for q in created_queues]})
gotten_queues = queue_wrapper.get_queues(unique_queue_name)
assert created_queues == gotten_queues
if sqs_stubber.needs_cleanup:
for queue in created_queues:
queue_wrapper.remove_queue(queue)
def test_get_queues(make_stubber, make_unique_name):
"""
Test that creating some queues, then retrieving all the queues, returns a list
that contains at least all of the newly created queues.
"""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
queue_name = make_unique_name('queue')
created_queues = []
for ind in range(1, 4):
name = queue_name + str(ind)
sqs_stubber.stub_create_queue(name, {}, 'url-' + name)
created_queues.append(queue_wrapper.create_queue(name))
sqs_stubber.stub_list_queues([q.url for q in created_queues])
# When running against the real AWS service, SQS may not return new queues right
# away. Use a simple exponential backoff to retry until it succeeds.
retries = 0
intersect_queues = []
while not intersect_queues and retries <= 5:
time.sleep(min(retries * 2, 32))
gotten_queues = queue_wrapper.get_queues()
intersect_queues = [cq for cq in created_queues if cq in gotten_queues]
retries += 1
assert created_queues == intersect_queues
if not sqs_stubber.use_stubs:
for queue in created_queues:
queue_wrapper.remove_queue(queue)
def test_create_dead_letter_queue(make_stubber, make_unique_name):
"""
Test that creating a queue with an associated dead-letter queue results in
the source queue being listed in the dead-letter queue's source queue list.
A dead-letter queue is any queue that is designated as a dead-letter target
by another queue's redrive policy.
"""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
dl_queue_name = make_unique_name('queue') + '_my_lost_messages'
dl_url = 'url-' + dl_queue_name
queue_name = make_unique_name('queue')
url = 'url-' + queue_name
# Create a dead-letter queue.
sqs_stubber.stub_create_queue(dl_queue_name, {}, dl_url)
dl_queue = queue_wrapper.create_queue(dl_queue_name)
sqs_stubber.stub_get_queue_attributes(dl_url, 'arn:' + dl_queue_name)
# Create the source queue that sends dead-letter messages to the dead-letter queue.
# The redrive policy must be in JSON format. Note that its attribute names
# start with lowercase letters.
redrive_attributes = {
'RedrivePolicy':
json.dumps({
'deadLetterTargetArn': dl_queue.attributes['QueueArn'],
'maxReceiveCount': str(5)
})
}
sqs_stubber.stub_create_queue(queue_name, redrive_attributes, url)
sqs_stubber.stub_list_dead_letter_source_queues(dl_url, [url])
queue = queue_wrapper.create_queue(queue_name, redrive_attributes)
sources = list(dl_queue.dead_letter_source_queues.all())
assert queue in sources
if not sqs_stubber.use_stubs:
queue_wrapper.remove_queue(queue)
queue_wrapper.remove_queue(dl_queue)
def test_create_standard_queue(make_stubber, make_unique_name, attributes):
"""Test that creating a standard queue returns a queue with the expected
form of URL."""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
queue_name = make_unique_name('queue')
url = 'url-' + queue_name
sqs_stubber.stub_create_queue(queue_name, attributes, url)
queue = queue_wrapper.create_queue(queue_name, attributes)
assert queue_name in queue.url
if not sqs_stubber.use_stubs:
queue_wrapper.remove_queue(queue)
def test_create_fifo_queue(make_stubber, make_unique_name, attributes):
"""Test that creating a FIFO queue returns a queue with the expected form of URL."""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
# FIFO queues require a '.fifo' suffix on the queue name.
queue_name = make_unique_name('queue') + '.fifo'
attributes['FifoQueue'] = str(True)
url = 'url-' + queue_name
sqs_stubber.stub_create_queue(queue_name, attributes, url)
queue = queue_wrapper.create_queue(queue_name, attributes)
assert queue.url.endswith(queue_name)
if not sqs_stubber.use_stubs:
queue_wrapper.remove_queue(queue)
def fixture_queue_stub(sqs_stubber, unique_queue_name):
"""Create a standard queue to use for message testing. In non-stubbed
scenarios, delete the queue after the test completes."""
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': unique_queue_name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + unique_queue_name}
)
queue = queue_wrapper.create_queue(unique_queue_name)
yield queue
if sqs_stubber.needs_cleanup:
queue.delete()
def test_get_queue(make_stubber, make_unique_name):
"""Test that creating a queue and then getting it by name returns the same queue."""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
queue_name = make_unique_name('queue')
url = 'url-' + queue_name
sqs_stubber.stub_create_queue(queue_name, {}, url)
sqs_stubber.stub_get_queue_url(queue_name, url)
created = queue_wrapper.create_queue(queue_name)
gotten = queue_wrapper.get_queue(queue_name)
assert created == gotten
if not sqs_stubber.use_stubs:
queue_wrapper.remove_queue(created)
def test_create_standard_queue(sqs_stubber, unique_queue_name, attributes):
"""Test that creating a standard queue returns a queue with the expected
form of URL."""
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': unique_queue_name,
'Attributes': attributes
},
service_response={'QueueUrl': 'url' + unique_queue_name})
queue = queue_wrapper.create_queue(unique_queue_name, attributes)
assert unique_queue_name in queue.url
if sqs_stubber.needs_cleanup:
queue_wrapper.remove_queue(queue)
def test_remove_queue(make_stubber, make_unique_name):
"""Test that creating a queue, deleting it, then trying to get it raises
an exception because the queue no longer exists."""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
queue_name = make_unique_name('queue')
url = 'url-' + queue_name
sqs_stubber.stub_create_queue(queue_name, {}, url)
sqs_stubber.stub_delete_queue(url)
sqs_stubber.stub_get_queue_url(
queue_name, url, error_code='AWS.SimpleQueueService.NonExistentQueue')
queue = queue_wrapper.create_queue(queue_name)
queue_wrapper.remove_queue(queue)
with pytest.raises(ClientError) as exc_info:
queue_wrapper.get_queue(queue_name)
assert exc_info.value.response['Error']['Code'] == \
'AWS.SimpleQueueService.NonExistentQueue'
def test_create_fifo_queue(sqs_stubber, unique_queue_name, attributes):
"""Test that creating a FIFO queue returns a queue with the expected form of URL."""
# FIFO queues require a '.fifo' suffix on the queue name.
name = unique_queue_name + '.fifo'
attributes['FifoQueue'] = str(True)
sqs_stubber.add_response('create_queue',
expected_params={
'QueueName': name,
'Attributes': attributes
},
service_response={'QueueUrl': 'url' + name})
queue = queue_wrapper.create_queue(name, attributes)
assert name in queue.url
if sqs_stubber.needs_cleanup:
queue_wrapper.remove_queue(queue)
def test_get_queue(sqs_stubber, unique_queue_name):
"""Test that creating a queue and then getting it by name returns the same queue."""
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': unique_queue_name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + unique_queue_name})
sqs_stubber.add_response(
'get_queue_url',
expected_params={'QueueName': unique_queue_name},
service_response={'QueueUrl': 'url' + unique_queue_name})
created = queue_wrapper.create_queue(unique_queue_name)
gotten = queue_wrapper.get_queue(unique_queue_name)
assert created == gotten
if sqs_stubber.needs_cleanup:
queue_wrapper.remove_queue(created)
def test_get_queues_prefix(make_stubber, make_unique_name):
"""
Test that creating some queues with a unique prefix, then retrieving a list of
queues that have that unique prefix, returns a list that contains exactly the
newly created queues.
"""
sqs_stubber = make_stubber(queue_wrapper.sqs.meta.client)
queue_name = make_unique_name('queue')
created_queues = []
for ind in range(1, 4):
name = queue_name + str(ind)
sqs_stubber.stub_create_queue(name, {}, 'url-' + name)
created_queues.append(queue_wrapper.create_queue(name))
sqs_stubber.stub_list_queues([q.url for q in created_queues], queue_name)
gotten_queues = queue_wrapper.get_queues(queue_name)
assert created_queues == gotten_queues
if not sqs_stubber.use_stubs:
for queue in created_queues:
queue_wrapper.remove_queue(queue)
def test_remove_queue(sqs_stubber, unique_queue_name):
"""Test that creating a queue, deleting it, then trying to get it raises
an exception because the queue no longer exists."""
sqs_stubber.add_response(
'create_queue',
expected_params={
'QueueName': unique_queue_name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + unique_queue_name})
sqs_stubber.add_response(
'delete_queue',
expected_params={'QueueUrl': 'url' + unique_queue_name},
service_response={})
sqs_stubber.add_client_error(
'get_queue_url',
expected_params={'QueueName': unique_queue_name},
service_error_code='AWS.SimpleQueueService.NonExistentQueue')
queue = queue_wrapper.create_queue(unique_queue_name)
queue_wrapper.remove_queue(queue)
with pytest.raises(sqs_stubber.client.exceptions.QueueDoesNotExist):
queue_wrapper.get_queue(unique_queue_name)
def test_get_queues(sqs_stubber, unique_queue_name):
"""
Test that creating some queues, then retrieving all the queues, returns a list
that contains at least all of the newly created queues.
"""
created_queues = []
for ind in range(1, 4):
name = unique_queue_name + str(ind)
sqs_stubber.add_response('create_queue',
expected_params={
'QueueName': name,
'Attributes': {}
},
service_response={'QueueUrl': 'url' + name})
created_queues.append(queue_wrapper.create_queue(name))
sqs_stubber.add_response(
'list_queues',
expected_params={},
service_response={'QueueUrls': [q.url for q in created_queues]})
# When running against the real AWS service, SQS may not return new queues right
# away. Use a simple exponential backoff to retry until it succeeds.
retries = 0
intersect_queues = []
while not intersect_queues and retries <= 5:
time.sleep(min(retries * 2, 32))
gotten_queues = queue_wrapper.get_queues()
intersect_queues = [cq for cq in created_queues if cq in gotten_queues]
retries += 1
assert created_queues == intersect_queues
if sqs_stubber.needs_cleanup:
for queue in created_queues:
queue_wrapper.remove_queue(queue)
def test_create_queue_bad_name():
"""Test that creating a queue with invalid characters in the name raises
an exception."""
with pytest.raises(ClientError):
queue_wrapper.create_queue("Queue names cannot contain spaces!")
def test_create_fifo_queue_without_extension(make_unique_name):
"""Test that creating a FIFO queue without the '.fifo' extension
raises an exception."""
with pytest.raises(ClientError):
queue_wrapper.create_queue(make_unique_name('queue'),
attributes={'FifoQueue': str(True)})
def usage_demo():
"""
Shows how to:
* Read the lines from this Python file and send the lines in
batches of 10 as messages to a queue.
* Receive the messages in batches until the queue is empty.
* Reassemble the lines of the file and verify they match the original file.
"""
def pack_message(msg_path, msg_body, msg_line):
return {
'body': msg_body,
'attributes': {
'path': {
'StringValue': msg_path,
'DataType': 'String'
},
'line': {
'StringValue': str(msg_line),
'DataType': 'String'
}
}
}
def unpack_message(msg):
return (msg.message_attributes['path']['StringValue'], msg.body,
int(msg.message_attributes['line']['StringValue']))
print('-' * 88)
print("Welcome to the Amazon Simple Queue Service (Amazon SQS) demo!")
print('-' * 88)
queue = queue_wrapper.create_queue('sqs-usage-demo-message-wrapper')
with open(__file__) as file:
lines = file.readlines()
line = 0
batch_size = 10
received_lines = [None] * len(lines)
print(f"Sending file lines in batches of {batch_size} as messages.")
while line < len(lines):
messages = [
pack_message(__file__, lines[index], index)
for index in range(line, min(line + batch_size, len(lines)))
]
line = line + batch_size
send_messages(queue, messages)
print('.', end='')
sys.stdout.flush()
print(f"Done. Sent {len(lines) - 1} messages.")
print(
f"Receiving, handling, and deleting messages in batches of {batch_size}."
)
more_messages = True
while more_messages:
received_messages = receive_messages(queue, batch_size, 2)
print('.', end='')
sys.stdout.flush()
for message in received_messages:
path, body, line = unpack_message(message)
received_lines[line] = body
if received_messages:
delete_messages(queue, received_messages)
else:
more_messages = False
print('Done.')
if all(
[lines[index] == received_lines[index]
for index in range(len(lines))]):
print(f"Successfully reassembled all file lines!")
else:
print(f"Uh oh, some lines were missed!")
queue.delete()
print("Thanks for watching!")
print('-' * 88)
def test_create_standard_queue_with_fifo_extension(make_unique_name):
"""Test that creating a standard queue with the '.fifo' extension
raises an exception."""
with pytest.raises(ClientError):
queue_wrapper.create_queue(make_unique_name('queue') + '.fifo')
