def cleanup_sqs_records(records: list, logger):
base_err_msg = (
"Unable to delete successful records messages from SQS queue. AWS should "
"still handle this automatically when the lambda finishes executing, but "
"this may result in successful messages being sent to the DLQ if any "
"other messages fail.")
try:
Message = namedtuple("Message", ["message_id", "receipt_handle"])
messages = defaultdict(list)
for record in records:
m = Message(
message_id=mindictive.get_nested(record, ["messageId"]),
receipt_handle=mindictive.get_nested(record,
["receiptHandle"]),
)
messages[mindictive.get_nested(record,
["eventSourceARN"])].append(m)
for k in messages.keys():
queue_url = sqs.get_queue_url(k)
sqs.batch_delete_messages(
queue_url,
[{
"Id": m.message_id,
"ReceiptHandle": m.receipt_handle
} for m in messages],
)
except KeyError as e:
logger.warning(
f"{base_err_msg} If you're testing, this is not an issue. {utils.exception_to_str(e)}"
)
except Exception as e:
logger.warning(f"{base_err_msg} {utils.exception_to_str(e)}")
def test_batch_put_messages_many(self):
sqs_queues = fixtures.SQS
boto3.client("sqs")
queue_url = sqs.get_queue_url(sqs_queues[0])
responses = sqs.batch_put_messages(
queue_url=queue_url,
messages=[
{
"foo": "bar",
"wiz": "bang"
},
{
"lorem": "ipsum",
"quid": "est"
},
{
"foo": "bar",
"wiz": "bang"
},
{
"lorem": "ipsum",
"quid": "est"
},
],
batch_size=2,
)
assert len(responses) == 2
assert all([check_status(r) for r in responses])
def put_record(queue: Queue, record: dict):
if queue.type == QueueType.KINESIS:
return kinesis.put_record(stream_name=queue.name, data=record)
if queue.type == QueueType.SQS:
if not queue.url:
queue.url = sqs.get_queue_url(queue.name)
try:
return sqs.put_message(queue_url=queue.url, data=record)
except Exception as e:
raise lpipe.exceptions.FailCatastrophically(
f"Failed to send message to {queue}") from e
def test_batch_put_message(self):
sqs_queues = fixtures.SQS
boto3.client("sqs")
queue_url = sqs.get_queue_url(sqs_queues[0])
responses = sqs.put_message(queue_url=queue_url,
data={
"foo": "bar",
"wiz": "bang"
})
assert len(responses) == 1
assert all([check_status(r) for r in responses])
def test_sqs_by_url(self, set_environment):
sqs_queues = fixtures.SQS
queue_url = get_queue_url(sqs_queues[0])
queue = Queue(type=QueueType.SQS, path="FOO", url=queue_url)
fixture = {"path": queue.path, "kwargs": {}}
put_record(queue=queue, record=fixture)