def test_batch():
things = [1, 2, 3, 4, 5, 6]
iter = list(utils.batch(things, 2))
assert len(iter) == 3
assert iter[0] == [1, 2]
assert iter[1] == [3, 4]
assert iter[2] == [5, 6]
def batch_put_records(stream_name, records, batch_size=500, **kwargs):
"""Put records into a kinesis stream, batched by the maximum of 500."""
client = lpipe.contrib.boto3.client("kinesis")
responses = []
for b in utils.batch(records, batch_size):
responses.append(
utils.call(
client.put_records,
StreamName=stream_name,
Records=[build(record) for record in b],
))
return tuple(responses)
def batch_put_messages(queue_url,
messages,
batch_size=10,
message_group_id=None,
**kwargs):
"""Put messages into a sqs queue, batched by the maximum of 10."""
assert batch_size <= 10 # send_message_batch will fail otherwise
client = _boto3.client("sqs")
responses = []
for b in batch(messages, batch_size):
responses.append(
call(
client.send_message_batch,
QueueUrl=queue_url,
Entries=[build(message, message_group_id) for message in b],
))
return tuple(responses)