def test_batch(): things = [1, 2, 3, 4, 5, 6] iter = list(utils.batch(things, 2)) assert len(iter) == 3 assert iter[0] == [1, 2] assert iter[1] == [3, 4] assert iter[2] == [5, 6]
def batch_put_records(stream_name, records, batch_size=500, **kwargs): """Put records into a kinesis stream, batched by the maximum of 500.""" client = lpipe.contrib.boto3.client("kinesis") responses = [] for b in utils.batch(records, batch_size): responses.append( utils.call( client.put_records, StreamName=stream_name, Records=[build(record) for record in b], )) return tuple(responses)
def batch_put_messages(queue_url, messages, batch_size=10, message_group_id=None, **kwargs): """Put messages into a sqs queue, batched by the maximum of 10.""" assert batch_size <= 10 # send_message_batch will fail otherwise client = _boto3.client("sqs") responses = [] for b in batch(messages, batch_size): responses.append( call( client.send_message_batch, QueueUrl=queue_url, Entries=[build(message, message_group_id) for message in b], )) return tuple(responses)