def _fetch_batch_queue(cls, batch_queue, attributes_to_get=None):
results = []
unprocessed = []
consumed_capacity = 0.0
while len(batch_queue):
batch_keys = batch_queue.pop()
if not len(batch_keys):
continue
batch = BatchList(Configure.get_connection())
batch.add_batch(cls._table,
[cls._hash_key_proto(k) for k in batch_keys],
attributes_to_get=attributes_to_get)
try:
batch_ret = batch.submit()
except DynamoDBKeyNotFoundError:
continue
# import pprint
# pprint.pprint(batch_ret)
if ('UnprocessedKeys' in batch_ret
and cls._full_table_name in batch_ret['UnprocessedKeys']):
u = batch_ret['UnprocessedKeys'][cls._full_table_name]
u = [k['HashKeyElement'] for k in u['Keys']]
unprocessed.extend(u)
if ('Responses' in batch_ret
and cls._full_table_name in batch_ret['Responses']):
tbl = batch_ret['Responses'][cls._full_table_name]
results.extend(tbl['Items'])
consumed_capacity += tbl['ConsumedCapacityUnits']
return results, unprocessed, consumed_capacity
def test_batch_list_consistent_read(self):
b = BatchList(self.layer2)
b.add_batch(self.table, ['k1'], ['foo'], consistent_read=True)
b.add_batch(self.table2, [('k2', 54)], ['bar'], consistent_read=False)
self.assertDictEqual(
b.to_dict(), {
'testtable': {
'AttributesToGet': ['foo'],
'Keys': [{
'HashKeyElement': {
'S': 'k1'
}
}],
'ConsistentRead': True
},
'testtable2': {
'AttributesToGet': ['bar'],
'Keys': [{
'HashKeyElement': {
'S': 'k2'
},
'RangeKeyElement': {
'N': '54'
}
}],
'ConsistentRead':
False
}
})
def __iter__(self):
while self.keys:
# Build the next batch
batch = BatchList(self.table.layer2)
batch.add_batch(self.table, self.keys[:100],
self.attributes_to_get)
res = batch.submit()
# parse the results
if not self.table.name in res['Responses']:
continue
self.consumed_units += res['Responses'][self.table.name]['ConsumedCapacityUnits']
for elem in res['Responses'][self.table.name]['Items']:
yield elem
# re-queue un processed keys
self.keys = self.keys[100:]
self._queue_unprocessed(res)
def new_batch_list(self):
"""
Return a new, empty :class:`boto.dynamodb.batch.BatchList`
object.
"""
return BatchList(self)