def _fetch_batch_queue(cls, batch_queue, attributes_to_get=None):
results = []
unprocessed = []
consumed_capacity = 0.0
while len(batch_queue):
batch_keys = batch_queue.pop()
if not len(batch_keys):
continue
batch = BatchList(Configure.get_connection())
batch.add_batch(cls._table, [cls._hash_key_proto(k)
for k in batch_keys],
attributes_to_get=attributes_to_get)
try:
batch_ret = batch.submit()
except DynamoDBKeyNotFoundError:
continue
# import pprint
# pprint.pprint(batch_ret)
if ('UnprocessedKeys' in batch_ret and cls._full_table_name
in batch_ret['UnprocessedKeys']):
u = batch_ret['UnprocessedKeys'][cls._full_table_name]
u = [k['HashKeyElement'] for k in u['Keys']]
unprocessed.extend(u)
if ('Responses' in batch_ret and cls._full_table_name
in batch_ret['Responses']):
tbl = batch_ret['Responses'][cls._full_table_name]
results.extend(tbl['Items'])
consumed_capacity += tbl['ConsumedCapacityUnits']
return results, unprocessed, consumed_capacity
def get_images(filenames):
if not filenames:
raise StopIteration
dynamo = Config().get_dynamodb()
table = Config().table('image')
batch_list = BatchList(dynamo)
batch_list.add_batch(table, filenames,
attributes_to_get=['tags', HASH_KEY])
response = dynamo.batch_get_item(batch_list)
items = response['Responses'][table.name]['Items']
for item in items:
item['filename'] = item.pop(HASH_KEY)
item['tags'] = json.loads(item['tags'])
unprocessed_keys = []
if response['UnprocessedKeys'] \
and table.name in response['UnprocessedKeys']:
for key in response['UnprocessedKeys'][table.name]['Keys']:
unprocessed_keys.append(key['HashKeyElement'])
for item in items:
yield item
if not unprocessed_keys:
raise StopIteration
for item in get_images(unprocessed_keys):
yield item
def _fetch_batch_queue(cls, batch_queue, attributes_to_get=None):
results = []
unprocessed = []
consumed_capacity = 0.0
while len(batch_queue):
batch_keys = batch_queue.pop()
if not len(batch_keys):
continue
batch = BatchList(Configure.get_connection())
batch.add_batch(cls._table,
[cls._hash_key_proto(k) for k in batch_keys],
attributes_to_get=attributes_to_get)
try:
batch_ret = batch.submit()
except DynamoDBKeyNotFoundError:
continue
# import pprint
# pprint.pprint(batch_ret)
if ('UnprocessedKeys' in batch_ret
and cls._full_table_name in batch_ret['UnprocessedKeys']):
u = batch_ret['UnprocessedKeys'][cls._full_table_name]
u = [k['HashKeyElement'] for k in u['Keys']]
unprocessed.extend(u)
if ('Responses' in batch_ret
and cls._full_table_name in batch_ret['Responses']):
tbl = batch_ret['Responses'][cls._full_table_name]
results.extend(tbl['Items'])
consumed_capacity += tbl['ConsumedCapacityUnits']
return results, unprocessed, consumed_capacity
def get_item_batch(tag_names, table_name, attributes_to_get):
if not tag_names:
raise StopIteration
tag_names = list(set(tag_names))
unprocessed_keys = tag_names[MAX_ITEMS_TO_REQUEST:]
tag_names = tag_names[:MAX_ITEMS_TO_REQUEST]
dynamo = Client().get_dynamodb()
table = Client().table(table_name)
batch_list = BatchList(dynamo)
batch_list.add_batch(table, tag_names,
attributes_to_get=attributes_to_get + [HASH_KEY])
response = dynamo.batch_get_item(batch_list)
items = response['Responses'][table.name]['Items']
if response['UnprocessedKeys'] \
and table.name in response['UnprocessedKeys']:
for key in response['UnprocessedKeys'][table.name]['Keys']:
unprocessed_keys.append(key['HashKeyElement'])
for item in items:
yield item
for item in get_item_batch(unprocessed_keys, table_name, attributes_to_get):
yield item
def __iter__(self):
while self.keys:
# Build the next batch
batch = BatchList(self.table.layer2)
batch.add_batch(self.table, self.keys[:100], self.attributes_to_get)
res = batch.submit()
# parse the results
if not self.table.name in res[u'Responses']:
continue
self.consumed_units += res[u'Responses'][self.table.name][u'ConsumedCapacityUnits']
for elem in res[u'Responses'][self.table.name][u'Items']:
yield elem
# re-queue un processed keys
self.keys = self.keys[100:]
self._queue_unprocessed(res)
def test_batch_list_consistent_read(self):
b = BatchList(self.layer2)
b.add_batch(self.table, ['k1'], ['foo'], consistent_read=True)
b.add_batch(self.table2, [('k2', 54)], ['bar'], consistent_read=False)
self.assertDictEqual(
b.to_dict(), {
'testtable': {
'AttributesToGet': ['foo'],
'Keys': [{
'HashKeyElement': {
'S': 'k1'
}
}],
'ConsistentRead': True
},
'testtable2': {
'AttributesToGet': ['bar'],
'Keys': [{
'HashKeyElement': {
'S': 'k2'
},
'RangeKeyElement': {
'N': '54'
}
}],
'ConsistentRead':
False
}
})
def __iter__(self):
while self.keys:
# Build the next batch
batch = BatchList(self.table.layer2)
batch.add_batch(self.table, self.keys[:100],
self.attributes_to_get)
res = batch.submit()
# parse the results
if not self.table.name in res['Responses']:
continue
self.consumed_units += res['Responses'][self.table.name]['ConsumedCapacityUnits']
for elem in res['Responses'][self.table.name]['Items']:
yield elem
# re-queue un processed keys
self.keys = self.keys[100:]
self._queue_unprocessed(res)
def get_tag_items(tag_names):
if not tag_names:
raise StopIteration
dynamo = Config().get_dynamodb()
table = Config().table('tag')
batch_list = BatchList(dynamo)
batch_list.add_batch(table, tag_names,
attributes_to_get=['filenames', HASH_KEY])
response = dynamo.batch_get_item(batch_list)
items = response['Responses'][table.name]['Items']
unprocessed_keys = []
if response['UnprocessedKeys'] \
and table.name in response['UnprocessedKeys']:
for key in response['UnprocessedKeys'][table.name]['Keys']:
unprocessed_keys.append(key['HashKeyElement'])
for item in items:
yield item
for item in get_tag_items(unprocessed_keys):
yield item
def test_batch_list_consistent_read(self):
b = BatchList(self.layer2)
b.add_batch(self.table, ['k1'], ['foo'], consistent_read=True)
b.add_batch(self.table2, [('k2', 54)], ['bar'], consistent_read=False)
self.assertDictEqual(
b.to_dict(),
{'testtable': {'AttributesToGet': ['foo'],
'Keys': [{'HashKeyElement': {'S': 'k1'}}],
'ConsistentRead': True},
'testtable2': {'AttributesToGet': ['bar'],
'Keys': [{'HashKeyElement': {'S': 'k2'},
'RangeKeyElement': {'N': '54'}}],
'ConsistentRead': False}})
def new_batch_list(self):
"""
Return a new, empty :class:`boto.dynamodb.batch.BatchList`
object.
"""
return BatchList(self)
