def add_lookup_key(self, item):
"""
Using the given item from the S3 index table, extract the lookup key
from the object key and write it back to the item as a new attribute.
If throttled, will use exponential backoff and retry 5 times.
Args:
item (dict): An item from the response dictionary returned by DynamoDB.Client.scan().
"""
if OBJ_KEY not in item or 'S' not in item[OBJ_KEY]:
return
if VERSION_NODE not in item:
return
parts = AWSObjectStore.get_object_key_parts(item[OBJ_KEY]['S'])
lookup_key = AWSObjectStore.generate_lookup_key(
parts.collection_id, parts.experiment_id, parts.channel_id,
parts.resolution)
NUM_RETRIES = 5
for backoff in range(0, NUM_RETRIES + 1):
try:
self.dynamodb.update_item(
TableName=self.table,
Key={OBJ_KEY: item[OBJ_KEY], VERSION_NODE: item[VERSION_NODE]},
ExpressionAttributeNames = {'#lookupkey': LOOKUP_KEY},
ExpressionAttributeValues = {':lookupkey': {'S': lookup_key}},
UpdateExpression='set #lookupkey = :lookupkey'
)
return
except botocore.exceptions.ClientError as ex:
if ex.response['Error']['Code'] == 'ProvisionedThroughputExceededException':
print('Throttled during update of item: {} - {}'.format(
item[OBJ_KEY]['S'], item[VERSION_NODE]['N']))
time.sleep(((2 ** backoff) + (random.randint(0, 1000) / 1000.0))/10.0)
else:
print('Failed updating item: {} - {}'.format(
item[OBJ_KEY]['S'], item[VERSION_NODE]['N']))
raise
except:
print('Failed updating item: {} - {}'.format(
item[OBJ_KEY]['S'], item[VERSION_NODE]['N']))
raise
print('Failed and giving up after {} retries trying to update item: {} - {}'
.format(NUM_RETRIES, item[OBJ_KEY]['S'], item[VERSION_NODE]['N']))
def test_get_object_key_parts_iso(self):
"""Test to get an object key parts after the iso split on an anisotropic channel"""
os = AWSObjectStore(self.object_store_config)
object_key = os.generate_object_key(self.resource, 5, 2, 56, iso=True)
parts = os.get_object_key_parts(object_key)
self.assertEqual(object_key, '068e7246f31aacac92ca74923b9da6f1&ISO&4&3&2&5&2&56')
self.assertEqual(parts.hash, "068e7246f31aacac92ca74923b9da6f1")
self.assertEqual(parts.collection_id, "4")
self.assertEqual(parts.experiment_id, "3")
self.assertEqual(parts.channel_id, "2")
self.assertEqual(parts.resolution, "5")
self.assertEqual(parts.time_sample, "2")
self.assertEqual(parts.morton_id, "56")
self.assertEqual(parts.is_iso, True)
def test_get_object_key_parts(self):
"""Test to get an object key parts"""
os = AWSObjectStore(self.object_store_config)
object_key = os.generate_object_key(self.resource, 0, 2, 56)
parts = os.get_object_key_parts(object_key)
self.assertEqual(object_key, '631424bf68302b683a0be521101c192b&4&3&2&0&2&56')
self.assertEqual(parts.hash, "631424bf68302b683a0be521101c192b")
self.assertEqual(parts.collection_id, "4")
self.assertEqual(parts.experiment_id, "3")
self.assertEqual(parts.channel_id, "2")
self.assertEqual(parts.resolution, "0")
self.assertEqual(parts.time_sample, "2")
self.assertEqual(parts.morton_id, "56")
self.assertEqual(parts.is_iso, False)
