def add_lookup_key(self, item): """ Using the given item from the S3 index table, extract the lookup key from the object key and write it back to the item as a new attribute. If throttled, will use exponential backoff and retry 5 times. Args: item (dict): An item from the response dictionary returned by DynamoDB.Client.scan(). """ if OBJ_KEY not in item or 'S' not in item[OBJ_KEY]: return if VERSION_NODE not in item: return parts = AWSObjectStore.get_object_key_parts(item[OBJ_KEY]['S']) lookup_key = AWSObjectStore.generate_lookup_key( parts.collection_id, parts.experiment_id, parts.channel_id, parts.resolution) NUM_RETRIES = 5 for backoff in range(0, NUM_RETRIES + 1): try: self.dynamodb.update_item( TableName=self.table, Key={OBJ_KEY: item[OBJ_KEY], VERSION_NODE: item[VERSION_NODE]}, ExpressionAttributeNames = {'#lookupkey': LOOKUP_KEY}, ExpressionAttributeValues = {':lookupkey': {'S': lookup_key}}, UpdateExpression='set #lookupkey = :lookupkey' ) return except botocore.exceptions.ClientError as ex: if ex.response['Error']['Code'] == 'ProvisionedThroughputExceededException': print('Throttled during update of item: {} - {}'.format( item[OBJ_KEY]['S'], item[VERSION_NODE]['N'])) time.sleep(((2 ** backoff) + (random.randint(0, 1000) / 1000.0))/10.0) else: print('Failed updating item: {} - {}'.format( item[OBJ_KEY]['S'], item[VERSION_NODE]['N'])) raise except: print('Failed updating item: {} - {}'.format( item[OBJ_KEY]['S'], item[VERSION_NODE]['N'])) raise print('Failed and giving up after {} retries trying to update item: {} - {}' .format(NUM_RETRIES, item[OBJ_KEY]['S'], item[VERSION_NODE]['N']))
def test_get_object_key_parts_iso(self): """Test to get an object key parts after the iso split on an anisotropic channel""" os = AWSObjectStore(self.object_store_config) object_key = os.generate_object_key(self.resource, 5, 2, 56, iso=True) parts = os.get_object_key_parts(object_key) self.assertEqual(object_key, '068e7246f31aacac92ca74923b9da6f1&ISO&4&3&2&5&2&56') self.assertEqual(parts.hash, "068e7246f31aacac92ca74923b9da6f1") self.assertEqual(parts.collection_id, "4") self.assertEqual(parts.experiment_id, "3") self.assertEqual(parts.channel_id, "2") self.assertEqual(parts.resolution, "5") self.assertEqual(parts.time_sample, "2") self.assertEqual(parts.morton_id, "56") self.assertEqual(parts.is_iso, True)
def test_get_object_key_parts(self): """Test to get an object key parts""" os = AWSObjectStore(self.object_store_config) object_key = os.generate_object_key(self.resource, 0, 2, 56) parts = os.get_object_key_parts(object_key) self.assertEqual(object_key, '631424bf68302b683a0be521101c192b&4&3&2&0&2&56') self.assertEqual(parts.hash, "631424bf68302b683a0be521101c192b") self.assertEqual(parts.collection_id, "4") self.assertEqual(parts.experiment_id, "3") self.assertEqual(parts.channel_id, "2") self.assertEqual(parts.resolution, "0") self.assertEqual(parts.time_sample, "2") self.assertEqual(parts.morton_id, "56") self.assertEqual(parts.is_iso, False)