def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from localhost filesystem with a key.
Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
buffer = None
try:
file_path = os.path.join(STORAGE_FOLDER, bucket_name, key)
with open(file_path, "rb") as f:
if 'Range' in extra_get_args:
byte_range = extra_get_args['Range'].replace('bytes=', '')
first_byte, last_byte = map(int, byte_range.split('-'))
f.seek(first_byte)
buffer = io.BytesIO(f.read(last_byte-first_byte+1))
else:
buffer = io.BytesIO(f.read())
if stream:
return buffer
else:
return buffer.read()
except Exception:
raise StorageNoSuchKeyError(os.path.join(STORAGE_FOLDER, bucket_name), key)
def put_object(self, bucket_name, key, data):
"""
Put an object in COS. Override the object if the key already exists.
:param key: key of the object.
:param data: data of the object
:type data: str/bytes
:return: None
"""
retries = 0
status = None
while status is None:
try:
res = self.cos_client.put_object(Bucket=bucket_name, Key=key, Body=data)
status = 'OK' if res['ResponseMetadata']['HTTPStatusCode'] == 200 else 'Error'
try:
logger.debug('PUT Object {} - Size: {} - {}'.format(key, sizeof_fmt(len(data)), status))
except Exception:
logger.debug('PUT Object {} {}'.format(key, status))
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('PUT Object timeout. Retrying request')
retries += 1
return True
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
data = None
retries = 0
while data is None:
try:
r = self.cos_client.get_object(Bucket=bucket_name, Key=key, **extra_get_args)
if stream:
data = r['Body']
else:
data = r['Body'].read()
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('GET Object timeout. Retrying request')
retries += 1
return data
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param bucket_name: Name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
try:
prefix = '' if prefix is None else prefix
paginator = self.cos_client.get_paginator('list_objects_v2')
page_iterator = paginator.paginate(Bucket=bucket_name,
Prefix=prefix)
key_list = []
for page in page_iterator:
if 'Contents' in page:
for item in page['Contents']:
key_list.append(item['Key'])
return key_list
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, prefix)
else:
raise e
def bucket_exists(self, bucket_name):
"""
Returns True if container exists in storage. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
"""
try:
self.blob_client.get_container_metadata(bucket_name)
return True
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def head_bucket(self, bucket_name):
"""
Head container from COS with a name. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
:return: Data of the object
"""
try:
return self.blob_client.get_container_metadata(bucket_name)
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def bucket_exists(self, bucket_name):
"""
Head bucket from COS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
"""
try:
self.cos_client.head_bucket(Bucket=bucket_name)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, '')
else:
raise e
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
try:
keys = [key for key in self.blob_client.list_blob_names(bucket_name, prefix).items]
return keys
except Exception:
raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix)
def head_bucket(self, bucket_name):
"""
Head bucket from Ceph with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
:return: Metadata of the bucket
:rtype: str/bytes
"""
try:
return self.cos_client.head_bucket(Bucket=bucket_name)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, '')
else:
raise e
def head_object(self, bucket_name, key):
"""
Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
try:
metadata = self.cos_client.head_object(Bucket=bucket_name, Key=key)
return metadata['ResponseMetadata']['HTTPHeaders']
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from localhost filesystem with a key.
Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
try:
file_path = "{}/{}".format(bucket_name, key)
with self.fs.open(file_path, "rb") as f:
if 'Range' in extra_get_args:
byte_range = extra_get_args['Range'].replace('bytes=', '')
first_byte, last_byte = map(int, byte_range.split('-'))
f.seek(first_byte)
return f.read(last_byte - first_byte + 1)
else:
return f.read()
except Exception as e:
raise StorageNoSuchKeyError(bucket_name, key)
def list_objects(self, bucket_name, prefix=None):
"""
Return a list of objects for the given bucket and prefix.
:param bucket_name: Name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of objects in bucket that match the given prefix.
:rtype: list of str
"""
# adapted to match ibm_cos method
try:
blobs = self.blob_client.list_blobs(bucket_name, prefix)
mod_list = []
for blob in blobs:
mod_list.append({
'Key' : blob.name,
'Size' : blob.properties.content_length
})
return mod_list
except Exception:
raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix)
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
try:
r = self.cos_client.get_object(Bucket=bucket_name,
Key=key,
**extra_get_args)
if stream:
data = r['Body']
else:
data = r['Body'].read()
return data
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
def head_object(self, bucket_name, key):
"""
Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
metadata = None
retries = 0
while metadata is None:
try:
metadata = self.cos_client.head_object(Bucket=bucket_name, Key=key)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('HEAD Object timeout. Retrying request')
retries += 1
return metadata['ResponseMetadata']['HTTPHeaders']
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
bytes_range = bytes_range.split('-')
extra_get_args['start_range'] = int(bytes_range[0])
extra_get_args['end_range'] = int(bytes_range[1])
try:
if stream:
stream_out = BytesIO()
self.blob_client.get_blob_to_stream(bucket_name, key, stream_out, **extra_get_args)
stream_out.seek(0)
return stream_out
else:
data = self.blob_client.get_blob_to_bytes(bucket_name, key, **extra_get_args)
return data.content
except AzureMissingResourceHttpError:
raise StorageNoSuchKeyError(bucket_name, key)
