def get_object(self, container_name, key, stream=False, extra_get_args={}):
"""
Get object from Swift with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if not container_name:
container_name = self.storage_container
url = '/'.join([self.endpoint, container_name, key])
headers = {'X-Auth-Token': self.token}
headers.update(extra_get_args)
try:
res = self.session.get(url, headers=headers, stream=stream)
if res.status_code == 200 or res.status_code == 206:
if stream:
data = res.raw
else:
data = res.content
return data
elif res.status_code == 404:
raise StorageNoSuchKeyError(container_name, key)
else:
raise Exception('{} - {}'.format(res.status_code, key))
except StorageNoSuchKeyError:
raise StorageNoSuchKeyError(container_name, key)
except Exception as e:
print(e)
raise StorageNoSuchKeyError(container_name, key)
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from Redis with a key.
Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
redis_key = self._format_key(bucket_name, key)
try:
if 'Range' in extra_get_args: # expected format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
start, end = self._parse_range(bytes_range)
data = self._client.getrange(redis_key, start, end)
else:
data = self._client.get(redis_key)
except redis.exceptions.ResponseError:
raise StorageNoSuchKeyError(bucket_name, key)
if data is None:
raise StorageNoSuchKeyError(bucket_name, key)
if stream:
return io.BytesIO(data)
else:
return data
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from localhost filesystem with a key.
Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
buffer = None
try:
file_path = os.path.join(STORAGE_FOLDER, bucket_name, key)
with open(file_path, "rb") as f:
if 'Range' in extra_get_args:
byte_range = extra_get_args['Range'].replace('bytes=', '')
first_byte, last_byte = map(int, byte_range.split('-'))
f.seek(first_byte)
buffer = io.BytesIO(f.read(last_byte - first_byte + 1))
else:
buffer = io.BytesIO(f.read())
if stream:
return buffer
else:
return buffer.read()
except Exception:
raise StorageNoSuchKeyError(
os.path.join(STORAGE_FOLDER, bucket_name), key)
def put_object(self, bucket_name, key, data):
"""
Put an object in COS. Override the object if the key already exists.
:param key: key of the object.
:param data: data of the object
:type data: str/bytes
:return: None
"""
retries = 0
status = None
while status is None:
try:
res = self.cos_client.put_object(Bucket=bucket_name,
Key=key,
Body=data)
status = 'OK' if res['ResponseMetadata'][
'HTTPStatusCode'] == 200 else 'Error'
try:
logger.debug('PUT Object {} - Size: {} - {}'.format(
key, sizeof_fmt(len(data)), status))
except Exception:
logger.debug('PUT Object {} {}'.format(key, status))
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('PUT Object timeout. Retrying request')
retries += 1
return True
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
data = None
retries = 0
while data is None:
try:
r = self.cos_client.get_object(Bucket=bucket_name,
Key=key,
**extra_get_args)
if stream:
data = r['Body']
else:
data = r['Body'].read()
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "NoSuchKey":
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('GET Object timeout. Retrying request')
retries += 1
return data
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
bytes_range = bytes_range.split('-')
extra_get_args['start_range'] = int(bytes_range[0])
extra_get_args['end_range'] = int(bytes_range[1])
try:
if stream:
stream_out = BytesIO()
self.blob_client.get_blob_to_stream(bucket_name, key,
stream_out,
**extra_get_args)
stream_out.seek(0)
return stream_out
else:
data = self.blob_client.get_blob_to_bytes(
bucket_name, key, **extra_get_args)
return data.content
except AzureMissingResourceHttpError:
raise StorageNoSuchKeyError(bucket_name, key)
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param bucket_name: Name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
try:
prefix = '' if prefix is None else prefix
paginator = self.cos_client.get_paginator('list_objects_v2')
page_iterator = paginator.paginate(Bucket=bucket_name,
Prefix=prefix)
key_list = []
for page in page_iterator:
if 'Contents' in page:
for item in page['Contents']:
key_list.append(item['Key'])
return key_list
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, prefix)
else:
raise e
def bucket_exists(self, container_name):
"""
Head container from Swift with a name. Throws StorageNoSuchKeyError if the given container does not exist.
:param container_name: name of the container
:return: Data of the bucket
:rtype: str/bytes
"""
url = '/'.join([self.endpoint, container_name])
try:
res = self.session.head(url)
if res.status_code == 204:
return res.headers
elif res.status_code == 404:
raise StorageNoSuchKeyError(container_name, '')
else:
raise Exception('{} - {}'.format(res.status_code))
except Exception as e:
raise StorageNoSuchKeyError(container_name, '')
def head_object(self, container_name, key):
"""
Head object from Swift with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
url = '/'.join([self.endpoint, container_name, key])
try:
res = self.session.head(url)
if res.status_code == 200:
return res.headers
elif res.status_code == 404:
raise StorageNoSuchKeyError(container_name, key)
else:
raise Exception('{} - {}'.format(res.status_code, key))
except Exception as e:
raise StorageNoSuchKeyError(container_name, key)
def bucket_exists(self, bucket_name):
"""
Returns True if container exists in storage. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
"""
try:
self.blob_client.get_container_metadata(bucket_name)
return True
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def head_bucket(self, bucket_name):
"""
Head container from COS with a name. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
:return: Data of the object
"""
try:
return self.blob_client.get_container_metadata(bucket_name)
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def bucket_exists(self, bucket_name):
"""
Returns True if bucket exists in storage. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
"""
bucket = self._connect_bucket(bucket_name)
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
return True
def bucket_exists(self, bucket_name):
"""
Head bucket from COS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
"""
try:
self.cos_client.head_bucket(Bucket=bucket_name)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, '')
else:
raise e
def head_bucket(self, bucket_name):
"""
Head bucket from OSS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
:return: metadata of the bucket
:rtype: dict
"""
bucket = self._connect_bucket(bucket_name)
try:
metadata = bucket.get_bucket_info()
return vars(metadata)
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
def head_bucket(self, bucket_name):
"""
Head bucket from Ceph with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
:return: Metadata of the bucket
:rtype: str/bytes
"""
try:
return self.cos_client.head_bucket(Bucket=bucket_name)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, '')
else:
raise e
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
try:
keys = [
key for key in self.blob_client.list_blob_names(
bucket_name, prefix).items
]
return keys
except Exception:
raise StorageNoSuchKeyError(bucket_name,
'' if prefix is None else prefix)
def head_object(self, bucket_name, key):
"""
Head object from Redis with a key.
Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: dict
"""
redis_key = self._format_key(bucket_name, key)
try:
meta = self._client.debug_object(redis_key)
except redis.exceptions.ResponseError:
raise StorageNoSuchKeyError(bucket_name, key)
meta['content-length'] = meta['serializedlength'] - 1
return meta
def put_object(self, bucket_name, key, data):
"""
Put an object in OSS. Override the object if the key already exists.
Throws StorageNoSuchKeyError if the bucket does not exist.
:param bucket_name: bucket name
:param key: key of the object.
:param data: data of the object
:type data: str/bytes
:return: None
"""
if isinstance(data, str):
data = data.encode()
try:
bucket = self._connect_bucket(bucket_name)
bucket.put_object(key, data)
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
def head_object(self, bucket_name, key):
"""
Head object from OSS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: dict
"""
bucket = self._connect_bucket(bucket_name)
try:
headobj = bucket.head_object(key)
# adapted to match ibm_cos method
metadata = vars(headobj)
metadata['content-length'] = metadata.pop('content_length')
return metadata
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
raise StorageNoSuchKeyError(bucket_name, key)
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param bucket_name: name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
bucket = self._connect_bucket(bucket_name)
# adapted to match ibm_cos method
prefix = '' if prefix is None else prefix
try:
res = bucket.list_objects(prefix=prefix)
keys = [obj.key for obj in res.object_list]
return keys
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
StorageNoSuchKeyError(bucket_name, prefix)
def list_objects(self, bucket_name, prefix=None):
"""
Return a list of objects for the given bucket and prefix.
:param bucket_name: name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of objects in bucket that match the given prefix.
:rtype: list of dict
"""
bucket = self._connect_bucket(bucket_name)
# adapted to match ibm_cos method
prefix = '' if prefix is None else prefix
try:
res = bucket.list_objects(prefix=prefix)
obj_list = [{'Key' : obj.key, 'Size' : obj.size} for obj in res.object_list]
print(obj_list)
return obj_list
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
StorageNoSuchKeyError(bucket_name, prefix)
def list_objects(self, bucket_name, prefix=None):
"""
Return a list of objects for the given bucket and prefix.
:param bucket_name: Name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of objects in bucket that match the given prefix.
:rtype: list of str
"""
# adapted to match ibm_cos method
try:
blobs = self.blob_client.list_blobs(bucket_name, prefix)
mod_list = []
for blob in blobs:
mod_list.append({
'Key': blob.name,
'Size': blob.properties.content_length
})
return mod_list
except Exception:
raise StorageNoSuchKeyError(bucket_name,
'' if prefix is None else prefix)
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from OSS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
bytes_range = bytes_range.split('-')
extra_get_args['byte_range'] = (int(bytes_range[0]), int(bytes_range[1]))
try:
bucket = self._connect_bucket(bucket_name)
data = bucket.get_object(key=key, **extra_get_args)
if stream:
return data
else:
return data.read()
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
raise StorageNoSuchKeyError(bucket_name, key)
def head_object(self, bucket_name, key):
"""
Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
metadata = None
retries = 0
while metadata is None:
try:
metadata = self.cos_client.head_object(Bucket=bucket_name,
Key=key)
except ibm_botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == '404':
raise StorageNoSuchKeyError(bucket_name, key)
else:
raise e
except ibm_botocore.exceptions.ReadTimeoutError as e:
if retries == OBJ_REQ_RETRIES:
raise e
logger.debug('HEAD Object timeout. Retrying request')
retries += 1
return metadata['ResponseMetadata']['HTTPHeaders']