def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
bytes_range = bytes_range.split('-')
extra_get_args['start_range'] = int(bytes_range[0])
extra_get_args['end_range'] = int(bytes_range[1])
try:
if stream:
stream_out = BytesIO()
self.blob_client.get_blob_to_stream(bucket_name, key,
stream_out,
**extra_get_args)
stream_out.seek(0)
return stream_out
else:
data = self.blob_client.get_blob_to_bytes(
bucket_name, key, **extra_get_args)
return data.content
except AzureMissingResourceHttpError:
raise StorageNoSuchKeyError(bucket_name, key)
def bucket_exists(self, bucket_name):
"""
Returns True if container exists in storage. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
"""
try:
self.blob_client.get_container_metadata(bucket_name)
return True
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def head_bucket(self, bucket_name):
"""
Head container from COS with a name. Throws StorageNoSuchKeyError if the given container does not exist.
:param bucket_name: name of the container
:return: Data of the object
"""
try:
return self.blob_client.get_container_metadata(bucket_name)
except Exception:
raise StorageNoSuchKeyError(bucket_name, '')
def bucket_exists(self, bucket_name):
"""
Returns True if bucket exists in storage. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
"""
bucket = self._connect_bucket(bucket_name)
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
return True
def head_bucket(self, bucket_name):
"""
Head bucket from OSS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist.
:param bucket_name: name of the bucket
:return: metadata of the bucket
:rtype: dict
"""
bucket = self._connect_bucket(bucket_name)
try:
metadata = bucket.get_bucket_info()
return vars(metadata)
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
try:
keys = [
key for key in self.blob_client.list_blob_names(
bucket_name, prefix).items
]
return keys
except Exception:
raise StorageNoSuchKeyError(bucket_name,
'' if prefix is None else prefix)
def head_object(self, bucket_name, key):
"""
Head object from OSS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: dict
"""
bucket = self._connect_bucket(bucket_name)
try:
headobj = bucket.head_object(key)
# adapted to match ibm_cos method
metadata = vars(headobj)
metadata['content-length'] = metadata.pop('content_length')
return metadata
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
raise StorageNoSuchKeyError(bucket_name, key)
def put_object(self, bucket_name, key, data):
"""
Put an object in OSS. Override the object if the key already exists.
Throws StorageNoSuchKeyError if the bucket does not exist.
:param bucket_name: bucket name
:param key: key of the object.
:param data: data of the object
:type data: str/bytes
:return: None
"""
if isinstance(data, str):
data = data.encode()
try:
bucket = self._connect_bucket(bucket_name)
bucket.put_object(key, data)
except oss2.exceptions.NoSuchBucket:
StorageNoSuchKeyError(bucket_name, '')
def list_keys(self, bucket_name, prefix=None):
"""
Return a list of keys for the given prefix.
:param bucket_name: name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of keys in bucket that match the given prefix.
:rtype: list of str
"""
bucket = self._connect_bucket(bucket_name)
# adapted to match ibm_cos method
prefix = '' if prefix is None else prefix
try:
res = bucket.list_objects(prefix=prefix)
keys = [obj.key for obj in res.object_list]
return keys
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
StorageNoSuchKeyError(bucket_name, prefix)
def list_objects(self, bucket_name, prefix=None):
"""
Return a list of objects for the given bucket and prefix.
:param bucket_name: Name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of objects in bucket that match the given prefix.
:rtype: list of str
"""
# adapted to match ibm_cos method
try:
blobs = self.blob_client.list_blobs(bucket_name, prefix)
mod_list = []
for blob in blobs:
mod_list.append({
'Key': blob.name,
'Size': blob.properties.content_length
})
return mod_list
except Exception:
raise StorageNoSuchKeyError(bucket_name,
'' if prefix is None else prefix)
def list_objects(self, bucket_name, prefix=None):
"""
Return a list of objects for the given bucket and prefix.
:param bucket_name: name of the bucket.
:param prefix: Prefix to filter object names.
:return: List of objects in bucket that match the given prefix.
:rtype: list of dict
"""
bucket = self._connect_bucket(bucket_name)
# adapted to match ibm_cos method
prefix = '' if prefix is None else prefix
try:
res = bucket.list_objects(prefix=prefix)
obj_list = [{
'Key': obj.key,
'Size': obj.size
} for obj in res.object_list]
print(obj_list)
return obj_list
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
StorageNoSuchKeyError(bucket_name, prefix)
def get_object(self, bucket_name, key, stream=False, extra_get_args={}):
"""
Get object from OSS with a key. Throws StorageNoSuchKeyError if the given key does not exist.
:param bucket_name: bucket name
:param key: key of the object
:return: Data of the object
:rtype: str/bytes
"""
if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H'
bytes_range = extra_get_args.pop('Range')[6:]
bytes_range = bytes_range.split('-')
extra_get_args['byte_range'] = (int(bytes_range[0]),
int(bytes_range[1]))
try:
bucket = self._connect_bucket(bucket_name)
data = bucket.get_object(key=key, **extra_get_args)
if stream:
return data
else:
return data.read()
except (oss2.exceptions.NoSuchKey, oss2.exceptions.NoSuchBucket):
raise StorageNoSuchKeyError(bucket_name, key)