def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from localhost filesystem with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ buffer = None try: file_path = os.path.join(STORAGE_FOLDER, bucket_name, key) with open(file_path, "rb") as f: if 'Range' in extra_get_args: byte_range = extra_get_args['Range'].replace('bytes=', '') first_byte, last_byte = map(int, byte_range.split('-')) f.seek(first_byte) buffer = io.BytesIO(f.read(last_byte-first_byte+1)) else: buffer = io.BytesIO(f.read()) if stream: return buffer else: return buffer.read() except Exception: raise StorageNoSuchKeyError(os.path.join(STORAGE_FOLDER, bucket_name), key)
def put_object(self, bucket_name, key, data): """ Put an object in COS. Override the object if the key already exists. :param key: key of the object. :param data: data of the object :type data: str/bytes :return: None """ retries = 0 status = None while status is None: try: res = self.cos_client.put_object(Bucket=bucket_name, Key=key, Body=data) status = 'OK' if res['ResponseMetadata']['HTTPStatusCode'] == 200 else 'Error' try: logger.debug('PUT Object {} - Size: {} - {}'.format(key, sizeof_fmt(len(data)), status)) except Exception: logger.debug('PUT Object {} {}'.format(key, status)) except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "NoSuchKey": raise StorageNoSuchKeyError(bucket_name, key) else: raise e except ibm_botocore.exceptions.ReadTimeoutError as e: if retries == OBJ_REQ_RETRIES: raise e logger.debug('PUT Object timeout. Retrying request') retries += 1 return True
def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ data = None retries = 0 while data is None: try: r = self.cos_client.get_object(Bucket=bucket_name, Key=key, **extra_get_args) if stream: data = r['Body'] else: data = r['Body'].read() except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "NoSuchKey": raise StorageNoSuchKeyError(bucket_name, key) else: raise e except ibm_botocore.exceptions.ReadTimeoutError as e: if retries == OBJ_REQ_RETRIES: raise e logger.debug('GET Object timeout. Retrying request') retries += 1 return data
def list_keys(self, bucket_name, prefix=None): """ Return a list of keys for the given prefix. :param bucket_name: Name of the bucket. :param prefix: Prefix to filter object names. :return: List of keys in bucket that match the given prefix. :rtype: list of str """ try: prefix = '' if prefix is None else prefix paginator = self.cos_client.get_paginator('list_objects_v2') page_iterator = paginator.paginate(Bucket=bucket_name, Prefix=prefix) key_list = [] for page in page_iterator: if 'Contents' in page: for item in page['Contents']: key_list.append(item['Key']) return key_list except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': raise StorageNoSuchKeyError(bucket_name, prefix) else: raise e
def bucket_exists(self, bucket_name): """ Returns True if container exists in storage. Throws StorageNoSuchKeyError if the given container does not exist. :param bucket_name: name of the container """ try: self.blob_client.get_container_metadata(bucket_name) return True except Exception: raise StorageNoSuchKeyError(bucket_name, '')
def head_bucket(self, bucket_name): """ Head container from COS with a name. Throws StorageNoSuchKeyError if the given container does not exist. :param bucket_name: name of the container :return: Data of the object """ try: return self.blob_client.get_container_metadata(bucket_name) except Exception: raise StorageNoSuchKeyError(bucket_name, '')
def bucket_exists(self, bucket_name): """ Head bucket from COS with a name. Throws StorageNoSuchKeyError if the given bucket does not exist. :param bucket_name: name of the bucket """ try: self.cos_client.head_bucket(Bucket=bucket_name) except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': raise StorageNoSuchKeyError(bucket_name, '') else: raise e
def list_keys(self, bucket_name, prefix=None): """ Return a list of keys for the given prefix. :param prefix: Prefix to filter object names. :return: List of keys in bucket that match the given prefix. :rtype: list of str """ try: keys = [key for key in self.blob_client.list_blob_names(bucket_name, prefix).items] return keys except Exception: raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix)
def head_bucket(self, bucket_name): """ Head bucket from Ceph with a name. Throws StorageNoSuchKeyError if the given bucket does not exist. :param bucket_name: name of the bucket :return: Metadata of the bucket :rtype: str/bytes """ try: return self.cos_client.head_bucket(Bucket=bucket_name) except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': raise StorageNoSuchKeyError(bucket_name, '') else: raise e
def head_object(self, bucket_name, key): """ Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ try: metadata = self.cos_client.head_object(Bucket=bucket_name, Key=key) return metadata['ResponseMetadata']['HTTPHeaders'] except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': raise StorageNoSuchKeyError(bucket_name, key) else: raise e
def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from localhost filesystem with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ try: file_path = "{}/{}".format(bucket_name, key) with self.fs.open(file_path, "rb") as f: if 'Range' in extra_get_args: byte_range = extra_get_args['Range'].replace('bytes=', '') first_byte, last_byte = map(int, byte_range.split('-')) f.seek(first_byte) return f.read(last_byte - first_byte + 1) else: return f.read() except Exception as e: raise StorageNoSuchKeyError(bucket_name, key)
def list_objects(self, bucket_name, prefix=None): """ Return a list of objects for the given bucket and prefix. :param bucket_name: Name of the bucket. :param prefix: Prefix to filter object names. :return: List of objects in bucket that match the given prefix. :rtype: list of str """ # adapted to match ibm_cos method try: blobs = self.blob_client.list_blobs(bucket_name, prefix) mod_list = [] for blob in blobs: mod_list.append({ 'Key' : blob.name, 'Size' : blob.properties.content_length }) return mod_list except Exception: raise StorageNoSuchKeyError(bucket_name, '' if prefix is None else prefix)
def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ try: r = self.cos_client.get_object(Bucket=bucket_name, Key=key, **extra_get_args) if stream: data = r['Body'] else: data = r['Body'].read() return data except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "NoSuchKey": raise StorageNoSuchKeyError(bucket_name, key) else: raise e
def head_object(self, bucket_name, key): """ Head object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ metadata = None retries = 0 while metadata is None: try: metadata = self.cos_client.head_object(Bucket=bucket_name, Key=key) except ibm_botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == '404': raise StorageNoSuchKeyError(bucket_name, key) else: raise e except ibm_botocore.exceptions.ReadTimeoutError as e: if retries == OBJ_REQ_RETRIES: raise e logger.debug('HEAD Object timeout. Retrying request') retries += 1 return metadata['ResponseMetadata']['HTTPHeaders']
def get_object(self, bucket_name, key, stream=False, extra_get_args={}): """ Get object from COS with a key. Throws StorageNoSuchKeyError if the given key does not exist. :param key: key of the object :return: Data of the object :rtype: str/bytes """ if 'Range' in extra_get_args: # expected common format: Range='bytes=L-H' bytes_range = extra_get_args.pop('Range')[6:] bytes_range = bytes_range.split('-') extra_get_args['start_range'] = int(bytes_range[0]) extra_get_args['end_range'] = int(bytes_range[1]) try: if stream: stream_out = BytesIO() self.blob_client.get_blob_to_stream(bucket_name, key, stream_out, **extra_get_args) stream_out.seek(0) return stream_out else: data = self.blob_client.get_blob_to_bytes(bucket_name, key, **extra_get_args) return data.content except AzureMissingResourceHttpError: raise StorageNoSuchKeyError(bucket_name, key)