def _validate_prefix(cls, path: str) -> None:
"""Validates the path prefix.
Args:
path: Path to the data object.
Raises:
ClientStorageError: When provided path is not valid.
"""
if not path.startswith("s3://"):
raise ClientStorageError("Path must start with 's3://'")
def save(self, data: bytes, path: str) -> None:
"""Saves data by delegation to the provided save method.
Args:
data: The data to be saved by provided save method.
path: Path to the file destination to store into.
Raises:
ClientStorageError: When arguments don't compline with the client requirements.
"""
self._logger.debug("Saving %s", str(self))
if data is None:
raise ClientStorageError("Saving `None` is not allowed")
try:
self._save(data, path)
except Exception as ex:
message = f"Failed while saving data to data set {str(self)}.\n{str(ex)}"
raise ClientStorageError(message) from ex
def _load(self, path: str) -> bytes:
path = f"{self.DIR_PATH}/{path}"
_reader: Callable = open
if path.endswith(".gz"):
_reader = gzip_open
try:
with _reader(path, "rb") as fread:
return fread.read()
except (FileExistsError, IOError, PermissionError) as ex:
raise ClientStorageError(ex) from ex
def _load(self, path: str) -> bytes:
self._validate_prefix(path)
bucket, path = self._split_path(path)
obj = self.client.get_object(Bucket=bucket, Key=path).get("Body")
if not obj:
raise ClientStorageError(f"No object {path} found")
if path.endswith(".gz"):
with GzipFile(fileobj=obj, mode="rb") as fread:
return fread.read()
return obj.read()
def _load(self, path: str) -> bytes:
self._validate_prefix(path)
if not self._exists(path):
raise ClientStorageError(f"No object {path} found")
bucket, path = self._split_path(path)
obj = self.client.bucket(bucket).blob(path).download_as_string()
if path.endswith(".gz"):
with GzipFile(fileobj=BytesIO(obj), mode="rb") as fread:
return fread.read()
return obj
def _save(self, data: bytes, path: str) -> None:
path = f"{self.DIR_PATH}/{path}"
_writer: Callable = open
if path.endswith(".gz"):
_writer = gzip_open
path_dir = dirname(path)
try:
if not isdir(path_dir):
makedirs(path_dir)
with _writer(path, "wb") as fwrite:
fwrite.write(data)
except (FileExistsError, IOError, PermissionError) as ex:
raise ClientStorageError(ex) from ex
def exists(self, path: str) -> bool:
"""Checks whether the file exists by calling the provided _exists() method.
Args:
path: Path to file to check.
Returns:
Flag indicating whether the file exists.
Raises:
ClientStorageError: When underlying exists method raises exception.
"""
try:
self._logger.debug("Checking whether target of %s exists",
str(self))
return self._exists(path)
except Exception as ex:
message = f"Failed during exists check for data set {str(self)}.\n{str(ex)}"
raise ClientStorageError(message) from ex
def load(self, path: str) -> bytes:
"""Loads data by delegation to the provided load method.
Args:
path: Path to file to read data.
Returns:
Bytes data returned by the provided load method.
Raises:
ClientStorageError: When underlying load method raises exception.
"""
self._logger.debug("Loading with %s", str(self))
try:
return self._load(path)
except Exception as ex:
message = f"Failed while loading data from data set {str(self)}.\n{str(ex)}"
raise ClientStorageError(message) from ex