def _numpy_saver(
fs: fsspec.AbstractFileSystem, filepath: str, array: np.ndarray, codec: BaseCodec
):
""" Saves a single numpy array into filepath given specific filesystem
"""
with fs.open(filepath, "wb") as f:
f.write(codec.encode(array))
def _numpy_load(fs: fsspec.AbstractFileSystem, filepath: str,
codec: BaseCodec) -> np.ndarray:
"""Given filesystem and filepath, loads numpy array"""
# assert fs.exists(
# filepath
# ), f"Dataset file {filepath} does not exists. Your dataset data is likely to be corrupted"
try:
with fs.open(filepath, "rb") as f:
return codec.decode(f.read())
except Exception as e:
logger.error(traceback.format_exc() + str(e))
raise Exception(
f"Dataset file {filepath} does not exists. Your dataset data is likely to be corrupted"
)