def __initialize__(self):
grid_file = None
cursor = self.fs.find({"filename": self.filename})
for requested_file in cursor:
exception_file = next(cursor, None)
if exception_file:
raise FsException("Multiple files found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if requested_file.metadata["type"] in ("file", "sym"):
grid_file = requested_file
self.file_type = requested_file.metadata["type"]
else:
raise FsException("Type isn't file",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if grid_file:
self._id = grid_file._id
self.fs.download_to_stream(self._id, self)
if "r" in self.mode:
self.seek(0, 0)
def dir_ls(self, storage):
"""
return folder content
:param storage: can be a str or list of str
:return: folder content
"""
try:
f = storage if isinstance(storage, str) else "/".join(storage)
files = []
dir_cursor = self.fs.find({"filename": f})
for requested_dir in dir_cursor:
exception_dir = next(dir_cursor, None)
if exception_dir:
raise FsException(
"Multiple directories found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if requested_dir.metadata["type"] != "dir":
raise FsException("File {} does not exist".format(f),
http_code=HTTPStatus.NOT_FOUND)
files_cursor = self.fs.find(
{"filename": {
"$regex": "^{}/([^/])*".format(f)
}})
for children_file in files_cursor:
files += [children_file.filename.replace(f + '/', '', 1)]
return files
except IOError:
raise FsException("File {} cannot be opened".format(f),
http_code=HTTPStatus.BAD_REQUEST)
def __initialize__(self):
grid_file = None
cursor = self.fs.find({"filename": self.filename})
for requested_file in cursor:
exception_file = next(cursor, None)
if exception_file:
raise FsException("Multiple files found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if requested_file.metadata["type"] in ("file", "dir"):
grid_file = requested_file
self.file_type = requested_file.metadata["type"]
else:
raise FsException("File type isn't file",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if grid_file:
stream = BytesIO()
self._id = grid_file._id
self.fs.download_to_stream(self._id, stream)
stream.seek(0)
self.write(stream.read().decode("utf-8"))
stream.close()
if "r" in self.mode:
self.seek(0, 0)
def fs_connect(self, config):
try:
if "logger_name" in config:
self.logger = logging.getLogger(config["logger_name"])
self.path = config["path"]
if not self.path.endswith("/"):
self.path += "/"
if not os.path.exists(self.path):
raise FsException("Invalid configuration param at '[storage]': path '{}' does not exist".format(
config["path"]))
except FsException:
raise
except Exception as e: # TODO refine
raise FsException(str(e))
def dir_ls(self, storage):
"""
return folder content
:param storage: can be a str or list of str
:return: folder content
"""
try:
if isinstance(storage, str):
f = storage
else:
f = "/".join(storage)
return os.listdir(self.path + f)
except NotADirectoryError:
raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
except IOError:
raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
def file_exists(self, storage, mode=None):
"""
Indicates if "storage" file exist
:param storage: can be a str or a str list
:param mode: can be 'file' exist as a regular file; 'dir' exists as a directory or; 'None' just exists
:return: True, False
"""
f = storage if isinstance(storage, str) else "/".join(storage)
cursor = self.fs.find({"filename": f})
for requested_file in cursor:
exception_file = next(cursor, None)
if exception_file:
raise FsException("Multiple files found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if requested_file.metadata["type"] == mode:
return True
if requested_file.metadata["type"] == "sym" and mode == "file":
return True
return False
def dir_rename(self, src, dst):
"""
Rename one directory name. If dst exist, it replaces (deletes) existing directory
:param src: source directory
:param dst: destination directory
:return: None or raises and exception
"""
try:
dst_cursor = self.fs.find(
{"filename": {
"$regex": "^{}(/|$)".format(dst)
}},
no_cursor_timeout=True)
for dst_file in dst_cursor:
self.fs.delete(dst_file._id)
src_cursor = self.fs.find(
{"filename": {
"$regex": "^{}(/|$)".format(src)
}},
no_cursor_timeout=True)
for src_file in src_cursor:
self.fs.rename(src_file._id,
src_file.filename.replace(src, dst, 1))
except Exception as e:
raise FsException(str(e),
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def file_open(self, storage, mode):
"""
Open a file
:param storage: can be a str or list of str
:param mode: file mode
:return: file object
"""
try:
if isinstance(storage, str):
f = storage
else:
f = "/".join(storage)
return open(self.path + f, mode)
except FileNotFoundError:
raise FsException("File {} does not exist".format(f), http_code=HTTPStatus.NOT_FOUND)
except IOError:
raise FsException("File {} cannot be opened".format(f), http_code=HTTPStatus.BAD_REQUEST)
def file_delete(self, storage, ignore_non_exist=False):
"""
Delete storage content recursively
:param storage: can be a str or list of str
:param ignore_non_exist: not raise exception if storage does not exist
:return: None
"""
try:
if isinstance(storage, str):
f = self.path + storage
else:
f = self.path + "/".join(storage)
if os.path.exists(f):
rmtree(f)
elif not ignore_non_exist:
raise FsException("File {} does not exist".format(storage), http_code=HTTPStatus.NOT_FOUND)
except (IOError, PermissionError) as e:
raise FsException("File {} cannot be deleted: {}".format(f, e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def file_open(self, storage, mode):
"""
Open a file
:param storage: can be a str or list of str
:param mode: file mode
:return: file object
"""
try:
f = storage if isinstance(storage, str) else "/".join(storage)
if "b" in mode:
return GridByteStream(f, self.fs, mode)
else:
return GridStringStream(f, self.fs, mode)
except errors.NoFile:
raise FsException("File {} does not exist".format(f),
http_code=HTTPStatus.NOT_FOUND)
except IOError:
raise FsException("File {} cannot be opened".format(f),
http_code=HTTPStatus.BAD_REQUEST)
def mkdir(self, folder):
"""
Creates a folder or parent object location
:param folder:
:return: None or raises and exception
"""
try:
os.mkdir(self.path + folder)
except FileExistsError: # make it idempotent
pass
except Exception as e:
raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def fs_connect(self, config):
try:
if "logger_name" in config:
self.logger = logging.getLogger(config["logger_name"])
if "path" in config:
self.path = config["path"]
else:
raise FsException("Missing parameter \"path\"")
if not self.path.endswith("/"):
self.path += "/"
if not os.path.exists(self.path):
raise FsException(
"Invalid configuration param at '[storage]': path '{}' does not exist"
.format(config["path"]))
elif not os.access(self.path, os.W_OK):
raise FsException(
"Invalid configuration param at '[storage]': path '{}' is not writable"
.format(config["path"]))
if all(key in config.keys() for key in ["uri", "collection"]):
self.client = MongoClient(config["uri"])
self.fs = GridFSBucket(self.client[config["collection"]])
elif all(key in config.keys()
for key in ["host", "port", "collection"]):
self.client = MongoClient(config["host"], config["port"])
self.fs = GridFSBucket(self.client[config["collection"]])
else:
if "collection" not in config.keys():
raise FsException("Missing parameter \"collection\"")
else:
raise FsException(
"Missing parameters: \"uri\" or \"host\" + \"port\"")
except FsException:
raise
except Exception as e: # TODO refine
raise FsException(str(e))
def file_delete(self, storage, ignore_non_exist=False):
"""
Delete storage content recursively
:param storage: can be a str or list of str
:param ignore_non_exist: not raise exception if storage does not exist
:return: None
"""
try:
f = storage if isinstance(storage, str) else "/".join(storage)
file_cursor = self.fs.find({"filename": f})
found = False
for requested_file in file_cursor:
found = True
exception_file = next(file_cursor, None)
if exception_file:
raise FsException(
"Multiple files found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
if requested_file.metadata["type"] == "dir":
dir_cursor = self.fs.find(
{"filename": {
"$regex": "^{}".format(f)
}})
for tmp in dir_cursor:
self.fs.delete(tmp._id)
else:
self.fs.delete(requested_file._id)
if not found and not ignore_non_exist:
raise FsException("File {} does not exist".format(storage),
http_code=HTTPStatus.NOT_FOUND)
except IOError as e:
raise FsException("File {} cannot be deleted: {}".format(f, e),
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def dir_rename(self, src, dst):
"""
Rename one directory name. If dst exist, it replaces (deletes) existing directory
:param src: source directory
:param dst: destination directory
:return: None or raises and exception
"""
try:
if os.path.exists(self.path + dst):
rmtree(self.path + dst)
os.rename(self.path + src, self.path + dst)
except Exception as e:
raise FsException(str(e), http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def mkdir(self, folder):
"""
Creates a folder or parent object location
:param folder:
:return: None or raises an exception
"""
try:
self.fs.upload_from_stream(folder,
BytesIO(),
metadata={"type": "dir"})
except errors.FileExists: # make it idempotent
pass
except Exception as e:
raise FsException(str(e),
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
def file_size(self, storage):
"""
return file size
:param storage: can be a str or a str list
:return: file size
"""
f = storage if isinstance(storage, str) else "/".join(storage)
cursor = self.fs.find({"filename": f})
for requested_file in cursor:
exception_file = next(cursor, None)
if exception_file:
raise FsException("Multiple files found",
http_code=HTTPStatus.INTERNAL_SERVER_ERROR)
return requested_file.length