def __init__(self):
self.configuration = Configuration()
self.mongo = Mongo()
self.files = {}
# Additional setup
GenericFile.mongo = self.mongo
GenericFile.configuration = self.configuration
if self.configuration.is_development():
# START DEBUG ONLY - Drop the old information from MongoDB
self.mongo.clean_database()
self.mongo = Mongo() # Create a new instance to be sure to have the top folder
def setUp(self):
Configuration.FILEPATH = 'test/resources/conf/mongofs.json'
self.obj = Mongo()
GenericFile.mongo = self.obj
GenericFile.configuration = Configuration()
self.utils = Utils(mongo=self.obj)
self.utils.load_files()
def __init__(self):
# We reuse the same connexion
if MongoCache.instance is None:
MongoCache.configuration = Configuration()
self.reset_cache()
retry_connection(self.connect())
retry_connection(self.load_internal())
def setUp(self):
Configuration.FILEPATH = 'test/resources/conf/mongofs.json'
self.mongo = Mongo(do_clean_up=True)
GenericFile.mongo = self.mongo
GenericFile.configuration = Configuration()
self.utils = Utils(mongo=self.mongo)
self.utils.load_files()
def __init__(self):
""" This class manages configurations, it extracts the configurations strings from the
config file and put the values in the *Configuration* class.
"""
logger.debug("(ConfigurationManager __init__)")
self.config = configparser.ConfigParser()
self.config.read("/usr/downloader/src/core/config/config.cfg")
self.configuration = Configuration()
self.load_configuration()
def __init__(self):
# We reuse the same connexion
Mongo.configuration = Configuration()
Mongo.cache = MongoCache()
# Collection name
self.gridfs_coll = Mongo.configuration.mongo_prefix() + 'files'
self.files_coll = Mongo.configuration.mongo_prefix() + 'files.files'
self.chunks_coll = Mongo.configuration.mongo_prefix() + 'files.chunks'
# We need to be sure to have the top folder created in MongoDB
GenericFile.mongo = self
GenericFile.new_generic_file(filepath='/',
mode=0o755,
file_type=GenericFile.DIRECTORY_TYPE)
# Create the initial indexes
self.create_indexes()
# Temporary cache for the file data
self.data_cache = {}
def __init__(self, do_clean_up=False):
# We reuse the same connexion
Mongo.configuration = Configuration()
Mongo.cache = MongoCache()
# Collection name
self.gridfs_coll = Mongo.configuration.mongo_prefix() + 'files'
self.files_coll = Mongo.configuration.mongo_prefix() + 'files.files'
self.chunks_coll = Mongo.configuration.mongo_prefix() + 'files.chunks'
if do_clean_up is True:
self.clean_database()
# Create the initial indexes
self.create_indexes()
# Temporary cache for the file data
self.data_cache = {}
# Temporary cache for user information
self.user_cache = ExpiringDict(max_len=1000, max_age_seconds=2)
# Temporary cache for group information
self.group_cache = ExpiringDict(max_len=1000, max_age_seconds=2)
# We need to be sure to have the top folder created in MongoDB
GenericFile.mongo = self
root = self.get_generic_file(filepath='/')
default_root_mode = S_IFDIR | self.configuration.default_root_mode()
if root is None:
GenericFile.new_generic_file(filepath='/',
mode=default_root_mode,
file_type=GenericFile.DIRECTORY_TYPE)
elif self.configuration.force_root_mode(
) and root.metadata['st_mode'] != default_root_mode:
root.metadata['st_mode'] = default_root_mode
root.basic_save()
class MongoFS(LoggingMixIn, Operations):
# This is useful to be able to umount if there is an error to access MongoDB for example
mounting_point = None
def __init__(self):
self.configuration = Configuration()
self.mongo = Mongo()
self.files = {}
# Additional setup
GenericFile.mongo = self.mongo
GenericFile.configuration = self.configuration
if self.configuration.is_development():
# START DEBUG ONLY - Drop the old information from MongoDB
self.mongo.clean_database()
self.mongo = Mongo() # Create a new instance to be sure to have the top folder
# END DEBUG ONLY
# The top folder of the FS is automatically created by the Mongo class.
"""
Create a file and returns a "file descriptor", which is in fact, simply the _id.
"""
def create(self, path, mode):
file = GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.FILE_TYPE)
return file.file_descriptor
"""
Acquire a lock on a specific file.
"""
def lock(self, path, fip, cmd, lock):
# Getting the file object with a lock is enough to be sure there is one on it.
self.mongo.get_generic_file(filepath=path, take_lock=True)
return lock
"""
Release the lock on a specific file.
"""
def release(self, path, fh):
print('Release the lock on '+str(path)+': '+str(fh))
gf = self.mongo.get_generic_file(filepath=path)
print('Before unlock: '+str(gf.lock))
gf.unlock(filepath=path)
return 0
"""
Release the lock on a specific directory.
"""
def releasedir(self, path, fh):
gf = self.mongo.get_generic_file(filepath=path)
gf.unlock(filepath=path)
return 0
"""
Rename a generic file
"""
def rename(self, old, new):
generic_file = self.mongo.get_generic_file(filepath=old)
generic_file.rename_to(initial_filepath=old, destination_filepath=new)
"""
Create a symbolic link to a generic file (source -> target). No need to return a file descriptor.
target: the file we want to display if we display "source"
source: the symbolic link itself
Important: It seems we receive the symlink parameters in a different order than "ln -s TARGET SYMLINK",
we receive them (SYMLINK, TARGET) which is kinda weird.
"""
def symlink(self, source, target):
GenericFile.new_generic_file(filepath=source, mode=0o777, file_type=GenericFile.SYMBOLIC_LINK_TYPE, target=target)
"""
Read a symbolic link and return the file we should be redirected to.
"""
def readlink(self, path):
link = self.mongo.get_generic_file(filepath=path)
return link.get_target()
"""
Read a part of a file
"""
def read(self, path, size, offset, fh):
file = self.mongo.get_generic_file(filepath=path)
tmp = file.read_data(offset=offset, size=size)
return tmp
"""
Delete a file
"""
def unlink(self, path):
file_or_link = self.mongo.get_generic_file(filepath=path)
self.mongo.remove_generic_file(generic_file=file_or_link)
"""
Create a directory, no need to return anything
"""
def mkdir(self, path, mode):
GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.DIRECTORY_TYPE)
"""
Delete a directory
"""
def rmdir(self, path):
directory = self.mongo.get_generic_file(filepath=path)
self.mongo.remove_generic_file(generic_file=directory)
"""
List files inside a directory
"""
def readdir(self, path, fh):
files = self.mongo.list_generic_files_in_directory(filepath=path)
# We need to only keep final filename
filenames = [file.filename for file in files]
return ['.', '..'] + filenames
"""
Write data to a file, from a specific offset. Returns the written data size
"""
def write(self, path, data, offset, fh):
file = self.mongo.get_generic_file(filepath=path)
file.add_data(data=data, offset=offset)
return len(data)
"""
Truncate a file to a specific length
"""
def truncate(self, path, length, fh=None):
file = self.mongo.get_generic_file(filepath=path)
file.truncate(length=length)
"""
Return general information for a given path
"""
def getattr(self, path, fh=None):
gf = self.mongo.get_generic_file(filepath=path)
if gf is None:
raise FuseOSError(ENOENT)
return gf.metadata
"""
Set permissions to a given path
"""
def chmod(self, path, mode):
gf = self.mongo.get_generic_file(filepath=path)
gf.metadata['st_mode'] &= 0o770000
gf.metadata['st_mode'] |= mode
gf.basic_save()
return 0
"""
Set owner (user & group) to a given path
"""
def chown(self, path, uid, gid):
gf = self.mongo.get_generic_file(filepath=path)
gf.metadata['st_uid'] = uid
gf.metadata['st_gid'] = gid
gf.basic_save()
"""
Return a specific special attribute for a given path (for selinux for example)
"""
def getxattr(self, path, name, position=0):
gf = self.mongo.get_generic_file(filepath=path)
try:
return gf.attrs[name]
except KeyError:
return '' # Should return ENOATTR
"""
Return all special attributes for a given path (for selinux for example)
"""
def listxattr(self, path):
gf = self.mongo.get_generic_file(filepath=path)
return gf.attrs.keys()
"""
Remove a specific special attribute for a given path
"""
def removexattr(self, path, name):
gf = self.mongo.get_generic_file(filepath=path)
if name in gf.attrs:
del gf.attrs[name]
gf.basic_save()
else:
# Should return ENOATTR
pass
"""
Update the access and update time for a given path
"""
def utimens(self, path, times=None):
now = time.time()
atime, mtime = times if times else (now, now)
gf = self.mongo.get_generic_file(filepath=path)
gf.metadata['st_atime'] = atime
gf.metadata['st_mtime'] = mtime
gf.basic_save()
"""
Update a specific special attribute for a given path
"""
def setxattr(self, path, name, value, options, position=0):
gf = self.mongo.get_generic_file(filepath=path)
gf.attrs[name] = value
gf.basic_save()
"""
General (static) information about the current file system.
"""
def statfs(self, path):
print('Call statfs for '+str(path)+'...')
return dict(f_bsize=65536*8, f_frsize=65536*8, f_blocks=65536*8, f_bavail=65536)
"""
Flush data to MongoDB
"""
def flush(self, path, fh):
file = self.mongo.get_generic_file(filepath=path)
self.mongo.flush_data_to_write(file=file)
return None
return dict(f_bsize=65536*8, f_frsize=65536*8, f_blocks=65536*8, f_bavail=65536)
"""
Flush data to MongoDB
"""
def flush(self, path, fh):
file = self.mongo.get_generic_file(filepath=path)
self.mongo.flush_data_to_write(file=file)
return None
if __name__ == '__main__':
if len(argv) < 2:
print('usage: %s <mountpoint> (<configuration_filepath>)' % argv[0])
exit(1)
if len(argv) == 3:
configuration_filepath = argv[2]
Configuration.FILEPATH = configuration_filepath
# If the previous mount failed, we need to be sure to umount it, otherwise you will not be able to mount anymore
Configuration.mounting_point = argv[1]
os.system('fusermount -u ' + str(argv[1])+' &>/dev/null')
configuration = Configuration()
if configuration.is_development():
#logging.basicConfig(level=logging.DEBUG)
logging.basicConfig(level=logging.ERROR)
fuse = FUSE(MongoFS(), argv[1], foreground=True, nothreads=True, allow_other=True)
else:
logging.basicConfig(level=logging.ERROR)
fuse = FUSE(MongoFS(), argv[1], foreground=False, nothreads=False, allow_other=True)
class MongoFS(LoggingMixIn, Operations):
# This is useful to be able to umount if there is an error to access MongoDB for example
mounting_point = None
def __init__(self):
self.configuration = Configuration()
self.mongo = Mongo()
self.files = {}
# Additional setup
GenericFile.mongo = self.mongo
GenericFile.configuration = self.configuration
if self.configuration.is_development():
# START DEBUG ONLY - Drop the old information from MongoDB
self.mongo.clean_database()
self.mongo = Mongo(
) # Create a new instance to be sure to have the top folder
# END DEBUG ONLY
# The top folder of the FS is automatically created by the Mongo class.
"""
Create a file and returns a "file descriptor", which is in fact, simply the _id.
"""
def create(self, path, mode, fi=None):
file = GenericFile.new_generic_file(filepath=path,
mode=mode,
file_type=GenericFile.FILE_TYPE)
return file.file_descriptor
"""
Acquire a lock on a specific file.
"""
def lock(self, path, fh, cmd, lock):
# lock is a pointer to a struct flock. The first field is a short
# with the lock type, so we can just cast it to a POINTER(c_short)
# We won't manage locking only part of file, so we don't care about the
# complete struct
lock_type_pointer = cast(lock, POINTER(c_short))
lock_type = lock_type_pointer[0]
if cmd == fcntl.F_GETLK:
blocking = self.mongo.test_lock_and_get_first_blocking(
filepath=path, lock={'type': lock_type})
if blocking is None:
# We modify the pointer by setting F_UNLCK
lock_type_pointer[0] = fcntl.F_UNLCK
else:
# We modify the pointer by setting the blocking type of the locks
lock_type_pointer[0] = blocking['type']
elif cmd == fcntl.F_SETLK or cmd == fcntl.F_SETLKW:
if self.mongo.get_generic_file(filepath=path,
lock={
'type': lock_type,
'wait': cmd == fcntl.F_SETLKW
}) is None:
raise FuseOSError(errno.ENOENT)
else:
raise FuseOSError(errno.EBADF)
return 0
"""
Release the lock on a specific file.
"""
def release(self, path, fh):
gf = self.mongo.get_generic_file(filepath=path)
gf.release(filepath=path)
return 0
"""
Release the lock on a specific directory.
"""
def releasedir(self, path, fh):
gf = self.mongo.get_generic_file(filepath=path)
gf.release(filepath=path)
return 0
"""
Rename a generic file
"""
def rename(self, old, new):
generic_file = self.mongo.get_generic_file(filepath=old)
generic_file.rename_to(initial_filepath=old, destination_filepath=new)
"""
Create a symbolic link to a generic file (source -> target). No need to return a file descriptor.
target: the file we want to display if we display "source"
source: the symbolic link itself
Important: It seems we receive the symlink parameters in a different order than "ln -s TARGET SYMLINK",
we receive them (SYMLINK, TARGET) which is kinda weird.
"""
def symlink(self, source, target):
GenericFile.new_generic_file(filepath=source,
mode=0o777,
file_type=GenericFile.SYMBOLIC_LINK_TYPE,
target=target)
"""
Read a symbolic link and return the file we should be redirected to.
"""
def readlink(self, path):
link = self.mongo.get_generic_file(filepath=path)
return link.get_target()
"""
Read a part of a file
"""
def read(self, path, size, offset, fh):
file = self.mongo.get_generic_file(filepath=path)
tmp = file.read_data(offset=offset, size=size)
return tmp
"""
Delete a file
"""
def unlink(self, path):
file_or_link = self.mongo.get_generic_file(filepath=path)
self.mongo.remove_generic_file(generic_file=file_or_link)
"""
Create a directory, no need to return anything
"""
def mkdir(self, path, mode):
GenericFile.new_generic_file(filepath=path,
mode=mode,
file_type=GenericFile.DIRECTORY_TYPE)
"""
Delete a directory
"""
def rmdir(self, path):
directory = self.mongo.get_generic_file(filepath=path)
self.mongo.remove_generic_file(generic_file=directory)
"""
List files inside a directory
"""
def readdir(self, path, fh):
files = self.mongo.list_generic_files_in_directory(filepath=path)
# We need to only keep final filename
filenames = [file.filename for file in files]
return ['.', '..'] + filenames
"""
Write data to a file, from a specific offset. Returns the written data size
"""
def write(self, path, data, offset, fh):
file = self.mongo.get_generic_file(filepath=path)
file.add_data(data=data, offset=offset)
return len(data)
"""
Truncate a file to a specific length
"""
def truncate(self, path, length, fh=None):
file = self.mongo.get_generic_file(filepath=path)
file.truncate(length=length)
"""
Return general information for a given path
"""
def getattr(self, path, fh=None):
gf = self.mongo.get_generic_file(filepath=path)
if gf is None:
raise FuseOSError(errno.ENOENT)
metadata = gf.metadata
if gf.host != self.configuration.hostname():
if metadata['st_uid'] != 0:
uid = self.mongo.get_userid(gf.uname)
if uid is not None:
metadata['st_uid'] = uid
if metadata['st_gid'] != 0:
gid = self.mongo.get_groupid(gf.gname)
if gid is not None:
metadata['st_gid'] = gid
return metadata
"""
Set permissions to a given path
"""
def chmod(self, path, mode):
if path == '/' and configuration.force_root_mode():
raise FuseOSError(errno.EACCESS)
gf = self.mongo.get_generic_file(filepath=path)
gf.metadata['st_mode'] &= 0o770000
gf.metadata['st_mode'] |= mode
gf.basic_save()
return 0
"""
Set owner (user & group) to a given path
"""
def chown(self, path, uid, gid):
gf = self.mongo.get_generic_file(filepath=path)
gf.host = self.configuration.hostname()
gf.metadata['st_uid'] = uid
gf.metadata['st_gid'] = gid
gf.uname = self.mongo.get_username(uid)
gf.gname = self.mongo.get_groupname(gid)
gf.basic_save()
"""
Return a specific special attribute for a given path (for selinux for example)
"""
def getxattr(self, path, name, position=0):
gf = self.mongo.get_generic_file(filepath=path)
try:
return gf.attrs[name]
except KeyError:
return '' # Should return ENOATTR
"""
Return all special attributes for a given path (for selinux for example)
"""
def listxattr(self, path):
gf = self.mongo.get_generic_file(filepath=path)
return gf.attrs.keys()
"""
Remove a specific special attribute for a given path
"""
def removexattr(self, path, name):
gf = self.mongo.get_generic_file(filepath=path)
if name in gf.attrs:
del gf.attrs[name]
gf.basic_save()
else:
raise FuseOSError(errno.EACCES)
"""
Update the access and update time for a given path
"""
def utimens(self, path, times=None):
now = time.time()
atime, mtime = times if times else (now, now)
gf = self.mongo.get_generic_file(filepath=path)
gf.metadata['st_atime'] = atime
gf.metadata['st_mtime'] = mtime
gf.basic_save()
"""
Update a specific special attribute for a given path
"""
def setxattr(self, path, name, value, options, position=0):
gf = self.mongo.get_generic_file(filepath=path)
gf.attrs[name] = value
gf.basic_save()
"""
General (static) information about the current file system.
"""
def statfs(self, path):
print('Call statfs for ' + str(path) + '...')
return dict(f_bsize=65536 * 8,
f_frsize=65536 * 8,
f_blocks=65536 * 8,
f_bavail=65536)
"""
Flush data to MongoDB
"""
def flush(self, path, fh):
file = self.mongo.get_generic_file(filepath=path)
self.mongo.flush_data_to_write(file=file)
return None
def setUpConfig(self, path):
Configuration.FILEPATH = path
GenericFile.configuration = Configuration()
self.obj = Mongo(do_clean_up=True)
GenericFile.mongo = self.obj
def setUp(self):
Configuration.FILEPATH = 'test/resources/conf/mongofs.json'
self.obj = Configuration()
class TestConfiguration(unittest.TestCase):
def setUp(self):
Configuration.FILEPATH = 'test/resources/conf/mongofs.json'
self.obj = Configuration()
def tearDown(self):
pass
def test_mongo_hosts(self):
self.assertEqual(self.obj.mongo_hosts(), ["127.0.0.1:27017"])
def test_mongo_database(self):
self.assertEqual(self.obj.mongo_database(), "mongofsTests")
def test_mongo_prefix(self):
self.assertEqual(self.obj.mongo_prefix(), "mongofsTests_")
def test_mongo_access_attempt(self):
self.assertEqual(self.obj.mongo_access_attempt(), 6)
def test_mongo_access_attempt_infinite(self):
self.obj.conf['mongo']['access_attempt_s'] = 0
self.assertTrue(self.obj.mongo_access_attempt() >= 3600 * 24 * 365)
def test_lock_timeout(self):
self.assertEqual(self.obj.lock_timeout(), 6)
def test_lock_timeout_infinite(self):
self.obj.conf['lock']['timeout_s'] = 0
self.assertTrue(self.obj.lock_timeout() >= 3600 * 24 * 365)
def test_lock_access_attempt(self):
self.assertEqual(self.obj.lock_access_attempt(), 6)
def test_lock_access_attempt_infinite(self):
self.obj.conf['lock']['access_attempt_s'] = 0
self.assertTrue(self.obj.lock_access_attempt() >= 3600 * 24 * 365)
def test_cache_timeout(self):
self.assertEqual(self.obj.cache_timeout(), 2)
def test_cache_max_elements(self):
self.assertEqual(self.obj.cache_max_elements(), 10000)
def test_data_cache_timeout(self):
self.assertEqual(self.obj.data_cache_timeout(), 2)
def test_data_cache_max_elements(self):
self.assertEqual(self.obj.data_cache_max_elements(), 50)
def test_is_development(self):
self.assertEqual(self.obj.is_development(), True)
def test_hostname(self):
self.assertEqual(self.obj.hostname(), "localhost")
