def __init__(self): self.configuration = Configuration() self.mongo = Mongo() self.files = {} # Additional setup GenericFile.mongo = self.mongo GenericFile.configuration = self.configuration if self.configuration.is_development(): # START DEBUG ONLY - Drop the old information from MongoDB self.mongo.clean_database() self.mongo = Mongo() # Create a new instance to be sure to have the top folder
def setUp(self): Configuration.FILEPATH = 'test/resources/conf/mongofs.json' self.obj = Mongo() GenericFile.mongo = self.obj GenericFile.configuration = Configuration() self.utils = Utils(mongo=self.obj) self.utils.load_files()
def __init__(self): # We reuse the same connexion if MongoCache.instance is None: MongoCache.configuration = Configuration() self.reset_cache() retry_connection(self.connect()) retry_connection(self.load_internal())
def setUp(self): Configuration.FILEPATH = 'test/resources/conf/mongofs.json' self.mongo = Mongo(do_clean_up=True) GenericFile.mongo = self.mongo GenericFile.configuration = Configuration() self.utils = Utils(mongo=self.mongo) self.utils.load_files()
def __init__(self): """ This class manages configurations, it extracts the configurations strings from the config file and put the values in the *Configuration* class. """ logger.debug("(ConfigurationManager __init__)") self.config = configparser.ConfigParser() self.config.read("/usr/downloader/src/core/config/config.cfg") self.configuration = Configuration() self.load_configuration()
def __init__(self): # We reuse the same connexion Mongo.configuration = Configuration() Mongo.cache = MongoCache() # Collection name self.gridfs_coll = Mongo.configuration.mongo_prefix() + 'files' self.files_coll = Mongo.configuration.mongo_prefix() + 'files.files' self.chunks_coll = Mongo.configuration.mongo_prefix() + 'files.chunks' # We need to be sure to have the top folder created in MongoDB GenericFile.mongo = self GenericFile.new_generic_file(filepath='/', mode=0o755, file_type=GenericFile.DIRECTORY_TYPE) # Create the initial indexes self.create_indexes() # Temporary cache for the file data self.data_cache = {}
def __init__(self, do_clean_up=False): # We reuse the same connexion Mongo.configuration = Configuration() Mongo.cache = MongoCache() # Collection name self.gridfs_coll = Mongo.configuration.mongo_prefix() + 'files' self.files_coll = Mongo.configuration.mongo_prefix() + 'files.files' self.chunks_coll = Mongo.configuration.mongo_prefix() + 'files.chunks' if do_clean_up is True: self.clean_database() # Create the initial indexes self.create_indexes() # Temporary cache for the file data self.data_cache = {} # Temporary cache for user information self.user_cache = ExpiringDict(max_len=1000, max_age_seconds=2) # Temporary cache for group information self.group_cache = ExpiringDict(max_len=1000, max_age_seconds=2) # We need to be sure to have the top folder created in MongoDB GenericFile.mongo = self root = self.get_generic_file(filepath='/') default_root_mode = S_IFDIR | self.configuration.default_root_mode() if root is None: GenericFile.new_generic_file(filepath='/', mode=default_root_mode, file_type=GenericFile.DIRECTORY_TYPE) elif self.configuration.force_root_mode( ) and root.metadata['st_mode'] != default_root_mode: root.metadata['st_mode'] = default_root_mode root.basic_save()
class MongoFS(LoggingMixIn, Operations): # This is useful to be able to umount if there is an error to access MongoDB for example mounting_point = None def __init__(self): self.configuration = Configuration() self.mongo = Mongo() self.files = {} # Additional setup GenericFile.mongo = self.mongo GenericFile.configuration = self.configuration if self.configuration.is_development(): # START DEBUG ONLY - Drop the old information from MongoDB self.mongo.clean_database() self.mongo = Mongo() # Create a new instance to be sure to have the top folder # END DEBUG ONLY # The top folder of the FS is automatically created by the Mongo class. """ Create a file and returns a "file descriptor", which is in fact, simply the _id. """ def create(self, path, mode): file = GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.FILE_TYPE) return file.file_descriptor """ Acquire a lock on a specific file. """ def lock(self, path, fip, cmd, lock): # Getting the file object with a lock is enough to be sure there is one on it. self.mongo.get_generic_file(filepath=path, take_lock=True) return lock """ Release the lock on a specific file. """ def release(self, path, fh): print('Release the lock on '+str(path)+': '+str(fh)) gf = self.mongo.get_generic_file(filepath=path) print('Before unlock: '+str(gf.lock)) gf.unlock(filepath=path) return 0 """ Release the lock on a specific directory. """ def releasedir(self, path, fh): gf = self.mongo.get_generic_file(filepath=path) gf.unlock(filepath=path) return 0 """ Rename a generic file """ def rename(self, old, new): generic_file = self.mongo.get_generic_file(filepath=old) generic_file.rename_to(initial_filepath=old, destination_filepath=new) """ Create a symbolic link to a generic file (source -> target). No need to return a file descriptor. target: the file we want to display if we display "source" source: the symbolic link itself Important: It seems we receive the symlink parameters in a different order than "ln -s TARGET SYMLINK", we receive them (SYMLINK, TARGET) which is kinda weird. """ def symlink(self, source, target): GenericFile.new_generic_file(filepath=source, mode=0o777, file_type=GenericFile.SYMBOLIC_LINK_TYPE, target=target) """ Read a symbolic link and return the file we should be redirected to. """ def readlink(self, path): link = self.mongo.get_generic_file(filepath=path) return link.get_target() """ Read a part of a file """ def read(self, path, size, offset, fh): file = self.mongo.get_generic_file(filepath=path) tmp = file.read_data(offset=offset, size=size) return tmp """ Delete a file """ def unlink(self, path): file_or_link = self.mongo.get_generic_file(filepath=path) self.mongo.remove_generic_file(generic_file=file_or_link) """ Create a directory, no need to return anything """ def mkdir(self, path, mode): GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.DIRECTORY_TYPE) """ Delete a directory """ def rmdir(self, path): directory = self.mongo.get_generic_file(filepath=path) self.mongo.remove_generic_file(generic_file=directory) """ List files inside a directory """ def readdir(self, path, fh): files = self.mongo.list_generic_files_in_directory(filepath=path) # We need to only keep final filename filenames = [file.filename for file in files] return ['.', '..'] + filenames """ Write data to a file, from a specific offset. Returns the written data size """ def write(self, path, data, offset, fh): file = self.mongo.get_generic_file(filepath=path) file.add_data(data=data, offset=offset) return len(data) """ Truncate a file to a specific length """ def truncate(self, path, length, fh=None): file = self.mongo.get_generic_file(filepath=path) file.truncate(length=length) """ Return general information for a given path """ def getattr(self, path, fh=None): gf = self.mongo.get_generic_file(filepath=path) if gf is None: raise FuseOSError(ENOENT) return gf.metadata """ Set permissions to a given path """ def chmod(self, path, mode): gf = self.mongo.get_generic_file(filepath=path) gf.metadata['st_mode'] &= 0o770000 gf.metadata['st_mode'] |= mode gf.basic_save() return 0 """ Set owner (user & group) to a given path """ def chown(self, path, uid, gid): gf = self.mongo.get_generic_file(filepath=path) gf.metadata['st_uid'] = uid gf.metadata['st_gid'] = gid gf.basic_save() """ Return a specific special attribute for a given path (for selinux for example) """ def getxattr(self, path, name, position=0): gf = self.mongo.get_generic_file(filepath=path) try: return gf.attrs[name] except KeyError: return '' # Should return ENOATTR """ Return all special attributes for a given path (for selinux for example) """ def listxattr(self, path): gf = self.mongo.get_generic_file(filepath=path) return gf.attrs.keys() """ Remove a specific special attribute for a given path """ def removexattr(self, path, name): gf = self.mongo.get_generic_file(filepath=path) if name in gf.attrs: del gf.attrs[name] gf.basic_save() else: # Should return ENOATTR pass """ Update the access and update time for a given path """ def utimens(self, path, times=None): now = time.time() atime, mtime = times if times else (now, now) gf = self.mongo.get_generic_file(filepath=path) gf.metadata['st_atime'] = atime gf.metadata['st_mtime'] = mtime gf.basic_save() """ Update a specific special attribute for a given path """ def setxattr(self, path, name, value, options, position=0): gf = self.mongo.get_generic_file(filepath=path) gf.attrs[name] = value gf.basic_save() """ General (static) information about the current file system. """ def statfs(self, path): print('Call statfs for '+str(path)+'...') return dict(f_bsize=65536*8, f_frsize=65536*8, f_blocks=65536*8, f_bavail=65536) """ Flush data to MongoDB """ def flush(self, path, fh): file = self.mongo.get_generic_file(filepath=path) self.mongo.flush_data_to_write(file=file) return None
return dict(f_bsize=65536*8, f_frsize=65536*8, f_blocks=65536*8, f_bavail=65536) """ Flush data to MongoDB """ def flush(self, path, fh): file = self.mongo.get_generic_file(filepath=path) self.mongo.flush_data_to_write(file=file) return None if __name__ == '__main__': if len(argv) < 2: print('usage: %s <mountpoint> (<configuration_filepath>)' % argv[0]) exit(1) if len(argv) == 3: configuration_filepath = argv[2] Configuration.FILEPATH = configuration_filepath # If the previous mount failed, we need to be sure to umount it, otherwise you will not be able to mount anymore Configuration.mounting_point = argv[1] os.system('fusermount -u ' + str(argv[1])+' &>/dev/null') configuration = Configuration() if configuration.is_development(): #logging.basicConfig(level=logging.DEBUG) logging.basicConfig(level=logging.ERROR) fuse = FUSE(MongoFS(), argv[1], foreground=True, nothreads=True, allow_other=True) else: logging.basicConfig(level=logging.ERROR) fuse = FUSE(MongoFS(), argv[1], foreground=False, nothreads=False, allow_other=True)
class MongoFS(LoggingMixIn, Operations): # This is useful to be able to umount if there is an error to access MongoDB for example mounting_point = None def __init__(self): self.configuration = Configuration() self.mongo = Mongo() self.files = {} # Additional setup GenericFile.mongo = self.mongo GenericFile.configuration = self.configuration if self.configuration.is_development(): # START DEBUG ONLY - Drop the old information from MongoDB self.mongo.clean_database() self.mongo = Mongo( ) # Create a new instance to be sure to have the top folder # END DEBUG ONLY # The top folder of the FS is automatically created by the Mongo class. """ Create a file and returns a "file descriptor", which is in fact, simply the _id. """ def create(self, path, mode, fi=None): file = GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.FILE_TYPE) return file.file_descriptor """ Acquire a lock on a specific file. """ def lock(self, path, fh, cmd, lock): # lock is a pointer to a struct flock. The first field is a short # with the lock type, so we can just cast it to a POINTER(c_short) # We won't manage locking only part of file, so we don't care about the # complete struct lock_type_pointer = cast(lock, POINTER(c_short)) lock_type = lock_type_pointer[0] if cmd == fcntl.F_GETLK: blocking = self.mongo.test_lock_and_get_first_blocking( filepath=path, lock={'type': lock_type}) if blocking is None: # We modify the pointer by setting F_UNLCK lock_type_pointer[0] = fcntl.F_UNLCK else: # We modify the pointer by setting the blocking type of the locks lock_type_pointer[0] = blocking['type'] elif cmd == fcntl.F_SETLK or cmd == fcntl.F_SETLKW: if self.mongo.get_generic_file(filepath=path, lock={ 'type': lock_type, 'wait': cmd == fcntl.F_SETLKW }) is None: raise FuseOSError(errno.ENOENT) else: raise FuseOSError(errno.EBADF) return 0 """ Release the lock on a specific file. """ def release(self, path, fh): gf = self.mongo.get_generic_file(filepath=path) gf.release(filepath=path) return 0 """ Release the lock on a specific directory. """ def releasedir(self, path, fh): gf = self.mongo.get_generic_file(filepath=path) gf.release(filepath=path) return 0 """ Rename a generic file """ def rename(self, old, new): generic_file = self.mongo.get_generic_file(filepath=old) generic_file.rename_to(initial_filepath=old, destination_filepath=new) """ Create a symbolic link to a generic file (source -> target). No need to return a file descriptor. target: the file we want to display if we display "source" source: the symbolic link itself Important: It seems we receive the symlink parameters in a different order than "ln -s TARGET SYMLINK", we receive them (SYMLINK, TARGET) which is kinda weird. """ def symlink(self, source, target): GenericFile.new_generic_file(filepath=source, mode=0o777, file_type=GenericFile.SYMBOLIC_LINK_TYPE, target=target) """ Read a symbolic link and return the file we should be redirected to. """ def readlink(self, path): link = self.mongo.get_generic_file(filepath=path) return link.get_target() """ Read a part of a file """ def read(self, path, size, offset, fh): file = self.mongo.get_generic_file(filepath=path) tmp = file.read_data(offset=offset, size=size) return tmp """ Delete a file """ def unlink(self, path): file_or_link = self.mongo.get_generic_file(filepath=path) self.mongo.remove_generic_file(generic_file=file_or_link) """ Create a directory, no need to return anything """ def mkdir(self, path, mode): GenericFile.new_generic_file(filepath=path, mode=mode, file_type=GenericFile.DIRECTORY_TYPE) """ Delete a directory """ def rmdir(self, path): directory = self.mongo.get_generic_file(filepath=path) self.mongo.remove_generic_file(generic_file=directory) """ List files inside a directory """ def readdir(self, path, fh): files = self.mongo.list_generic_files_in_directory(filepath=path) # We need to only keep final filename filenames = [file.filename for file in files] return ['.', '..'] + filenames """ Write data to a file, from a specific offset. Returns the written data size """ def write(self, path, data, offset, fh): file = self.mongo.get_generic_file(filepath=path) file.add_data(data=data, offset=offset) return len(data) """ Truncate a file to a specific length """ def truncate(self, path, length, fh=None): file = self.mongo.get_generic_file(filepath=path) file.truncate(length=length) """ Return general information for a given path """ def getattr(self, path, fh=None): gf = self.mongo.get_generic_file(filepath=path) if gf is None: raise FuseOSError(errno.ENOENT) metadata = gf.metadata if gf.host != self.configuration.hostname(): if metadata['st_uid'] != 0: uid = self.mongo.get_userid(gf.uname) if uid is not None: metadata['st_uid'] = uid if metadata['st_gid'] != 0: gid = self.mongo.get_groupid(gf.gname) if gid is not None: metadata['st_gid'] = gid return metadata """ Set permissions to a given path """ def chmod(self, path, mode): if path == '/' and configuration.force_root_mode(): raise FuseOSError(errno.EACCESS) gf = self.mongo.get_generic_file(filepath=path) gf.metadata['st_mode'] &= 0o770000 gf.metadata['st_mode'] |= mode gf.basic_save() return 0 """ Set owner (user & group) to a given path """ def chown(self, path, uid, gid): gf = self.mongo.get_generic_file(filepath=path) gf.host = self.configuration.hostname() gf.metadata['st_uid'] = uid gf.metadata['st_gid'] = gid gf.uname = self.mongo.get_username(uid) gf.gname = self.mongo.get_groupname(gid) gf.basic_save() """ Return a specific special attribute for a given path (for selinux for example) """ def getxattr(self, path, name, position=0): gf = self.mongo.get_generic_file(filepath=path) try: return gf.attrs[name] except KeyError: return '' # Should return ENOATTR """ Return all special attributes for a given path (for selinux for example) """ def listxattr(self, path): gf = self.mongo.get_generic_file(filepath=path) return gf.attrs.keys() """ Remove a specific special attribute for a given path """ def removexattr(self, path, name): gf = self.mongo.get_generic_file(filepath=path) if name in gf.attrs: del gf.attrs[name] gf.basic_save() else: raise FuseOSError(errno.EACCES) """ Update the access and update time for a given path """ def utimens(self, path, times=None): now = time.time() atime, mtime = times if times else (now, now) gf = self.mongo.get_generic_file(filepath=path) gf.metadata['st_atime'] = atime gf.metadata['st_mtime'] = mtime gf.basic_save() """ Update a specific special attribute for a given path """ def setxattr(self, path, name, value, options, position=0): gf = self.mongo.get_generic_file(filepath=path) gf.attrs[name] = value gf.basic_save() """ General (static) information about the current file system. """ def statfs(self, path): print('Call statfs for ' + str(path) + '...') return dict(f_bsize=65536 * 8, f_frsize=65536 * 8, f_blocks=65536 * 8, f_bavail=65536) """ Flush data to MongoDB """ def flush(self, path, fh): file = self.mongo.get_generic_file(filepath=path) self.mongo.flush_data_to_write(file=file) return None
def setUpConfig(self, path): Configuration.FILEPATH = path GenericFile.configuration = Configuration() self.obj = Mongo(do_clean_up=True) GenericFile.mongo = self.obj
def setUp(self): Configuration.FILEPATH = 'test/resources/conf/mongofs.json' self.obj = Configuration()
class TestConfiguration(unittest.TestCase): def setUp(self): Configuration.FILEPATH = 'test/resources/conf/mongofs.json' self.obj = Configuration() def tearDown(self): pass def test_mongo_hosts(self): self.assertEqual(self.obj.mongo_hosts(), ["127.0.0.1:27017"]) def test_mongo_database(self): self.assertEqual(self.obj.mongo_database(), "mongofsTests") def test_mongo_prefix(self): self.assertEqual(self.obj.mongo_prefix(), "mongofsTests_") def test_mongo_access_attempt(self): self.assertEqual(self.obj.mongo_access_attempt(), 6) def test_mongo_access_attempt_infinite(self): self.obj.conf['mongo']['access_attempt_s'] = 0 self.assertTrue(self.obj.mongo_access_attempt() >= 3600 * 24 * 365) def test_lock_timeout(self): self.assertEqual(self.obj.lock_timeout(), 6) def test_lock_timeout_infinite(self): self.obj.conf['lock']['timeout_s'] = 0 self.assertTrue(self.obj.lock_timeout() >= 3600 * 24 * 365) def test_lock_access_attempt(self): self.assertEqual(self.obj.lock_access_attempt(), 6) def test_lock_access_attempt_infinite(self): self.obj.conf['lock']['access_attempt_s'] = 0 self.assertTrue(self.obj.lock_access_attempt() >= 3600 * 24 * 365) def test_cache_timeout(self): self.assertEqual(self.obj.cache_timeout(), 2) def test_cache_max_elements(self): self.assertEqual(self.obj.cache_max_elements(), 10000) def test_data_cache_timeout(self): self.assertEqual(self.obj.data_cache_timeout(), 2) def test_data_cache_max_elements(self): self.assertEqual(self.obj.data_cache_max_elements(), 50) def test_is_development(self): self.assertEqual(self.obj.is_development(), True) def test_hostname(self): self.assertEqual(self.obj.hostname(), "localhost")