Beispiel #1
0
    def get_file(self, path):
        name = self._get_filename(path)
        debug('get_file', path, name)
        if name != None and name in self.files:
            return self.files[name]

        return None
Beispiel #2
0
    def    get_file(self, path):
        name = self._get_filename(path)
        debug('get_file', path, name)
        if name != None and name in self.files:
            return self.files[name]

        return None
Beispiel #3
0
    def __init__(self, name, callback_on_read, callback_on_change = None):
        VirtualFile.__init__(self, name)

        self.callback_on_read = callback_on_read
        self.callback_on_change = callback_on_change

        self.content = None
        debug('SVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', coc: ' + str(callback_on_change))
Beispiel #4
0
    def open(self, path, flags):
        debug('PersistentCacheFs.open', path, flags)
        if self.vfs.contains(path):
            return self.vfs.open(path, flags)

        if not is_read_only_flags(flags):
            return E_PERM_DENIED

        return 0
Beispiel #5
0
    def readdir(self, path, offset):
        debug('PersistentCacheFs.readdir', path, offset)
        for f in self.vfs.readdir(path, offset):
            if f is None:
                return
            yield f

        for f in self.cacher.readdir(path, offset):
            yield f
Beispiel #6
0
    def read(self, path, size, offset):
        debug('ufs.read', path, str(size), str(offset))
        real_path = self._get_real_path(path)

        with __builtin__.open(real_path, 'rb') as f:
            f.seek(offset)
            result = f.read(size)

        return result
Beispiel #7
0
    def __init__(self, name, callback_on_read, callback_on_change=None):
        VirtualFile.__init__(self, name)

        self.callback_on_read = callback_on_read
        self.callback_on_change = callback_on_change

        self.content = None
        debug('SVF init, name: ' + str(name) + ', cor: ' +
              str(callback_on_read) + ', coc: ' + str(callback_on_change))
Beispiel #8
0
    def read(self, path, size, offset):
        debug('UnderlyingFs.read', path, size, offset)
        real_path = self._get_real_path(path)

        with __builtin__.open(real_path, 'rb') as f:
            f.seek(offset)
            result = f.read(size)

        return result
Beispiel #9
0
    def main(self, args=None):
        options = self.cmdline[0]

        try:
            if options.cache_dir == None:
                raise ValueError('Need to specify --cache-dir')
            if options.target_dir == None:
                raise ValueError('Need to specify --target-dir')
        except Exception, e:
            debug('PersistentCacheFs Exception', e)
            sys.exit(1)
Beispiel #10
0
    def readdir(self, path, offset):
        dirents = []

        # Only add files if we're in the root directory
        if path == '/':
            for k in self.files.keys():
                # strip leading '/' from self.prefix before building filename
                dirents.append(self.prefix[1:] + k)

        # return a generator over the entries in the directory
        debug('vfs readdir', dirents)
        return (fuse.Direntry(r) for r in dirents)
Beispiel #11
0
    def readdir(self, path, offset):
        dirents = []

        # Only add files if we're in the root directory
        if path == '/':
            for k in self.files.keys():
                # strip leading '/' from self.prefix before building filename
                dirents.append(self.prefix[1:] + k)

        # return a generator over the entries in the directory
        debug('vfs readdir', dirents)
        return (fuse.Direntry(r) for r in dirents)
Beispiel #12
0
    def readdir(self, path, offset):
        debug('UnderlyingFs.readdir', path, offset)
        real_path = self._get_real_path(path)

        dirents = []
        if os.path.isdir(real_path):
            dirents.extend(['.', '..'])

        dirents.extend(os.listdir(real_path))

        # return a generator over the entries in the directory
        return (fuse.Direntry(r) for r in dirents)
Beispiel #13
0
    def open(self, path, flags):
        debug('VirtualFS.open', path, flags)
        virtual_path = self.get_relative_path(path)
        if virtual_path is None:
            return E_NO_SUCH_FILE

        if os.path.basename(virtual_path) in ['cached']:
            return 0

        if not is_read_only_flags(flags):
            return E_PERM_DENIED

        return 0
Beispiel #14
0
    def readdir(self, path, offset):
        debug('VirtualFS.readdir', path, offset)
        virtual_path = self.get_relative_path(path)
        if virtual_path is not None:
            is_file = stat.S_ISREG(self.cacher.getattr(os.sep + virtual_path).st_mode)
            if is_file:
                yield fuse.Direntry('cached')
            else:
                for f in self.cacher.readdir(os.sep + virtual_path, offset):
                    yield fuse.Direntry(f.name)
            yield None

        if path == '/':
            yield fuse.Direntry(self.root)
Beispiel #15
0
    def read(self, path, size, offset, force_reload=False):
        """Read the given data from the given path on the filesystem.

        Any parts which are requested and are not in the cache are read
        from the underlying filesystem
        """
        debug('Cacher.read', path, size, offset)

        self.init_cached_data(path)

        if force_reload:
            self.remove_cached_blocks(path)

        cached_blocks = self.get_cached_blocks(path)
        debug('Cacher.cached_blocks', cached_blocks)

        blocks_needed = cached_blocks.get_uncovered_portions(
            Range(offset, offset + size))
        debug('Cacher.blocks_needed', blocks_needed)

        blocks_to_read = Ranges()
        for block in blocks_needed:
            # make sure this block isn't already included in a previous block + read_ahead
            if not blocks_to_read.contains(block):
                blocks_to_read.add_range(
                    Range(block.start,
                          max(block.end, block.start + self.read_ahead)))

        debug('Cacher.blocks_to_read', blocks_to_read)
        self.update_cached_data(path, blocks_to_read.ranges)
        self.update_cached_blocks(
            path, cached_blocks.add_ranges(blocks_to_read.ranges))

        return self.get_cached_data(path, size, offset)
Beispiel #16
0
    def __init__(self, name, callback_on_read = None, callback_on_true = None, callback_on_false = None):
        debug('BVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', cot: ' + str(callback_on_true) + ', cof: ' + str(callback_on_false))
        # "2" so as not to override superclass method
        self.callback_on_read2 = callback_on_read

        self.callback_on_true = callback_on_true
        self.callback_on_false = callback_on_false

        coc = self._change
        if callback_on_true == None and callback_on_false == None:
            coc = None

        SimpleVirtualFile.__init__(self, name, self._read, callback_on_change=coc)

        self.value = False
Beispiel #17
0
    def open(self, path, flags):
        debug('VirtualFS.open', path, flags)
        virtual_path = self.get_relative_path(path)
        if virtual_path is None:
            return E_NO_SUCH_FILE

        if os.path.basename(virtual_path) in ['cached']:
            return 0
        else:
            # Only support for 'READ ONLY' flag
            access_flags = os.O_RDONLY | os.O_WRONLY | os.O_RDWR
            if flags & access_flags != os.O_RDONLY:
                return E_PERM_DENIED
            else:
                return 0
Beispiel #18
0
    def _change(self, value):
        debug('booleanVirtualFile: ' + value)
        if value == '0':
            self.value = False
        elif value.strip() == '':
            self.value = False
        else:
            self.value = True

        if self.value:
            if self.callback_on_true != None:
                self.callback_on_true()
        else:
            if self.callback_on_false != None:
                self.callback_on_false()
Beispiel #19
0
    def _change(self, value):
        debug('booleanVirtualFile: ' + value)
        if value == '0':
            self.value = False
        elif value.strip() == '':
            self.value = False
        else:
            self.value = True

        if self.value:
            if self.callback_on_true != None:
                self.callback_on_true()
        else:
            if self.callback_on_false != None:
                self.callback_on_false()
Beispiel #20
0
    def getattr(self, path):
        """Retrieve attributes of a path in the VirtualFS."""
        debug('VirtualFS.getattr', path)
        virtual_path = self.get_relative_path(path)
        if virtual_path is None:
            return E_NO_SUCH_FILE

        parent_path = os.sep + os.path.dirname(virtual_path)
        parent_is_file = stat.S_ISREG(self.cacher.getattr(parent_path).st_mode)
        if parent_is_file:
            if os.path.basename(virtual_path) not in ['cached']:
                return E_NO_SUCH_FILE
            return self.cacher.getattr(parent_path)
        else:
            a = self.cacher.getattr(os.sep + virtual_path)
            a.st_mode = stat.S_IFDIR | 0o777
            return a
Beispiel #21
0
    def read(self, path, size, offset):
        debug('VirtualFS.read', path, size, offset)
        virtual_path = self.get_relative_path(path)
        if virtual_path is None:
            return E_NO_SUCH_FILE

        parent_path = os.sep + os.path.dirname(virtual_path)
        parent_is_file = stat.S_ISREG(self.cacher.getattr(parent_path).st_mode)
        if not parent_is_file:
            return E_NO_SUCH_FILE

        basename = os.path.basename(virtual_path)
        if basename != 'cached':
            return E_NO_SUCH_FILE

        attr = self.cacher.getattr(parent_path)
        return str(self.cacher.get_cached_blocks(parent_path).number() / float(attr.st_size * attr.st_blksize))
Beispiel #22
0
    def getattr(self, path):
        cache_dir = self._get_cache_dir(path, 'cache.stat')

        result = None
        if os.path.exists(cache_dir):
            with __builtin__.open(cache_dir, 'rb') as stat_cache_file:
                result = pickle.load(stat_cache_file)
                debug('cacher.getattr', 'fetching from cache', path)

        else:
            result = self.underlying_fs.getattr(path)
            debug('cacher.getattr getting from filesystem', path)

            self._create_cache_dir(path)
            with __builtin__.open(cache_dir, 'wb') as stat_cache_file:
                pickle.dump(result, stat_cache_file)

        return result
Beispiel #23
0
    def get_cached_blocks(self, path):
        cached_blocks = self.cached_blocks_cache.get(path)
        if not cached_blocks is None:
            debug("cached_blocks_cache hit for", path, cached_blocks)
            return cached_blocks

        debug("cached_blocks_cache miss for", path)
        data_cache_range = self._get_cache_dir(path, 'cache.data.range')

        cached_blocks = None
        if os.path.exists(data_cache_range):
            with __builtin__.open(data_cache_range, 'rb') as f:
                cached_blocks = pickle.load(f)
        else:
            cached_blocks = Ranges()

        self.cached_blocks_cache[path] = cached_blocks
        return cached_blocks
Beispiel #24
0
    def getattr(self, path):
        """Retrieve stat information for a particular file from the cache."""
        debug('Cacher.getattr', path)
        cache_dir = self._get_cache_dir(path, 'cache.stat')

        result = None
        if os.path.exists(cache_dir):
            with __builtin__.open(cache_dir, 'rb') as stat_cache_file:
                result = pickle.load(stat_cache_file)

        else:
            result = self.underlying_fs.getattr(path)

            self._create_cache_dir(path)
            with __builtin__.open(cache_dir, 'wb') as stat_cache_file:
                pickle.dump(result, stat_cache_file)

        return result
Beispiel #25
0
    def getattr(self, path):
        debug('vfs getattr', path)
        virtual_file = self.get_file(path)
        debug('vfs getattr', virtual_file)

        if virtual_file == None:
            return E_NO_SUCH_FILE

        result = fuse.Stat()

        if virtual_file.is_read_only():
            result.st_mode = stat.S_IFREG | 0444
        else:
            result.st_mode = stat.S_IFREG | 0644

        # Always 1 for now (seems to be safe for files and dirs)
        result.st_nlink = 1

        result.st_size = virtual_file.size()

        # Must return seconds-since-epoch timestamps
        result.st_atime = virtual_file.atime()
        result.st_mtime = virtual_file.mtime()
        result.st_ctime = virtual_file.ctime()

        # You can set these to anything, they're set by FUSE
        result.st_dev = 1
        result.st_ino = 1

        # GetContext() returns uid/gid of the process that
        # initiated the syscall currently being handled
        context = fuse.FuseGetContext()
        if virtual_file.uid() == None:
            result.st_uid = context['uid']
        else:
            result.st_uid = virtual_file.uid()

        if virtual_file.gid() == None:
            result.st_gid = context['gid']
        else:
            result.st_gid = virtual_file.gid()

        return result
Beispiel #26
0
    def getattr(self, path):
        debug('vfs getattr', path)
        virtual_file = self.get_file(path)
        debug('vfs getattr', virtual_file)

        if virtual_file == None:
            return E_NO_SUCH_FILE

        result = fuse.Stat()

        if virtual_file.is_read_only():
            result.st_mode = stat.S_IFREG | 0444
        else:
            result.st_mode = stat.S_IFREG | 0644

        # Always 1 for now (seems to be safe for files and dirs)
        result.st_nlink = 1

        result.st_size = virtual_file.size()

        # Must return seconds-since-epoch timestamps
        result.st_atime = virtual_file.atime()
        result.st_mtime = virtual_file.mtime()
        result.st_ctime = virtual_file.ctime()

        # You can set these to anything, they're set by FUSE
        result.st_dev = 1
        result.st_ino = 1

        # GetContext() returns uid/gid of the process that
        # initiated the syscall currently being handled
        context = fuse.FuseGetContext()
        if virtual_file.uid() == None:
            result.st_uid = context['uid']
        else:
            result.st_uid = virtual_file.uid()

        if virtual_file.gid() == None:
            result.st_gid = context['gid']
        else:
            result.st_gid = virtual_file.gid()

        return result
Beispiel #27
0
    def readdir(self, path, offset):
        """List the given directory, from the cache."""
        debug('Cacher.readdir', path, offset)
        cache_dir = self._get_cache_dir(path, 'cache.list')

        result = None
        if os.path.exists(cache_dir):
            with __builtin__.open(cache_dir, 'rb') as list_cache_file:
                result = pickle.load(list_cache_file)

        else:
            result_generator = self.underlying_fs.readdir(path, offset)
            result = list(result_generator)

            self._create_cache_dir(path)
            with __builtin__.open(cache_dir, 'wb') as list_cache_file:
                pickle.dump(result, list_cache_file)

        # Return a new generator over our list of items
        return (x for x in result)
Beispiel #28
0
    def write(self, path, buf, offset):
        debug('VirtualFS.write', path, buf, offset)
        virtual_path = self.get_relative_path(path)
        if virtual_path is None:
            return E_NO_SUCH_FILE

        basename = os.path.basename(virtual_path)
        if basename == 'cached':
            real_path = os.sep + os.path.dirname(virtual_path)
            if buf == '1':
                attr = self.cacher.underlying_fs.getattr(real_path)
                size = attr.st_size * attr.st_blksize
                self.cacher.read(real_path, size, 0, force_reload=True)
            elif buf == '0':
                self.cacher.remove_cached_data(real_path)
            else:
                return E_NOT_IMPL
            return len(buf)
        else:
            return E_NO_SUCH_FILE
Beispiel #29
0
    def readdir(self, path, offset):
        cache_dir = self._get_cache_dir(path, 'cache.list')

        result = None
        if os.path.exists(cache_dir):
            debug('cacher.readdir getting from cache', path)
            with __builtin__.open(cache_dir, 'rb') as list_cache_file:
                result = pickle.load(list_cache_file)

        else:
            debug('cacher.readdir asking ufs for listing', path)
            result_generator = self.underlying_fs.readdir(path, offset)
            result = list(result_generator)

            self._create_cache_dir(path)
            with __builtin__.open(cache_dir, 'wb') as list_cache_file:
                pickle.dump(result, list_cache_file)

        # Return a new generator over our list of items
        return (x for x in result)
Beispiel #30
0
    def read(self, path, size, offset, force_reload=False):
        """Read the given data from the given path on the filesystem.

        Any parts which are requested and are not in the cache are read
        from the underlying filesystem
        """
        debug('Cacher.read', path, size, offset)

        self.init_cached_data(path)

        if force_reload:
            self.remove_cached_blocks(path)

        cached_blocks = self.get_cached_blocks(path)
        blocks_to_read = cached_blocks.get_uncovered_portions(
            Range(offset, offset + size))

        self.update_cached_data(path, blocks_to_read)
        self.update_cached_blocks(path,
                                  cached_blocks.add_ranges(blocks_to_read))

        return self.get_cached_data(path, size, offset)
Beispiel #31
0
    def __init__(self,
                 name,
                 callback_on_read=None,
                 callback_on_true=None,
                 callback_on_false=None):
        debug('BVF init, name: ' + str(name) + ', cor: ' +
              str(callback_on_read) + ', cot: ' + str(callback_on_true) +
              ', cof: ' + str(callback_on_false))
        # "2" so as not to override superclass method
        self.callback_on_read2 = callback_on_read

        self.callback_on_true = callback_on_true
        self.callback_on_false = callback_on_false

        coc = self._change
        if callback_on_true == None and callback_on_false == None:
            coc = None

        SimpleVirtualFile.__init__(self,
                                   name,
                                   self._read,
                                   callback_on_change=coc)

        self.value = False
Beispiel #32
0
    def __init__(self, cachedir, underlying_fs):
        self.cachedir = cachedir
        self.underlying_fs = underlying_fs

        # If this is set to True, the cacher will fail if any
        # requests are made for data that does not exist in the cache
        self.cache_only_mode = False

        debug('cdir: ' + self.cachedir)
        debug('os: ' + str(type(os)))
        debug('pathexists: ' + str(os.path.exists(self.cachedir)))

        if not os.path.exists(self.cachedir):
            self._mkdir(self.cachedir)
Beispiel #33
0
 def _get_filename(self, path):
     # self.prefix contains full prefix, including root path element '/'
     debug('get_filename', self.prefix, path)
     if path.startswith(self.prefix):
         return path[len(self.prefix):]
     return None
Beispiel #34
0
 def getattr(self, path):
     debug('UFS GETATTR C_F_S')
     return factory.create(FuseStat, os.stat(self._get_real_path(path)))
Beispiel #35
0
    def read(self, path, size, offset):
        f = self.get_file(path)

        debug('vfs read', path, str(size), str(offset))
        return f.read(size, offset)
Beispiel #36
0
 def cache_only_mode_disable(self):
     debug('Cacher.cache_only_mode_disable')
     self.cache_only_mode = False
Beispiel #37
0
 def truncate(self, path, size):  # pylint: disable=no-self-use
     debug('VirtualFS.truncate', path, size)
     return 0
Beispiel #38
0
 def _get_filename(self, path):
     # self.prefix contains full prefix, including root path element '/'
     debug('get_filename', self.prefix, path)
     if path.startswith(self.prefix):
         return path[len(self.prefix):]
     return None
Beispiel #39
0
    def read(self, path, size, offset):
        debug('cacher.read', path, str(size), str(offset))
        cache_data = self._get_cache_dir(path, 'cache.data')
        data_cache_range = self._get_cache_dir(path, 'cache.data.range')

        # list of Range objects indicating which chunks of the requested data
        # we have not yet cached and will need to get from the underlying fs
        blocks_to_read = []

        # Ranges object indicating which chunks of the file we have cached
        cached_blocks = None
        if os.path.exists(data_cache_range):
            with __builtin__.open(data_cache_range, 'rb') as f:
                debug('  loading cached_blocks from file')
                cached_blocks = pickle.load(f)
        else:
            cached_blocks = Ranges()

        requested_range = Range(offset, offset+size)

        debug('   read', 'path=' + path, 'size=' + str(size), 'offset=' + str(offset))
        debug('   requested_range', requested_range)
        debug('   cached_blocks', cached_blocks)

        blocks_to_read = cached_blocks.get_uncovered_portions(requested_range)

        debug('   blocks_to_read', blocks_to_read)

        # First, create the cache file if it does not exist already
        if not os.path.exists(cache_data):
            # We create a file full of zeroes the same size as the real file
            file_stat = self.getattr(path)
            self._create_cache_dir(path)

            with __builtin__.open(cache_data, 'wb') as f:
                debug('  creating blank file, size', str(file_stat.st_size))
                f.seek(file_stat.st_size - 1)
                f.write('\0')

                #for i in range(1, file_stat.st_size):
                #    f.write('\0')

        # If there are no blocks_to_read, then don't bother opening
        # the cache_data file for updates or dumping our cached_blocks.
        # This will slightly improve performance when getting data which
        # is already in the cache.
        if len(blocks_to_read) > 0:

            # Now open it up in update mode so we can add data to it as
            # we read the data from the underlying filesystem
            with __builtin__.open(cache_data, 'r+b') as cache_data_file:

                # Now loop through all the blocks we need to get
                # and append them to the cached file as we go
                for block in blocks_to_read:
                    block_data = self.underlying_fs.read(path, block.size, block.start)

                    cached_blocks.add_range(block)

                    cache_data_file.seek(block.start)
                    cache_data_file.write(block_data) # overwrites existing data in the file

            # update our cached_blocks file
            with __builtin__.open(data_cache_range, 'wb') as f:
                pickle.dump(cached_blocks, f)

        # Now we have loaded all the data we need to into the cache, we do the read
        # from the cached file
        result = None
        with __builtin__.open(cache_data, 'rb') as f:
            f.seek(offset)
            result = f.read(size)

        debug('  returning result from cache', type(result), len(result))
        return result
Beispiel #40
0
def create(t, *args, **kwargs):
    debug("create", str(t), str(args), str(kwargs))
    return t(*args, **kwargs)
Beispiel #41
0
 def _mkdir(self, path):
     debug('mkdir "' + path + '", os: ' + str(type(os)))
     if not os.path.exists(path):
         os.makedirs(path)
Beispiel #42
0
 def mknod(self, path, mode, dev):  # pylint: disable=no-self-use
     debug('VirtualFS.mknod', path, mode, dev)
     # Don't allow creation of new files
     return E_PERM_DENIED
Beispiel #43
0
 def release(self, path, fh=None):  # pylint: disable=no-self-use, unused-argument
     debug('VirtualFS.release', path)
     return 0
Beispiel #44
0
 def flush(self, path, fh=None):  # pylint: disable=no-self-use, unused-argument
     debug('VirtualFS.flush', path)
     return 0
Beispiel #45
0
    def release(self, path, what):
        debug('release ' + str(path) + ', ' + str(what))
        if self.vfs.contains(path):
            return self.vfs.release(path)

        return 0 # success
Beispiel #46
0
 def cache_only_mode_disable(self):
     debug('cacher cache_only_mode disabled')
     self.cache_only_mode = False
Beispiel #47
0
 def write(self, path, buf, offset):  # pylint: disable=no-self-use
     debug('Cacher.write', path, buf, offset)
     return E_NOT_IMPL
Beispiel #48
0
 def cache_only_mode_enable(self):
     debug('cacher cache_only_mode enabled')
     self.cache_only_mode = True
Beispiel #49
0
    def read(self, path, size, offset):
        f = self.get_file(path)

        debug('vfs read', path, str(size), str(offset))
        return f.read(size, offset)
Beispiel #50
0
 def unlink(self, path):  # pylint: disable=no-self-use
     debug('VirtualFS.unlink', path)
     # Don't allow removal of files
     return E_PERM_DENIED