def get_file(self, path): name = self._get_filename(path) debug('get_file', path, name) if name != None and name in self.files: return self.files[name] return None
def __init__(self, name, callback_on_read, callback_on_change = None): VirtualFile.__init__(self, name) self.callback_on_read = callback_on_read self.callback_on_change = callback_on_change self.content = None debug('SVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', coc: ' + str(callback_on_change))
def open(self, path, flags): debug('PersistentCacheFs.open', path, flags) if self.vfs.contains(path): return self.vfs.open(path, flags) if not is_read_only_flags(flags): return E_PERM_DENIED return 0
def readdir(self, path, offset): debug('PersistentCacheFs.readdir', path, offset) for f in self.vfs.readdir(path, offset): if f is None: return yield f for f in self.cacher.readdir(path, offset): yield f
def read(self, path, size, offset): debug('ufs.read', path, str(size), str(offset)) real_path = self._get_real_path(path) with __builtin__.open(real_path, 'rb') as f: f.seek(offset) result = f.read(size) return result
def __init__(self, name, callback_on_read, callback_on_change=None): VirtualFile.__init__(self, name) self.callback_on_read = callback_on_read self.callback_on_change = callback_on_change self.content = None debug('SVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', coc: ' + str(callback_on_change))
def read(self, path, size, offset): debug('UnderlyingFs.read', path, size, offset) real_path = self._get_real_path(path) with __builtin__.open(real_path, 'rb') as f: f.seek(offset) result = f.read(size) return result
def main(self, args=None): options = self.cmdline[0] try: if options.cache_dir == None: raise ValueError('Need to specify --cache-dir') if options.target_dir == None: raise ValueError('Need to specify --target-dir') except Exception, e: debug('PersistentCacheFs Exception', e) sys.exit(1)
def readdir(self, path, offset): dirents = [] # Only add files if we're in the root directory if path == '/': for k in self.files.keys(): # strip leading '/' from self.prefix before building filename dirents.append(self.prefix[1:] + k) # return a generator over the entries in the directory debug('vfs readdir', dirents) return (fuse.Direntry(r) for r in dirents)
def readdir(self, path, offset): debug('UnderlyingFs.readdir', path, offset) real_path = self._get_real_path(path) dirents = [] if os.path.isdir(real_path): dirents.extend(['.', '..']) dirents.extend(os.listdir(real_path)) # return a generator over the entries in the directory return (fuse.Direntry(r) for r in dirents)
def open(self, path, flags): debug('VirtualFS.open', path, flags) virtual_path = self.get_relative_path(path) if virtual_path is None: return E_NO_SUCH_FILE if os.path.basename(virtual_path) in ['cached']: return 0 if not is_read_only_flags(flags): return E_PERM_DENIED return 0
def readdir(self, path, offset): debug('VirtualFS.readdir', path, offset) virtual_path = self.get_relative_path(path) if virtual_path is not None: is_file = stat.S_ISREG(self.cacher.getattr(os.sep + virtual_path).st_mode) if is_file: yield fuse.Direntry('cached') else: for f in self.cacher.readdir(os.sep + virtual_path, offset): yield fuse.Direntry(f.name) yield None if path == '/': yield fuse.Direntry(self.root)
def read(self, path, size, offset, force_reload=False): """Read the given data from the given path on the filesystem. Any parts which are requested and are not in the cache are read from the underlying filesystem """ debug('Cacher.read', path, size, offset) self.init_cached_data(path) if force_reload: self.remove_cached_blocks(path) cached_blocks = self.get_cached_blocks(path) debug('Cacher.cached_blocks', cached_blocks) blocks_needed = cached_blocks.get_uncovered_portions( Range(offset, offset + size)) debug('Cacher.blocks_needed', blocks_needed) blocks_to_read = Ranges() for block in blocks_needed: # make sure this block isn't already included in a previous block + read_ahead if not blocks_to_read.contains(block): blocks_to_read.add_range( Range(block.start, max(block.end, block.start + self.read_ahead))) debug('Cacher.blocks_to_read', blocks_to_read) self.update_cached_data(path, blocks_to_read.ranges) self.update_cached_blocks( path, cached_blocks.add_ranges(blocks_to_read.ranges)) return self.get_cached_data(path, size, offset)
def __init__(self, name, callback_on_read = None, callback_on_true = None, callback_on_false = None): debug('BVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', cot: ' + str(callback_on_true) + ', cof: ' + str(callback_on_false)) # "2" so as not to override superclass method self.callback_on_read2 = callback_on_read self.callback_on_true = callback_on_true self.callback_on_false = callback_on_false coc = self._change if callback_on_true == None and callback_on_false == None: coc = None SimpleVirtualFile.__init__(self, name, self._read, callback_on_change=coc) self.value = False
def open(self, path, flags): debug('VirtualFS.open', path, flags) virtual_path = self.get_relative_path(path) if virtual_path is None: return E_NO_SUCH_FILE if os.path.basename(virtual_path) in ['cached']: return 0 else: # Only support for 'READ ONLY' flag access_flags = os.O_RDONLY | os.O_WRONLY | os.O_RDWR if flags & access_flags != os.O_RDONLY: return E_PERM_DENIED else: return 0
def _change(self, value): debug('booleanVirtualFile: ' + value) if value == '0': self.value = False elif value.strip() == '': self.value = False else: self.value = True if self.value: if self.callback_on_true != None: self.callback_on_true() else: if self.callback_on_false != None: self.callback_on_false()
def getattr(self, path): """Retrieve attributes of a path in the VirtualFS.""" debug('VirtualFS.getattr', path) virtual_path = self.get_relative_path(path) if virtual_path is None: return E_NO_SUCH_FILE parent_path = os.sep + os.path.dirname(virtual_path) parent_is_file = stat.S_ISREG(self.cacher.getattr(parent_path).st_mode) if parent_is_file: if os.path.basename(virtual_path) not in ['cached']: return E_NO_SUCH_FILE return self.cacher.getattr(parent_path) else: a = self.cacher.getattr(os.sep + virtual_path) a.st_mode = stat.S_IFDIR | 0o777 return a
def read(self, path, size, offset): debug('VirtualFS.read', path, size, offset) virtual_path = self.get_relative_path(path) if virtual_path is None: return E_NO_SUCH_FILE parent_path = os.sep + os.path.dirname(virtual_path) parent_is_file = stat.S_ISREG(self.cacher.getattr(parent_path).st_mode) if not parent_is_file: return E_NO_SUCH_FILE basename = os.path.basename(virtual_path) if basename != 'cached': return E_NO_SUCH_FILE attr = self.cacher.getattr(parent_path) return str(self.cacher.get_cached_blocks(parent_path).number() / float(attr.st_size * attr.st_blksize))
def getattr(self, path): cache_dir = self._get_cache_dir(path, 'cache.stat') result = None if os.path.exists(cache_dir): with __builtin__.open(cache_dir, 'rb') as stat_cache_file: result = pickle.load(stat_cache_file) debug('cacher.getattr', 'fetching from cache', path) else: result = self.underlying_fs.getattr(path) debug('cacher.getattr getting from filesystem', path) self._create_cache_dir(path) with __builtin__.open(cache_dir, 'wb') as stat_cache_file: pickle.dump(result, stat_cache_file) return result
def get_cached_blocks(self, path): cached_blocks = self.cached_blocks_cache.get(path) if not cached_blocks is None: debug("cached_blocks_cache hit for", path, cached_blocks) return cached_blocks debug("cached_blocks_cache miss for", path) data_cache_range = self._get_cache_dir(path, 'cache.data.range') cached_blocks = None if os.path.exists(data_cache_range): with __builtin__.open(data_cache_range, 'rb') as f: cached_blocks = pickle.load(f) else: cached_blocks = Ranges() self.cached_blocks_cache[path] = cached_blocks return cached_blocks
def getattr(self, path): """Retrieve stat information for a particular file from the cache.""" debug('Cacher.getattr', path) cache_dir = self._get_cache_dir(path, 'cache.stat') result = None if os.path.exists(cache_dir): with __builtin__.open(cache_dir, 'rb') as stat_cache_file: result = pickle.load(stat_cache_file) else: result = self.underlying_fs.getattr(path) self._create_cache_dir(path) with __builtin__.open(cache_dir, 'wb') as stat_cache_file: pickle.dump(result, stat_cache_file) return result
def getattr(self, path): debug('vfs getattr', path) virtual_file = self.get_file(path) debug('vfs getattr', virtual_file) if virtual_file == None: return E_NO_SUCH_FILE result = fuse.Stat() if virtual_file.is_read_only(): result.st_mode = stat.S_IFREG | 0444 else: result.st_mode = stat.S_IFREG | 0644 # Always 1 for now (seems to be safe for files and dirs) result.st_nlink = 1 result.st_size = virtual_file.size() # Must return seconds-since-epoch timestamps result.st_atime = virtual_file.atime() result.st_mtime = virtual_file.mtime() result.st_ctime = virtual_file.ctime() # You can set these to anything, they're set by FUSE result.st_dev = 1 result.st_ino = 1 # GetContext() returns uid/gid of the process that # initiated the syscall currently being handled context = fuse.FuseGetContext() if virtual_file.uid() == None: result.st_uid = context['uid'] else: result.st_uid = virtual_file.uid() if virtual_file.gid() == None: result.st_gid = context['gid'] else: result.st_gid = virtual_file.gid() return result
def readdir(self, path, offset): """List the given directory, from the cache.""" debug('Cacher.readdir', path, offset) cache_dir = self._get_cache_dir(path, 'cache.list') result = None if os.path.exists(cache_dir): with __builtin__.open(cache_dir, 'rb') as list_cache_file: result = pickle.load(list_cache_file) else: result_generator = self.underlying_fs.readdir(path, offset) result = list(result_generator) self._create_cache_dir(path) with __builtin__.open(cache_dir, 'wb') as list_cache_file: pickle.dump(result, list_cache_file) # Return a new generator over our list of items return (x for x in result)
def write(self, path, buf, offset): debug('VirtualFS.write', path, buf, offset) virtual_path = self.get_relative_path(path) if virtual_path is None: return E_NO_SUCH_FILE basename = os.path.basename(virtual_path) if basename == 'cached': real_path = os.sep + os.path.dirname(virtual_path) if buf == '1': attr = self.cacher.underlying_fs.getattr(real_path) size = attr.st_size * attr.st_blksize self.cacher.read(real_path, size, 0, force_reload=True) elif buf == '0': self.cacher.remove_cached_data(real_path) else: return E_NOT_IMPL return len(buf) else: return E_NO_SUCH_FILE
def readdir(self, path, offset): cache_dir = self._get_cache_dir(path, 'cache.list') result = None if os.path.exists(cache_dir): debug('cacher.readdir getting from cache', path) with __builtin__.open(cache_dir, 'rb') as list_cache_file: result = pickle.load(list_cache_file) else: debug('cacher.readdir asking ufs for listing', path) result_generator = self.underlying_fs.readdir(path, offset) result = list(result_generator) self._create_cache_dir(path) with __builtin__.open(cache_dir, 'wb') as list_cache_file: pickle.dump(result, list_cache_file) # Return a new generator over our list of items return (x for x in result)
def read(self, path, size, offset, force_reload=False): """Read the given data from the given path on the filesystem. Any parts which are requested and are not in the cache are read from the underlying filesystem """ debug('Cacher.read', path, size, offset) self.init_cached_data(path) if force_reload: self.remove_cached_blocks(path) cached_blocks = self.get_cached_blocks(path) blocks_to_read = cached_blocks.get_uncovered_portions( Range(offset, offset + size)) self.update_cached_data(path, blocks_to_read) self.update_cached_blocks(path, cached_blocks.add_ranges(blocks_to_read)) return self.get_cached_data(path, size, offset)
def __init__(self, name, callback_on_read=None, callback_on_true=None, callback_on_false=None): debug('BVF init, name: ' + str(name) + ', cor: ' + str(callback_on_read) + ', cot: ' + str(callback_on_true) + ', cof: ' + str(callback_on_false)) # "2" so as not to override superclass method self.callback_on_read2 = callback_on_read self.callback_on_true = callback_on_true self.callback_on_false = callback_on_false coc = self._change if callback_on_true == None and callback_on_false == None: coc = None SimpleVirtualFile.__init__(self, name, self._read, callback_on_change=coc) self.value = False
def __init__(self, cachedir, underlying_fs): self.cachedir = cachedir self.underlying_fs = underlying_fs # If this is set to True, the cacher will fail if any # requests are made for data that does not exist in the cache self.cache_only_mode = False debug('cdir: ' + self.cachedir) debug('os: ' + str(type(os))) debug('pathexists: ' + str(os.path.exists(self.cachedir))) if not os.path.exists(self.cachedir): self._mkdir(self.cachedir)
def _get_filename(self, path): # self.prefix contains full prefix, including root path element '/' debug('get_filename', self.prefix, path) if path.startswith(self.prefix): return path[len(self.prefix):] return None
def getattr(self, path): debug('UFS GETATTR C_F_S') return factory.create(FuseStat, os.stat(self._get_real_path(path)))
def read(self, path, size, offset): f = self.get_file(path) debug('vfs read', path, str(size), str(offset)) return f.read(size, offset)
def cache_only_mode_disable(self): debug('Cacher.cache_only_mode_disable') self.cache_only_mode = False
def truncate(self, path, size): # pylint: disable=no-self-use debug('VirtualFS.truncate', path, size) return 0
def read(self, path, size, offset): debug('cacher.read', path, str(size), str(offset)) cache_data = self._get_cache_dir(path, 'cache.data') data_cache_range = self._get_cache_dir(path, 'cache.data.range') # list of Range objects indicating which chunks of the requested data # we have not yet cached and will need to get from the underlying fs blocks_to_read = [] # Ranges object indicating which chunks of the file we have cached cached_blocks = None if os.path.exists(data_cache_range): with __builtin__.open(data_cache_range, 'rb') as f: debug(' loading cached_blocks from file') cached_blocks = pickle.load(f) else: cached_blocks = Ranges() requested_range = Range(offset, offset+size) debug(' read', 'path=' + path, 'size=' + str(size), 'offset=' + str(offset)) debug(' requested_range', requested_range) debug(' cached_blocks', cached_blocks) blocks_to_read = cached_blocks.get_uncovered_portions(requested_range) debug(' blocks_to_read', blocks_to_read) # First, create the cache file if it does not exist already if not os.path.exists(cache_data): # We create a file full of zeroes the same size as the real file file_stat = self.getattr(path) self._create_cache_dir(path) with __builtin__.open(cache_data, 'wb') as f: debug(' creating blank file, size', str(file_stat.st_size)) f.seek(file_stat.st_size - 1) f.write('\0') #for i in range(1, file_stat.st_size): # f.write('\0') # If there are no blocks_to_read, then don't bother opening # the cache_data file for updates or dumping our cached_blocks. # This will slightly improve performance when getting data which # is already in the cache. if len(blocks_to_read) > 0: # Now open it up in update mode so we can add data to it as # we read the data from the underlying filesystem with __builtin__.open(cache_data, 'r+b') as cache_data_file: # Now loop through all the blocks we need to get # and append them to the cached file as we go for block in blocks_to_read: block_data = self.underlying_fs.read(path, block.size, block.start) cached_blocks.add_range(block) cache_data_file.seek(block.start) cache_data_file.write(block_data) # overwrites existing data in the file # update our cached_blocks file with __builtin__.open(data_cache_range, 'wb') as f: pickle.dump(cached_blocks, f) # Now we have loaded all the data we need to into the cache, we do the read # from the cached file result = None with __builtin__.open(cache_data, 'rb') as f: f.seek(offset) result = f.read(size) debug(' returning result from cache', type(result), len(result)) return result
def create(t, *args, **kwargs): debug("create", str(t), str(args), str(kwargs)) return t(*args, **kwargs)
def _mkdir(self, path): debug('mkdir "' + path + '", os: ' + str(type(os))) if not os.path.exists(path): os.makedirs(path)
def mknod(self, path, mode, dev): # pylint: disable=no-self-use debug('VirtualFS.mknod', path, mode, dev) # Don't allow creation of new files return E_PERM_DENIED
def release(self, path, fh=None): # pylint: disable=no-self-use, unused-argument debug('VirtualFS.release', path) return 0
def flush(self, path, fh=None): # pylint: disable=no-self-use, unused-argument debug('VirtualFS.flush', path) return 0
def release(self, path, what): debug('release ' + str(path) + ', ' + str(what)) if self.vfs.contains(path): return self.vfs.release(path) return 0 # success
def cache_only_mode_disable(self): debug('cacher cache_only_mode disabled') self.cache_only_mode = False
def write(self, path, buf, offset): # pylint: disable=no-self-use debug('Cacher.write', path, buf, offset) return E_NOT_IMPL
def cache_only_mode_enable(self): debug('cacher cache_only_mode enabled') self.cache_only_mode = True
def unlink(self, path): # pylint: disable=no-self-use debug('VirtualFS.unlink', path) # Don't allow removal of files return E_PERM_DENIED