def copyRecentFilesIntoCache(self, transfer_q: MaxPrioQueue):
print(Col.b('Transfering files...'))
while not transfer_q.empty() and not self.disk.isFull(
use_threshold=True):
timestamp, (inode, file_size) = transfer_q.pop_nowait()
path = self.vfs._inode_path_map[inode].src
# skip symbolic links for now
if os.path.islink(path):
continue
try:
dest = self.disk.cp2Cache(path)
except NotEnoughSpaceError:
# filter the Queue
purged_list = MaxPrioQueue()
while not transfer_q.empty():
timestamp_i, (inode_i, size_i) = transfer_q.pop_nowait()
if size_i < file_size:
purged_list.push_nowait(
(timestamp_i, (inode_i, size_i)))
transfer_q = purged_list
# print summary
diskUsage, usedCache, maxCache = self.disk.getCurrentStatus()
diskUsage = Col.by(f'{diskUsage:.8f}%')
usedCache = Col.by(f'{Col.BY}{formatByteSize(usedCache)} ')
maxCache = Col.by(f'{formatByteSize(maxCache)} ')
copySummary = \
Col.bw(f'Finished transfering {self.disk.in_cache.qsize()} elements.\nCache is now {diskUsage} ') + Col.bw(
'full') + \
Col.bw(f" (used: {usedCache}") + Col.bw(f" / {maxCache}") + Col.bw(")")
print(copySummary)
async def open(self, inode: int, flags: int, ctx):
# return early for already opened fd
print(f'open({inode}, {flags}, {ctx})')
if self.vfs.already_open(inode):
fd = self.vfs._inode_fd_map[inode]
self.vfs._fd_open_count[fd] += 1
print(Col.by(f'open: (fd, inode): ({fd}, {inode})'))
return pyfuse3.FileInfo(fh=fd)
# disable creation handling here
assert flags & os.O_CREAT == 0
try:
f: Path = self._inode_to_path(inode)
info: FileInfo = self.vfs._inode_path_map[inode]
# fetch file from remote if not in cache already
if not f.exists():
f_src = self.disk.toSrcPath(f)
# self.remote.makeAvailable()
self.__fetchFile(f_src, info.entry.st_size)
# File is in Cache now
fd: int = os.open(f, flags)
info.updateEntry(fd=fd)
except OSError as exc:
raise FUSEError(exc.errno)
self.vfs._inode_fd_map[inode] = fd
self.vfs._fd_inode_map[fd] = inode
self.vfs._fd_open_count[fd] = 1
return pyfuse3.FileInfo(fh=fd)
def _cp2Dir(src: Path, dst: Path):
"""
Create a copy of `src` in `dst` while also keeping meta-data.
Creates parent directories on the fly.
Ignores special files.
:arg src: original File
:arg dst: destination File to have the same attributes and contents of `src` after the call
:returns: Path of copied file in cacheDir
"""
if src == dst:
# same File no need for copying,
# file should exist already too as self.__canStore would throw errors otherwise
return src
src_mode = os.stat(src).st_mode
if src.is_dir():
Disk.cpdir(src, dst)
elif src.is_file():
if not dst.parent.exists():
Disk.cpdir(src.parent, dst.parent)
shutil.copy2(src, dst)
else:
msg = Col.by(f' Unrecognized filetype: {src} -> ignoring')
log.error(msg)
raise IOError(msg)
# book-keeping of file-attributes (also takes care of parent dirs having wrong modes from previous runs)
dst.chmod(src_mode)
shutil.copystat(src, dst)
return dst
def printSummary(self):
diskUsage, usedCache, maxCache = self.getCurrentStatus()
diskUsage = Col.by(f'{diskUsage:.8f}%')
usedCache = Col.by(f'{Col.BY}{formatByteSize(usedCache)} ')
maxCache = Col.by(f'{formatByteSize(maxCache)} ')
copySummary = \
Col.bw(
f'Cache is currently storing {self.in_cache.qsize()} elements and is {diskUsage} ') + Col.bw(
'full\n') + \
Col.bw(f" (used: {usedCache}") + Col.bw(f" / {maxCache}") + Col.bw(")")
print(copySummary)
async def statfs(self, ctx):
"""Easisest function to get a entrypoint into the code (not many df calls all around)"""
self.embed_active = not self.embed_active
root = self.vfs.inode_to_path(pyfuse3.ROOT_INODE)
stat_ = pyfuse3.StatvfsData()
try:
statfs = os.statvfs(root)
except OSError as exc:
raise FUSEError(exc.errno)
for attr in ('f_bsize', 'f_frsize', 'f_blocks', 'f_bfree', 'f_bavail',
'f_files', 'f_ffree', 'f_favail'):
setattr(stat_, attr, getattr(statfs, attr))
stat_.f_namemax = statfs.f_namemax - (len(root.__str__()) + 1)
print(Col.bg(f'RAM-Usage: of _inode_path_map: {self.vfs.getRamUsage()} | ' +
f'elements: {str(len(self.vfs._inode_path_map))}'))
self.disk.printSummary()
return stat_
async def lookup(self, inode_p, name, ctx=None):
name = fsdecode(name)
log.debug(Col.br(f'lookup for {name} in {inode_p}'))
return await self.__lookup(inode_p, name, ctx)
async def releasedir(self, fh):
# same as normal release() no more fh are using it
print(Col.by(f"releasing dir: {self.vfs.inode_to_path(fh)}"))
async def opendir(self, inode, ctx):
print(Col.by(f"opendir: {self.vfs.inode_to_path(inode)}"))
# ctx contains gid, uid, pid and umask
return inode