def getattr(self) -> pyfuse3.EntryAttributes:
entry = pyfuse3.EntryAttributes()
if self.paths:
try:
stat_ = os.lstat(self.path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
elif self.fds:
# file is unlinked already, but opened.
try:
stat_ = os.fstat(list(self.fds)[0])
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
else:
# when?
raise RuntimeError()
#raise pyfuse3.FUSEError(errno.ENOENT) # No such file or directory
# copy attrs from base FS.
for attr in ('st_mode', 'st_nlink', 'st_uid', 'st_gid', 'st_rdev',
'st_size', 'st_atime_ns', 'st_mtime_ns', 'st_ctime_ns'):
setattr(entry, attr, getattr(stat_, attr))
entry.st_ino = self.vnode
entry.generation = 0
entry.entry_timeout = 0
entry.attr_timeout = 0
entry.st_blksize = 512
entry.st_blocks = ((entry.st_size+entry.st_blksize-1) // entry.st_blksize)
return entry
async def create(self, vnode_parent, name, mode, flags, ctx):
path = self.vm.make_path(self.vm[vnode_parent].path, os.fsdecode(name))
if path in self.vm and self.vm[path].virtual:
raise pyfuse3.FUSEError(
errno.EACCES
) # Permission denied, since pseudo file should not be created.
if not self.auditor.ask_writable(path):
_opslog.info(
'Creating to PATH <{}> is not permitted.'.format(path))
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
if self.auditor.ask_discard(path):
try:
fd = FD(os.open('/dev/null', flags & ~os.O_CREAT))
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
self.vinfo_null.open_vnode(fd)
_acslog.info('CREATE-FAKE: {}'.format(path))
return pyfuse3.FileInfo(fh=fd), self._getattr(self.vinfo_null)
vinfo = self.vm.create_vinfo()
try:
fd = FD(os.open(path, flags | os.O_CREAT | os.O_TRUNC, mode))
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
vinfo.add_path(path)
vinfo.open_vnode(FD(fd))
_acslog.info('CREATE: {}'.format(path))
return pyfuse3.FileInfo(fh=fd), self._getattr(vinfo)
async def rename(self, parent_inode_old, name_old, parent_inode_new,
name_new, flags, ctx):
inode_moved = self.db.get_inode_from_parent_and_name(
parent_inode_old, name_old)
if not inode_moved:
raise pyfuse3.FUSEError(errno.EINVAL)
inode_deref = self.db.get_inode_from_parent_and_name(
parent_inode_new, name_new)
if inode_deref:
if flags & pyfuse3.RENAME_NOREPLACE:
raise pyfuse3.FUSEError(errno.EEXIST)
elif flags & pyfuse3.RENAME_EXCHANGE:
self.db.update_link(inode_deref['link_id'],
inode=inode_moved['id'])
self.db.update_link(inode_moved['link_id'],
inode=inode_deref['id'])
self.db.commit()
else:
if inode_deref['nchild']:
raise pyfuse3.FUSEError(errno.ENOTEMPTY)
self.db.update_link(inode_deref['link_id'],
inode=inode_moved['id'])
self.db.delete_link(inode_moved['link_id'])
# need to delete inode - read doco its confusing for now just
# cleanup orphaned inodes on umount
self.db.commit()
else:
self.db.update_link(inode_moved['link_id'],
parent_inode=parent_inode_new,
name=name_new)
self.db.commit()
async def readlink(self, inode, ctx):
row = self.db.get_inode_from_id(inode)
if not row:
raise pyfuse3.FUSEError(errno.EINVAL)
if not stat.S_ISLNK(row['mode']):
raise pyfuse3.FUSEError(errno.EINVAL)
return row['target']
async def rename(self, old_folder_inode, old_bname, new_folder_inode,
new_bname, flags, ctx):
self.logger.debug("rename: %s %s %s %s", old_folder_inode, old_bname,
new_folder_inode, new_bname)
if flags != 0:
raise pyfuse3.FUSEError(errno.EINVAL)
if old_folder_inode != pyfuse3.ROOT_INODE:
raise pyfuse3.FUSEError(errno.EINVAL)
if new_folder_inode != pyfuse3.ROOT_INODE:
raise pyfuse3.FUSEError(errno.EINVAL)
entry_old = await self.lookup(old_folder_inode, old_bname, ctx)
try:
entry_new = await self.lookup(new_folder_inode, new_bname, ctx)
except pyfuse3.FUSEError as exc:
if exc.errno != errno.ENOENT:
raise
target_exists = False
else:
target_exists = True
if target_exists:
await self.unlink(new_folder_inode, new_bname, ctx)
file_id = self.inode2id.get(entry_old.st_ino)
new_filename = new_bname.decode(self._filename_encoding, 'replace')
self.gridfsbucket.rename(file_id, new_filename)
async def open(self, inode, flags, ctx):
LOG.info('open with flags %s', flags2str(flags))
# We don't allow to append or open in RW mode
if (flags & os.O_RDWR or flags & os.O_APPEND):
raise pyfuse3.FUSEError(errno.EPERM)
# Get the underlying path
path = self.add_extension(self._inode_to_path(inode))
# If we create the file
if (flags & os.O_WRONLY):
# Sanity check: Since we must have one of O_RDWR/O_RDONLY/O_WRONLY
if flags & os.O_RDONLY:
raise pyfuse3.FUSEError(errno.EINVAL)
attrs = self._getattr(path, no_extension=True)
# We enforce truncation
fd = await self._create(path, attrs.st_mode,
flags | os.O_TRUNC | os.O_CLOEXEC)
return pyfuse3.FileInfo(fh=fd)
# we are reading a file
try:
dec = FileDecryptor(path, flags, self.keys)
fd = dec.fd
self._fd2cryptors[fd] = dec
LOG.debug('added fd %d to map', fd)
except OSError as exc:
LOG.error('OSError opening %s: %s', path, exc)
raise FUSEError(exc.errno)
except Exception as exc:
LOG.error('Error opening %s: %s', path, exc)
raise FUSEError(errno.EACCES)
return pyfuse3.FileInfo(fh=fd)
async def getattr(self, inode, ctx=None):
self.logger.debug("getattr: %s", inode)
if inode == pyfuse3.ROOT_INODE:
entry = pyfuse3.EntryAttributes()
entry.st_mode = (stat.S_IFDIR | 0o755)
entry.st_size = 0
entry.st_atime_ns = self.root_stamp
entry.st_ctime_ns = self.root_stamp
entry.st_mtime_ns = self.root_stamp
entry.st_gid = os.getgid()
entry.st_uid = os.getuid()
entry.st_ino = inode
else:
file_id = self.inode2id.get(inode)
if file_id is None:
raise pyfuse3.FUSEError(errno.ENOENT)
file_stats = self.gridfs.find_one({'_id': file_id})
if file_stats is None:
raise pyfuse3.FUSEError(errno.ENOENT)
entry = await self._getattr(file_stats, inode)
return entry
async def setxattr(self, inode, name, value, ctx):
if inode != pyfuse3.ROOT_INODE or name != b'command':
raise pyfuse3.FUSEError(errno.ENOTSUP)
if value == b'terminate':
pyfuse3.terminate()
else:
raise pyfuse3.FUSEError(errno.EINVAL)
async def mknod(self, parent_inode, name, mode, rdev, ctx):
_inode = BDFile.get_from_fs_id(parent_inode)
if not _inode:
path = '/'
full_path = path
else:
path = _inode.path
full_path = path + '/'
name_bytes = name
name = name.decode('utf-8')
tmp = is_tmp(name)
if Env.CLOUD_HOME not in path:
raise pyfuse3.FUSEError(errno.EACCES)
if not os.path.isdir(Env.PHYSICS_DIR + path):
os.makedirs(Env.PHYSICS_DIR + path)
file_path = Env.PHYSICS_DIR + full_path + name
if tmp:
with open(file_path, 'wb') as f:
f.write(b'')
inode = random.randint(1000000000000000000, 9999999999999999999)
ns = (datetime.now().timestamp() * 1e9)
_cache = self.fs.cache.get(path, None)
_file = BDFile(isdir=False,
server_ctime=ns,
server_mtime=ns,
local_ctime=ns,
local_mtime=ns,
fs_id=inode,
path=full_path + name,
filename=name,
filename_bytes=name_bytes,
size=0,
p_inode=parent_inode)
BDFile.set_inode_name_pool(parent_inode, name, _file)
if not _cache:
self.fs.cache[path] = {
'items': [_file],
'expire':
datetime.now().timestamp() + BaiDu.DIR_EXPIRE_THRESHOLD
}
else:
self.fs.cache[path]['items'].append(_file)
return await self.getattr(inode, ctx)
with open(file_path, 'wb') as f:
f.write(Env.EMPTY_FILE_FLAG)
upload_path = full_path + name
# 因为百度不允许创建空文件，这里使用占位符上传临时文件
fs_id = self.fs.upload(parent_inode, file_path, upload_path)
if fs_id:
UploadInfo.add(parent_inode, fs_id, file_path, upload_path)
# 上传成功后将本地临时文件删除，在写入时重新创建，避免文件写入内容包含占位符
# os.remove(file_path)
return await self.getattr(fs_id, ctx)
raise pyfuse3.FUSEError(errno.EAGAIN)
async def rmdir(self, parent_inode, name, ctx):
row = self.db.get_inode_from_parent_and_name(parent_inode, name)
if not stat.S_ISDIR(row['mode']):
raise pyfuse3.FUSEError(errno.ENOTDIR)
if row['nchild'] > 2:
raise pyfuse3.FUSEError(errno.ENOTEMPTY)
self.db.delete_link_dir(row['id'])
# need to delete inode - for now just cleanup orphaned inodes on umount
self.db.commit()
async def getxattr(self, vnode, name_enced, ctx):
name = os.fsdecode(name_enced)
vinfo = self.vm[vnode]
if vinfo.virtual:
raise pyfuse3.FUSEError(errno.ENODATA) # No data available
else:
try:
return os.getxattr(vinfo.path, name)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
def _check_mk_validity(self, path, name):
path = self.normpath(path)
log.debug('check validity for %s' % path)
if not self.is_exists(path):
raise pyfuse3.FUSEError(errno.ENOENT)
if not self.is_dir(path):
raise pyfuse3.FUSEError(errno.ENOTDIR)
if self.is_contains(path, name):
raise pyfuse3.FUSEError(errno.EEXIST)
async def getattr(self, vnode, ctx=None):
try:
vinfo = self.vm[vnode]
except KeyError:
# when?
raise pyfuse3.FUSEError(errno.ENOENT) # no such file or directory
_opslog.debug('getattr path: {}, fd: {}'.format(
vinfo.paths, vinfo.fds))
if self.path_mountpoint in vinfo.paths:
raise pyfuse3.FUSEError(errno.ENOENT)
return vinfo.getattr()
async def open(self, vnode, flags, ctx):
vinfo = self.vm[vnode]
_acslog.debug('OPEN: {}'.format(vinfo.path))
if vinfo.virtual:
fd = FD(os.open('/dev/null',
flags)) # reserve file descriptor number
vinfo.open_vnode(fd, '/dev/null', flags, discard=False)
return pyfuse3.FileInfo(fh=fd)
elif self.auditor.ask_discard(vinfo.path):
try:
# open with readonly mode
fd = FD(
os.open(
vinfo.path,
flags & ~(os.O_TRUNC | os.O_RDWR | os.O_WRONLY)
| os.O_RDONLY))
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
vinfo.open_vnode(fd,
vinfo.path,
flags & ~(os.O_TRUNC | os.O_RDWR | os.O_WRONLY)
| os.O_RDONLY,
discard=True)
return pyfuse3.FileInfo(fh=fd)
else:
if flags & os.O_RDWR and not (self.auditor.ask_writable(
vinfo.path) and self.auditor.ask_readable(vinfo.path)):
_opslog.info(
'Reading and writing to PATH <{}> is not permitted.'.
format(vinfo.path))
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
if flags & os.O_WRONLY and not self.auditor.ask_writable(
vinfo.path):
_opslog.info('Writing to PATH <{}> is not permitted.'.format(
vinfo.path))
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
if not flags & (os.O_RDWR | os.O_WRONLY
) and not self.auditor.ask_readable(vinfo.path):
_opslog.info('Reading from PATH <{}> is not permitted.'.format(
vinfo.path))
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
try:
fd = FD(os.open(vinfo.path, flags))
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
# Record accessed files;
if flags & os.O_RDWR:
self.stat_path_open_rw.add(vinfo.path)
elif flags & os.O_WRONLY:
self.stat_path_open_w.add(vinfo.path)
else:
self.stat_path_open_r.add(vinfo.path)
vinfo.open_vnode(fd, vinfo.path, flags, discard=False)
return pyfuse3.FileInfo(fh=fd)
async def open(self, inode, flags, ctx):
if inode not in self._files:
raise pyfuse3.FUSEError(errno.ENOENT)
logvfs.info("open(%s)", self._files[inode].fname)
if flags & os.O_RDWR or flags & os.O_WRONLY:
logvfs.info("error: readonly")
raise pyfuse3.FUSEError(errno.EPERM)
return pyfuse3.FileInfo(fh=inode)
async def rmdir(self, vnode_parent, name, ctx):
path = self.vm.make_path(self.vm[vnode_parent].path, os.fsdecode(name))
vinfo = self.vm[path]
if not self.auditor.ask_writable(path):
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
if self.auditor.ask_discard(path):
_acslog.info('RMDIR-FAKE: {}'.format(path))
return
try:
os.rmdir(path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
vinfo.remove_path(path)
_acslog.info('RMDIR: {}'.format(path))
async def write(self, inode, offset, data):
self.logger.debug("write: %s %s %s", inode, offset, len(data))
# Only 'append once' semantics are supported.
grid_in = self.active_writes.get(inode)
if grid_in is None:
raise pyfuse3.FUSEError(errno.EINVAL)
if offset != grid_in_size(grid_in):
raise pyfuse3.FUSEError(errno.EINVAL)
grid_in.write(data)
return len(data)
async def lookup(self, vnode_parent, name_enced, ctx=None):
name = os.fsdecode(name_enced)
path = self.vm.make_path(self.vm[vnode_parent].path, name)
_opslog.debug("lookup called with path: {}".format(path))
if self._path_mountpoint in (self.vm.make_path(p, name)
for p in self.vm[vnode_parent].paths):
raise pyfuse3.FUSEError(
errno.ENOENT) # Response that mountpoint is not exists.
vinfo = self.vm[path] if path in self.vm else self.vm.create_vinfo()
if not os.path.lexists(path) and not vinfo.virtual:
raise pyfuse3.FUSEError(errno.ENOENT)
if name != '.' and name != '..':
vinfo.add_path(path)
return self._getattr(vinfo)
async def lookup(self, parent_inode, name, ctx=None):
logvfs.debug("lookup(%s,%s)" % (parent_inode, name))
if parent_inode != pyfuse3.ROOT_INODE or name not in self.file_by_name:
raise pyfuse3.FUSEError(errno.ENOENT)
return self.file_by_name[name].attr
async def rmdir(self, inode_p, name, ctx):
entry = await self.lookup(inode_p, name)
if not stat.S_ISDIR(entry.st_mode):
raise pyfuse3.FUSEError(errno.ENOTDIR)
self._remove(inode_p, name, entry)
async def getattr(self, inode: int, ctx=None):
if inode == pyfuse3.ROOT_INODE:
return root_attr()
elif inode in self._files:
return self._files[inode].attr
else:
raise pyfuse3.FUSEError(errno.ENOENT)
async def unlink(self, vnode_parent, name_enced, ctx):
name = os.fsdecode(name_enced)
vinfo_p = self.vm[vnode_parent]
path = self.vm.make_path(vinfo_p.path, name)
vinfo = self.vm[path]
if not self.auditor.ask_writable(path):
raise pyfuse3.FUSEError(errno.EACCES) # Permission denied
if self.auditor.ask_discard(path):
_acslog.info('UNLINK-FAKE: {}'.format(path))
return
try:
os.unlink(path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
vinfo.remove_path(path)
_acslog.info('UNLINK: {}'.format(path))
def inode2entry(self, inode: int) -> FuseEntry:
""" Return the entry matching a given inode """
try:
return self._inode2entry[inode]
except KeyError:
raise pyfuse3.FUSEError(errno.ENOENT)
async def readlink(self, vnode, ctx):
path = self.vm[vnode].path
try:
target = os.readlink(path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
return os.fsencode(target)
async def removexattr(self, vnode, name_enced, ctx):
name = os.fsdecode(name_enced)
path = self.vm[vnode].path
try:
os.removexattr(path, name)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
async def get_blob(self, swhid: CoreSWHID) -> bytes:
""" Retrieve the blob bytes for a given content SWHID using Software
Heritage API """
if swhid.object_type != ObjectType.CONTENT:
raise pyfuse3.FUSEError(errno.EINVAL)
# Make sure the metadata cache is also populated with the given SWHID
await self.get_metadata(swhid)
cache = await self.cache.blob.get(swhid)
if cache:
self.logger.debug("Found blob %s in cache", swhid)
return cache
try:
self.logger.debug("Retrieving blob %s via web API...", swhid)
loop = asyncio.get_event_loop()
resp = await loop.run_in_executor(None, self.web_api.content_raw,
swhid)
blob = b"".join(list(resp))
await self.cache.blob.set(swhid, blob)
return blob
except requests.HTTPError as err:
self.logger.error("Cannot fetch blob for object %s: %s", swhid,
err)
raise
def _delete_inode(self, folder_inode, bname, entry_check):
# On insert the order is like this
# 1. write into the database.
# the unique index (parent_inode, filename) protects
# 2. Update the folder inode
# On remove the order must be vice verca
# 1. Remove from the folder inode
# 2. Remove from the database
# In that case the unique index protection is true
# Names are in bytes, so translate to UTF-8
name = bname.decode(self._filename_encoding, 'replace')
parent = self._entry_by_inode(folder_inode)
if name not in parent.childs:
raise pyfuse3.FUSEError(errno.ENOENT)
inode = parent.childs[name]
entry = self._entry_by_inode(inode)
entry_check(entry)
# Remove from the folder node
query = {"_id": folder_inode}
update = {"$pull": {'childs': (name, inode)}}
self.meta.update_one(query, update)
# Remove from the database
self.meta.delete_one({"_id": inode})
# Remove from the grids collections
self.gridfs.delete(inode)
def getattr_sync(self, inode, ctx=None):
try:
row = self.get_row("SELECT * FROM inodes WHERE id=?", (inode, ))
except NoSuchRowError:
raise (pyfuse3.FUSEError(errno.ENOENT))
entry = pyfuse3.EntryAttributes()
entry.st_ino = inode
entry.generation = 0
entry.entry_timeout = 300
entry.attr_timeout = 300
entry.st_mode = row['mode']
entry.st_nlink = self.get_row(
"SELECT COUNT(inode) FROM contents WHERE inode=?", (inode, ))[0]
entry.st_uid = row['uid']
entry.st_gid = row['gid']
entry.st_rdev = row['rdev']
entry.st_size = row['size']
entry.st_blksize = 512
entry.st_blocks = 1
entry.st_atime_ns = row['atime_ns']
entry.st_mtime_ns = row['mtime_ns']
entry.st_ctime_ns = row['ctime_ns']
return entry
async def listxattr(self, vnode, ctx):
path = self.vm[vnode].path
try:
xattrs = os.listxattr(path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
return list(map(os.fsencode, xattrs))
async def get_history(self, swhid: CoreSWHID) -> List[CoreSWHID]:
""" Retrieve a revision's history using Software Heritage Graph API """
if swhid.object_type != ObjectType.REVISION:
raise pyfuse3.FUSEError(errno.EINVAL)
cache = await self.cache.history.get(swhid)
if cache:
self.logger.debug("Found history of %s in cache (%d ancestors)",
swhid, len(cache))
return cache
try:
# Use the swh-graph API to retrieve the full history very fast
self.logger.debug("Retrieving history of %s via graph API...",
swhid)
call = f"graph/visit/edges/{swhid}?edges=rev:rev"
loop = asyncio.get_event_loop()
history = await loop.run_in_executor(None, self.web_api._call,
call)
await self.cache.history.set(history.text)
# Retrieve it from cache so it is correctly typed
res = await self.cache.history.get(swhid)
return res
except requests.HTTPError as err:
self.logger.error("Cannot fetch history for object %s: %s", swhid,
err)
# Ignore exception since swh-graph does not necessarily contain the
# most recent artifacts from the archive. Computing the full history
# from the Web API is too computationally intensive so simply return
# an empty list.
return []