def rename(self, old, new):
logger.debug("rename(%s, %s)", old, new)
pathinfo_old = PathInfo(old)
pathinfo_new = PathInfo(new)
# Fusepy should block these possibilities:
assert pathinfo_old != pathinfo_new
assert not self._is_reserved_name(os.path.basename(old))
if self._is_reserved_name(os.path.basename(new)):
raise FuseOSError(errno.EINVAL)
# if self._exists(new):
# raise FuseOSError(errno.EEXIST)
# +---------------+----------+-------------+-------------+-----------------+
# | Source \ Dest | Normal | Entry Point | Tag | Tagged File/Dir |
# | | File/Dir | | | |
# +---------------+----------+-------------+-------------+-----------------+
# | Normal file | STD | fail | fail | OK |
# | Normal dir | STD | OK | OK if empty | OK |
# | Entry point | OK? | STD | OK? | OK? |
# | Tag | OK | OK? | OK | OK |
# | Tagged file | OK | fail | fail | OK |
# | Tagged folder | OK | OK | OK if empty | OK |
# +---------------+----------+-------------+-------------+-----------------+
if pathinfo_old.is_entrypoint:
self._move_entry_point(pathinfo_old, pathinfo_new)
elif pathinfo_old.is_tag:
self._move_tag(pathinfo_old, pathinfo_new)
elif pathinfo_old.is_tagged_object:
self._move_tagged_obj(pathinfo_old, pathinfo_new)
else:
self._move_standard_obj(pathinfo_old, pathinfo_new)
def test_path_type_recognition(self):
for k in self._testpaths.keys():
with self.subTest(k=k):
self.assertEqual(
PathInfo(k).is_standard_object, self._testpaths[k][0], k)
self.assertEqual(
PathInfo(k).is_tagged_object, self._testpaths[k][1], k)
self.assertEqual(PathInfo(k).is_tag, self._testpaths[k][2], k)
self.assertEqual(
PathInfo(k).is_entrypoint, self._testpaths[k][3], k)
def unlink(self, path):
"""
* Standard file: standard behavior
* Tagged file:
- if path points to a file directly under the entry poiny,
it completely deletes the file.
- if path points to a file with one or more tags, remove
from the file the last tag in the path.
:param path:
:return:
"""
pathinfo = PathInfo(path)
if pathinfo.is_tag or pathinfo.is_entrypoint:
raise FuseOSError(errno.EISDIR)
elif pathinfo.is_tagged_object:
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
assert len(pathinfo.tagged_object) > 0
if len(pathinfo.tags) == 0:
# If it's directly under the entry point, delete it.
os.unlink(self._datastore_path(path))
semfolder.filetags.remove_file(pathinfo.tagged_object)
else:
# If it's a tagged path, remove the last tag.
semfolder.filetags.discard_tag(pathinfo.tagged_object,
pathinfo.tags[-1])
self._save_semantic_folder(semfolder)
else:
os.unlink(self._datastore_path(path))
def readdir(self, path: str, fh):
"""
Yelds the list of files and directories within the provided one.
Already traversed tags of a semantic directory are not shown.
:param path:
:param fh:
"""
dirents = []
storepath = self._datastore_path(path)
if os.path.isdir(storepath):
pathinfo = PathInfo(path)
if pathinfo.is_tag:
folder = self._get_semantic_folder(pathinfo.entrypoint)
# Show tags first
dirents.extend(folder.graph.outgoing_arcs(pathinfo.tags[-1]))
dirents.extend(folder.filetags.tagged_files(pathinfo.tags))
elif pathinfo.is_entrypoint:
dirents.extend(os.listdir(storepath))
else:
dirents.extend(os.listdir(storepath))
# Remove reserved names and already traversed tags
dirents = [
x for x in dirents if not SemanticFS._is_reserved_name(x)
and x not in pathinfo.tags
]
for r in ['.', '..'] + dirents:
yield r
def _move_standard_obj(self, old: PathInfo, new: PathInfo):
"""
Helper method for renaming a standard file or folder.
We can have the following cases:
* Destination is a standard object:
The source file is renamed just like the standard file system behavior
* Destination is an entry point:
The source object must be a directory.
[TO BE COMPLETED]
* Destination is a tag: not supported
* Destination is a tagged object: [TO BE COMPLETED]
:param old:
:param new:
"""
old_dspath = self._datastore_path(old.path)
new_dspath = self._datastore_path(new.path)
is_file = os.path.isfile(old_dspath)
if new.is_standard_object:
os.rename(old_dspath, new_dspath)
elif new.is_entrypoint:
if is_file:
# Fail: trying to convert a file to an entry point
raise FuseOSError(errno.ENOTSUP)
else:
# Convert src dir to an entry point
# Fails if source dir contains an entry point
for p in os.listdir(old_dspath):
if not self._is_reserved_name(
p) and PathInfo.is_semantic_name(p):
raise FuseOSError(errno.ENOTSUP)
os.rename(old_dspath, new_dspath)
semfolder = SemanticFolder(new.path)
for f in os.listdir(new_dspath):
semfolder.filetags.add_file(f)
self._save_semantic_folder(semfolder)
elif new.is_tag:
if is_file:
# Fail: trying to convert a file to a tag
raise FuseOSError(errno.ENOTSUP)
else:
# Convert src dir to a tag
self._convert_folder_to_tag(old, new)
elif new.is_tagged_object:
# Move this obj to the destination entry point, then add the tags.
semfolder = self._get_semantic_folder(new.entrypoint)
os.rename(
old_dspath,
new_dspath) # Fails if new is an existing non-empty directory
try:
semfolder.filetags.add_file(new.tagged_object, new.tags)
except ValueError:
semfolder.filetags.assign_tags(new.tagged_object, new.tags)
self._save_semantic_folder(semfolder)
else:
# Impossible!
assert False, "Impossible destination"
def _exists(self, path: str) -> bool:
"""
Test whether the specified virtual path exists in the file system.
:param path: a virtual path
:return:
"""
# Extracts from the path all the files that belong to a semantic directory in the path.
# E.g., given the path "/a/_b/_c/d/e/_f/g/_h", we get
# [ '/a/_b/_c/d',
# '/a/_b/_c/d/e/_f/g',
# '/a/_b/_c/d/e/_f/g/_h' (because it's the last one and it's semantic)
# ]
components = os.path.normcase(os.path.normpath(path)).split(os.sep)
semantic_endpoints = []
prev_was_semantic = False
for i, name in enumerate(components):
curr_is_semantic = PathInfo.is_semantic_name(name)
if prev_was_semantic and not curr_is_semantic:
semantic_endpoints.append(os.sep.join(components[0:i + 1]))
prev_was_semantic = curr_is_semantic
assert not PathInfo.is_semantic_name(components[-1]) or os.sep.join(
components) not in semantic_endpoints
if len(components) > 0 and PathInfo.is_semantic_name(components[-1]):
semantic_endpoints.append(os.sep.join(components))
for subpath in semantic_endpoints:
pathinfo = PathInfo(subpath)
assert pathinfo.is_tag or pathinfo.is_tagged_object or pathinfo.is_entrypoint
try:
folder = self._get_semantic_folder(pathinfo.entrypoint)
except FileNotFoundError:
return False
if pathinfo.is_tagged_object and not folder.filetags.has_file(
pathinfo.tagged_object):
return False
if not folder.graph.has_path(pathinfo.tags):
return False
if pathinfo.is_tagged_object and not folder.filetags.has_tags(
pathinfo.tagged_object, pathinfo.tags):
return False
return os.path.lexists(self._datastore_path(path))
def _datastore_path(self, virtualpath: str) -> str:
"""
Returns the path (of another file system) where the provided virtual object is actually stored.
For example:
* /a/_b/_c/x -> dsroot/a/_b/x
* /a/_b/_c/ -> dsroot/a/_b/_c/
* /a/_b/_c/_d/ -> dsroot/a/_b/_d/
:param virtualpath: an absolute virtual path
:return:
"""
# NB: Using a 1-1 mapping for file names, we inherit the limitations of the underlying fs (e.g.
# special file names, unallowed characters, case sensitivity, etc. In addition, this fs will
# behave differently depending on the file system on which it's run.
if not os.path.isabs(virtualpath):
raise ValueError("virtualpath should be absolute")
components = os.path.normcase(os.path.normpath(virtualpath)).split(
os.sep)
tmppath = []
for i, name in enumerate(components):
if i == 0 or i == 1:
tmppath.append(name)
else:
if PathInfo.is_semantic_name(
tmppath[-2]) and PathInfo.is_semantic_name(
tmppath[-1]):
# _a/_b/_c => _a/_c
# _a/_b/x => _a/x
del tmppath[-1]
tmppath.append(name)
tmppath = os.sep.join(tmppath)
# Remove the root from the path
tmppath = os.path.splitdrive(tmppath)[1]
if tmppath.startswith(os.sep):
tmppath = tmppath[len(os.sep):]
# Join the path with the datastore path
path = os.path.join(self._dsroot, tmppath)
return path
def release(self, path, fh):
logger.debug("close(%s, %d)", path, fh)
if fh in self._sem_write_descriptors:
assert self._has_ghost_file(path) and PathInfo(
path).is_tagged_object
self._get_ghost_file(path).apply(fh)
self._delete_ghost_file(path)
self._sem_write_descriptors.remove(fh)
return os.close(fh)
def symlink(self, name, target):
logger.debug("symlink(%s, %s)", name, target)
target_norm = os.path.normcase(os.path.normpath(target))
pathinfo_name = PathInfo(name)
if pathinfo_name.is_standard_object:
return os.symlink(target, self._datastore_path(name))
elif not os.path.isabs(target_norm) and len(target_norm.split(os.sep)) == 1 and \
pathinfo_name.is_tag and target_norm == pathinfo_name.tags[-1]:
# ln -s /_sem/_c /_sem/_a/_b/_c
if self._exists(os.path.join(pathinfo_name.entrypoint,
target_norm)):
# Add the link to the tag
if len(pathinfo_name.tags) >= 2:
semfolder = self._get_semantic_folder(
pathinfo_name.entrypoint)
semfolder.graph.add_arc(pathinfo_name.tags[-2],
pathinfo_name.tags[-1])
self._save_semantic_folder(semfolder)
else:
raise FuseOSError(errno.ENOENT)
elif not os.path.isabs(target_norm) and len(target_norm.split(os.sep)) == 1 and \
pathinfo_name.is_tagged_object and target_norm == pathinfo_name.tagged_object:
# Add the tags to the tagged object
if self._exists(os.path.join(pathinfo_name.entrypoint,
target_norm)):
semfolder = self._get_semantic_folder(pathinfo_name.entrypoint)
assert semfolder.filetags.has_file(pathinfo_name.tagged_object)
semfolder.filetags.assign_tags(pathinfo_name.tagged_object,
pathinfo_name.tags)
self._save_semantic_folder(semfolder)
else:
raise FuseOSError(errno.ENOENT)
elif pathinfo_name.is_tagged_object:
semfolder = self._get_semantic_folder(pathinfo_name.entrypoint)
if semfolder.filetags.has_file(pathinfo_name.tagged_object):
raise FuseOSError(errno.EEXIST)
else:
os.symlink(target, self._datastore_path(name))
semfolder.filetags.add_file(pathinfo_name.tagged_object,
pathinfo_name.tags)
self._save_semantic_folder(semfolder)
else:
raise FuseOSError(errno.ENOTSUP)
def open(self, path, flags):
dspath = self._datastore_path(path)
f = os.open(dspath, flags)
logger.debug("open(%s, %s) -> %d", path,
SemanticFS._stringify_open_flags(flags), f)
if flags & (os.O_WRONLY | os.O_RDWR) != 0:
pathinfo = PathInfo(path)
if pathinfo.is_tagged_object:
assert f not in self._sem_write_descriptors
self._sem_write_descriptors.add(f)
self._add_ghost_file(path)
return f
def _convert_folder_to_tag(self, old: PathInfo, new: PathInfo):
if not (old.is_tagged_object or old.is_standard_object):
raise ValueError(
"Can only convert tagged object or standard object.")
if not new.is_tag:
raise ValueError(
"Tagged object or standard object can only be converted to tag."
)
old_dspath = self._datastore_path(old.path)
# Fails if source dir contains an entry point
for p in os.listdir(old_dspath):
if not self._is_reserved_name(p) and PathInfo.is_semantic_name(p):
raise FuseOSError(errno.ENOTSUP)
srcfiles = os.listdir(old_dspath)
semfolder = SemanticFolder(new.entrypoint)
if set(srcfiles) & set(semfolder.filetags.files()):
# Name conflict: fails
raise FuseOSError(errno.ENOTSUP)
else:
semfolder = self._get_semantic_folder(new.entrypoint)
dir_mode = os.lstat(old_dspath).st_mode & 0o777
self.mkdir(
new.path,
dir_mode) # FIXME Avoid calling mkdir... do this internally
if not semfolder.graph.has_node(new.tags[-1]):
semfolder.graph.add_node(new.tags[-1])
if len(new.tags) > 1:
semfolder.graph.add_arc(new.tags[-2], new.tags[-1])
entrypoint_dspath = self._datastore_path(new.entrypoint)
for f in srcfiles:
file_dspath = os.path.join(old_dspath, f)
if os.path.isfile(file_dspath):
shutil.copy2(file_dspath, entrypoint_dspath)
else:
shutil.copytree(file_dspath,
os.path.join(entrypoint_dspath, f))
semfolder.filetags.add_file(f, new.tags)
self._save_semantic_folder(semfolder)
self.rmdir(
old.path) # FIXME Avoid calling rmdir... do this internally
def mknod(self, path, mode, dev):
logger.debug("mknod(%s)", path)
pathinfo = PathInfo(path)
dspath = self._datastore_path(path)
os.mknod(dspath, mode, dev)
# Files starting with the semantic prefix are not allowed
if not (pathinfo.is_tagged_object or pathinfo.is_standard_object):
raise FuseOSError(errno.ENOTSUP)
if pathinfo.is_tagged_object:
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
if semfolder.filetags.has_file(pathinfo.tagged_object):
semfolder.filetags.assign_tags(pathinfo.tagged_object,
pathinfo.tags)
else:
semfolder.filetags.add_file(pathinfo.tagged_object,
pathinfo.tags)
self._save_semantic_folder(semfolder)
def create(self, path, mode, fi=None):
"""
* Standard file: standard behavior
* Tagged file: create the file directly under the entry point, and add
the appropriate tags. If the file already exists under the entry point,
just add the tags.
:param path:
:param mode:
:param fi:
:return: write descriptor for the file
"""
pathinfo = PathInfo(path)
dspath = self._datastore_path(path)
f = os.open(dspath, os.O_WRONLY | os.O_CREAT, mode)
logger.debug(
"create(%s, %s) -> %d", path,
SemanticFS._stringify_open_flags(os.O_WRONLY | os.O_CREAT), f)
# Files starting with the semantic prefix are not allowed
if not (pathinfo.is_tagged_object or pathinfo.is_standard_object):
raise FuseOSError(errno.ENOTSUP)
if pathinfo.is_tagged_object:
assert f not in self._sem_write_descriptors
self._sem_write_descriptors.add(f)
self._add_ghost_file(path).truncate(0)
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
if semfolder.filetags.has_file(pathinfo.tagged_object):
semfolder.filetags.assign_tags(pathinfo.tagged_object,
pathinfo.tags)
else:
semfolder.filetags.add_file(pathinfo.tagged_object,
pathinfo.tags)
self._save_semantic_folder(semfolder)
assert self._has_ghost_file(path)
return f
def _add_ghost_file(self, ghost_path: str) -> GhostFile:
"""
Adds a ghost file for the specified virtual path.
If a ghost file already exists for that path, it doesn't add another one but keeps track
of this additional reference (see `SemanticFS._delete_ghost_file`).
:param ghost_path: the virtual path for the ghost file
:return: the added GhostFile
"""
dspath = self._datastore_path(ghost_path)
normpath = os.path.normcase(os.path.normpath(ghost_path))
if (dspath, normpath) in self._sem_writing_files:
assert self._sem_writing_files_count[dspath, normpath] > 0
self._sem_writing_files_count[dspath, normpath] += 1
else:
assert (dspath, normpath) not in self._sem_writing_files_count
self._sem_writing_files[dspath, normpath] = GhostFile(
dspath, lambda: self._clear_stat_cache(PathInfo(ghost_path)))
self._sem_writing_files_count[dspath, normpath] = 1
assert (dspath, normpath
) in self._sem_writing_files and self._sem_writing_files_count[
dspath, normpath] > 0
return self._sem_writing_files[dspath, normpath]
def mkdir(self, path, mode):
"""
* Standard directory: standard behavior
* Entry point: creates the specified directory and adds the
necessary metadata.
* Tag:
- if path points to a tag directly under the entry point, it adds the folder
to the entry point and adds the relative node to the graph. Fails if the
tag did exist.
- if path points to a tag contained within another tag, it adds a link in the
graph from the containing tag to the new one (if the tag that is being
added didn't already exist within the semantic directory, it first adds the
node to the graph and the tag folder to the entry point).
Fails if the specified tag (associated to this semantic folder) is already
present within the destination path. In other words, a tag can't be added
if it has already been traversed.
* Tagged folder: create the folder directly under the entry point, and add
the appropriate tags. If the folder already exists under the entry point,
just add the tags.
:param path:
:param mode:
:raise FuseOSError:
"""
pathinfo = PathInfo(path)
if pathinfo.is_tag:
# Creating a new tag
if pathinfo.tags[-1] in pathinfo.tags[0:-1]:
raise FuseOSError(errno.EEXIST)
logger.debug("Creating tag: %s", path)
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
if not semfolder.graph.has_node(pathinfo.tags[-1]):
# Create the tag dir in the entry point's root
os.mkdir(self._datastore_path(path), mode)
semfolder.graph.add_node(pathinfo.tags[-1])
if len(pathinfo.tags) >= 2:
semfolder.graph.add_arc(pathinfo.tags[-2], pathinfo.tags[-1])
self._save_semantic_folder(semfolder)
elif pathinfo.is_entrypoint:
# Creating a new entry point
logger.debug("Creating entry point: %s", path)
os.mkdir(self._datastore_path(path), mode)
self._save_semantic_folder(SemanticFolder(path))
elif pathinfo.is_tagged_object:
# Adding a standard folder to a semantic directory
logger.debug("Adding standard folder to semantic dir: %s", path)
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
if semfolder.filetags.has_file(pathinfo.tagged_object):
semfolder.filetags.assign_tags(pathinfo.tagged_object,
pathinfo.tags)
else:
os.mkdir(self._datastore_path(path), mode)
semfolder.filetags.add_file(pathinfo.tagged_object,
pathinfo.tags)
self._save_semantic_folder(semfolder)
else:
# No semantic parts... do a normal mkdir
os.mkdir(self._datastore_path(path), mode)
def rmdir(self, path):
"""
* Standard directory: standard behavior
* Entry point: standard behavior
* Tag:
- if path points to a tag directly under the entry point,
it completely deletes the tag. Fails if tag is not empty.
- if path points to a tag contained within another tag,
it removes the corresponding link in the graph. Doesn't
fail if the tag is not empty.
* Tagged folder:
- if path points to a folder directly under the entry point,
it completely deletes the folder. Fails if the folder is not
empty, as would do the standard os call.
- if path points to a folder with one or more tags, remove
from the folder the last tag in the path. Doesn't fail if
the folder is not empty.
:param path:
:return:
"""
pathinfo = PathInfo(path)
if pathinfo.is_tag:
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
self._rmdir_tag(pathinfo, semfolder)
self._save_semantic_folder(semfolder)
elif pathinfo.is_entrypoint:
dspath = self._datastore_path(path)
# Even if the dir is logically empty, we can't remove it from the datastore because it contains
# some special files. So first we make sure that the dir is empty from the user point-of-view, then
# we unlink the special files, and at last we remove the directory.
files = os.listdir(dspath)
fsfiles = [
SemanticFS.SEMANTIC_FS_GRAPH_FILE_NAME,
SemanticFS.SEMANTIC_FS_ASSOC_FILE_NAME
]
if set(files).issubset(fsfiles):
for f in fsfiles:
os.unlink(os.path.join(dspath, f))
os.rmdir(dspath)
else:
raise FuseOSError(errno.ENOTEMPTY)
elif pathinfo.is_tagged_object:
semfolder = self._get_semantic_folder(pathinfo.entrypoint)
assert len(pathinfo.tagged_object) > 0
if len(pathinfo.tags) == 0:
# If it's directly under the entry point, delete it.
os.rmdir(self._datastore_path(
path)) # Raises error if dir is not empty
semfolder.filetags.remove_file(pathinfo.tagged_object)
else:
# If it's a tagged path, remove the last tag.
semfolder.filetags.discard_tag(pathinfo.tagged_object,
pathinfo.tags[-1])
self._save_semantic_folder(semfolder)
else:
os.rmdir(self._datastore_path(path))
def _move_tagged_obj(self, old: PathInfo, new: PathInfo):
"""
Helper method for renaming a tagged file or folder.
:param old:
:param new:
"""
old_dspath = self._datastore_path(old.path)
new_dspath = self._datastore_path(new.path)
is_file = os.path.isfile(old_dspath)
same_semantic_space = old.entrypoint == new.entrypoint
if new.is_standard_object:
# Remove the object from src and put it outside
self._extract_tagged_object(old, new)
elif new.is_entrypoint:
if is_file:
# Fail: trying to convert a file to an entry point
raise FuseOSError(errno.ENOTSUP)
else:
# Convert src dir to an entry point
# Fails if source dir contains an entry point
for p in os.listdir(old_dspath):
if not self._is_reserved_name(
p) and PathInfo.is_semantic_name(p):
raise FuseOSError(errno.ENOTSUP)
self._extract_tagged_object(old, new)
semfolder = SemanticFolder(new.entrypoint)
for f in os.listdir(new_dspath):
semfolder.filetags.add_file(f)
self._save_semantic_folder(semfolder)
elif new.is_tag:
if is_file:
# Fail: trying to convert a file to a tag
raise FuseOSError(errno.ENOTSUP)
else:
# Convert src dir to a tag
self._convert_folder_to_tag(old, new)
elif new.is_tagged_object:
if same_semantic_space:
# Moving over itself. This case should have already been prevented by FUSE!
assert not (old.tagged_object == new.tagged_object
and set(old.tags) == set(new.tags))
if old.tagged_object != new.tagged_object and set(
old.tags) == set(new.tags):
# These cases:
# * mv /_sem/_t1/x /_sem/_t1/y
# * mv /_sem/x /_sem/y
# Rename the file in the root and in filestagsassociations
assert old.tagged_object != "" and new.tagged_object != ""
os.rename(old_dspath, new_dspath)
semfolder = self._get_semantic_folder(new.entrypoint)
semfolder.filetags.rename_file(old.tagged_object,
new.tagged_object)
self._save_semantic_folder(semfolder)
elif old.tagged_object != new.tagged_object and set(
old.tags) != set(new.tags):
# mv /_sem/_t1/x /_sem/_t2/y is not supported
raise FuseOSError(errno.ENOTSUP)
elif old.tagged_object == new.tagged_object and set(
old.tags) != set(new.tags):
# These cases:
# * mv /_sem/_t1/x /_sem/_t2/x
# * mv /_sem/x /_sem/_t3/x
semfolder = self._get_semantic_folder(new.entrypoint)
if len(old.tags) > 0:
semfolder.filetags.discard_tag(old.tagged_object,
old.tags[-1])
semfolder.filetags.assign_tags(new.tagged_object, new.tags)
self._save_semantic_folder(semfolder)
else:
self._extract_tagged_object(old, new)
semfolder = self._get_semantic_folder(new.entrypoint)
semfolder.filetags.add_file(new.tagged_object, new.tags)
self._save_semantic_folder(semfolder)
else:
# Impossible!
assert False, "Impossible destination"