def test_containment(self):
vault = DummyVault()
with TemporaryDirectory() as root:
# Set up vault like so: /path/to/tmp/${_DUMMY_VAULT}/
# + foo/
# | + foo
# + bar/
# + bar
root = T.Path(root)
for branch in DummyBranch:
bpath = branch.value
path = root / _DUMMY_VAULT / bpath
path.mkdir(parents=True)
filename = path / bpath
filename.touch()
vault.root = root
for branch in DummyBranch:
bpath = branch.value
self.assertEqual(vault.branch(bpath), branch)
self.assertTrue(bpath in vault)
not_in_vault = T.Path("path/to/nowhere")
self.assertIsNone(vault.branch(not_in_vault))
self.assertFalse(not_in_vault in vault)
def test_sibling_to_work_dir(self):
work_dir = T.Path("this/is/my/path")
vault_relative_path = T.Path("this/is/my/file3")
expected = T.Path("../file3")
work_dir_rel = relativise(vault_relative_path, work_dir)
self.assertEqual(expected, work_dir_rel)
def test_child_to_work_dir(self):
work_dir = T.Path("some/path")
vault_relative_path = T.Path("some/path/file1")
expected = T.Path("file1")
work_dir_rel = relativise(vault_relative_path, work_dir)
self.assertEqual(expected, work_dir_rel)
class Branch(core.vault.base.Branch):
""" HGI vault branches """
Keep = T.Path("keep")
Archive = T.Path("archive")
Staged = T.Path(".staged")
Limbo = T.Path(".limbo")
Stash = T.Path(".stash")
def test_constructor(self):
self.assertEqual(VFK(_DUMMY, 0x1).path, T.Path(f"01-{_B64_DUMMY}"))
self.assertEqual(VFK(_DUMMY, 0x12).path, T.Path(f"12-{_B64_DUMMY}"))
self.assertEqual(VFK(_DUMMY, 0x123).path,
T.Path(f"01/23-{_B64_DUMMY}"))
self.assertEqual(VFK(_DUMMY, 0x1234).path,
T.Path(f"12/34-{_B64_DUMMY}"))
def test_change_location_of_vaulted_file(self):
self.child_of_child_dir_one = self.child_dir_one / "child_of_child_dir_one"
self.child_of_child_dir_one.mkdir()
self.child_of_child_dir_one.chmod(0o330)
self.new_location_tmp_file_a = self.child_of_child_dir_one / "new_location_tmp_file_a"
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no_old = self.tmp_file_a.stat().st_ino
vault_file_key_path_old = VFK(T.Path("a"), inode_no_old).path
vault_file_path_old = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path_old
self.assertTrue(os.path.isfile(vault_file_path_old))
shutil.move(self.tmp_file_a, self.new_location_tmp_file_a)
self.vault.add(Branch.Keep, self.new_location_tmp_file_a)
inode_no = self.new_location_tmp_file_a.stat().st_ino
vault_file_key_path = VFK(
T.Path("child_of_child_dir_one") / "new_location_tmp_file_a", inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path
self.assertTrue(os.path.isfile(vault_file_path))
self.assertFalse(os.path.isfile(vault_file_path_old))
def test_child_to_work_dir(self):
work_dir = T.Path("some/path")
work_dir_rel = T.Path("file1")
vault_path = T.Path("/this/is/vault/root")
vault_relative_path = derelativise(work_dir_rel, work_dir, vault_path)
expected = T.Path("some/path/file1")
self.assertEqual(expected, vault_relative_path)
def test_sibling_to_work_dir(self):
work_dir = T.Path("this/is/my/path")
work_dir_rel = T.Path("../file3")
vault_path = T.Path("/this/is/vault/root")
vault_relative_path = derelativise(work_dir_rel, work_dir, vault_path)
expected = T.Path("this/is/my/file3")
self.assertEqual(expected, vault_relative_path)
def test_archive_stash_fofn(self, mock_add, mock_remove, mock_file):
main(["__init__", "archive", "--stash", "--fofn", "mock_file"])
args = mock_add.call_args.args
files = list(args[1])
branch = args[0]
self.assertEqual(files, [T.Path("/file1"), T.Path("/file2")])
self.assertEqual(branch, Branch.Stash)
mock_remove.assert_not_called()
def test_list(self):
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no = self.tmp_file_a.stat().st_ino
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
VFK(T.Path("a"), inode_no).path
self.assertEqual(next(self.vault.list(Branch.Keep)),
(self.tmp_file_a, vault_file_path))
def test_reconstructor_long(self):
self.assertEqual(VFK_k(T.Path(
f"01-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG)
self.assertEqual(VFK_k(T.Path(
f"12-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG)
self.assertEqual(VFK_k(T.Path(
f"01/23-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG)
self.assertEqual(VFK_k(T.Path(
f"12/34-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG)
def test_remove_not_existing_file(self):
inode_no = self.tmp_file_b.stat().st_ino
vault_file_key_path = VFK(T.Path("a"), inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
VFK(T.Path("a"), inode_no).path
self.assertFalse(os.path.isfile(vault_file_path))
self.vault.remove(Branch.Keep, self.tmp_file_a)
self.assertFalse(os.path.isfile(vault_file_path))
def test_constructor_longest(self):
self.assertEqual(VFK(_DUMMY_LONGEST, 0x1).path, T.Path(
f"01-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}"))
self.assertEqual(VFK(_DUMMY_LONGEST, 0x12).path, T.Path(
f"12-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}"))
self.assertEqual(VFK(_DUMMY_LONGEST, 0x123).path, T.Path(
f"01/23-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}"))
self.assertEqual(VFK(_DUMMY_LONGEST, 0x1234).path, T.Path(
f"12/34-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}"))
def setUp(self) -> None:
"""
The following tests will emulate the following directory structure
+- tmp
+- parent/
+- child_dir_one
+- a
+- b
+-.vault/
+- keep
+- archive
...
+- child_dir_two
+- c
"""
_dummy_idm = DummyIDM(config)
self._tmp = TemporaryDirectory()
self._path = path = T.Path(self._tmp.name).resolve()
# Form a directory hierarchy
self.parent_dir = path / "parent_dir"
self.child_dir_one = self.parent_dir / "child_dir_one"
self.child_dir_two = self.parent_dir / "child_dir_two"
self.tmp_file_a = self.child_dir_one / "a"
self.tmp_file_b = self.child_dir_one / "b"
self.tmp_file_c = self.child_dir_two / "c"
self.child_dir_one.mkdir(parents=True, exist_ok=True)
self.child_dir_two.mkdir(parents=True, exist_ok=True)
self.tmp_file_a.touch()
self.tmp_file_b.touch()
self.tmp_file_c.touch()
# The following conditions should be checked upfront for each file and, if not satisfied, that action should fail for that file, logged appropriately:
# Check that the permissions of the file are at least ug+rw; 660+
# Check that the user and group permissions of the file are equal;66* or 77*
# Check that the file's parent directory permissions are at least
# ug+wx. 330+
# Default file permissions can be unsuitable for archiving, like 644
# (rw-r--r--, where owner and group dont have same permissions.
self.tmp_file_a.chmod(0o660) # rw, rw, _
self.tmp_file_b.chmod(0o644) # rw, r, r
self.tmp_file_c.chmod(0o777) # rwx, rwx, rwx
# Default parent dir permissions can be unsuitable for archiving, like
# 755 - write permissions are missing.
self.child_dir_one.chmod(0o730) # wx, wx, _
self.parent_dir.chmod(0o777) # rwx, rwx, rwx
Vault._find_root = MagicMock(
return_value=self._path / T.Path("parent_dir/child_dir_one"))
self.vault = Vault(relative_to=self._path /
T.Path("parent_dir/child_dir_one/a"), idm=_dummy_idm)
def list(self, branch: Branch) -> T.Iterator[T.Tuple[T.Path, T.Path]]:
# NOTE The order in which the listing is generated is
# unspecified (I suspect it will be by inode ID); it is up to
# downstream to modify this, as required
bpath = self.location / branch
return ((self.root / VaultFileKey.Reconstruct(
T.Path(dirname, file).relative_to(bpath)).source,
T.Path(dirname, file)) for dirname, _, files in os.walk(bpath)
for file in files)
def test_add(self):
# Add child_dir_one/tmp_file_b to vault and check whether hard link
# exists at desired location.
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no = self.tmp_file_a.stat().st_ino
vault_file_key_path = VFK(T.Path("a"), inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path
self.assertTrue(os.path.isfile(vault_file_path))
def setUp(self) -> None:
"""
The following tests will emulate the following directory structure
+- tmp
+- parent/
+- child_dir_one
+- a
+- b
+- perms_mod
+- perms_mod_dir
+- d
+-.vault/
+- keep
+- archive
...
+- child_dir_two
+- c
"""
_dummy_idm = DummyIDM(config)
self._tmp = TemporaryDirectory()
self._path = path = T.Path(self._tmp.name).resolve()
# Form a directory hierarchy
self.parent_dir = path / "parent_dir"
self.child_dir_one = self.parent_dir / "child_dir_one"
self.child_dir_two = self.parent_dir / "child_dir_two"
self.tmp_file_a = self.child_dir_one / "a"
self.tmp_file_b = self.child_dir_one / "b"
self.tmp_file_c = self.child_dir_two / "c"
self.perms_mod = self.child_dir_one / "perms_mod"
self.perms_mod_dir = self.child_dir_one / "perms_mod_dir"
self.tmp_file_d = self.perms_mod_dir / "d"
self.child_dir_one.mkdir(parents=True, exist_ok=True)
self.child_dir_two.mkdir(parents=True, exist_ok=True)
self.perms_mod_dir.mkdir(parents=True, exist_ok=True)
self.tmp_file_a.touch()
self.tmp_file_b.touch()
self.tmp_file_c.touch()
self.perms_mod.touch()
self.tmp_file_d.touch()
# The permissions of the file ought to be least ug+rw; 660+
# The user and group permissions of the file are equal;66* or 77*
# Thefile's parent directory permissions are at least ug+wx. 330+
self.tmp_file_a.chmod(0o660)
self.tmp_file_b.chmod(0o644)
self.tmp_file_c.chmod(0o777)
self.child_dir_one.chmod(0o330)
self.parent_dir.chmod(0o777)
self.perms_mod_dir.chmod(0o777)
self.tmp_file_d.chmod(0o664)
Vault._find_root = MagicMock(
return_value=self._path / T.Path("parent_dir/child_dir_one"))
self.vault = Vault(relative_to=self._path /
T.Path("parent_dir/child_dir_one/a"), idm=_dummy_idm)
def test_constructor(self):
inode_no = self.tmp_file_a.stat().st_ino
vault_file_key_path = VFK(T.Path("a"), inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path
# Test source and path
self.assertEqual(VaultFile(vault=self.vault, branch=Branch.Keep,
path=self.tmp_file_a).path, vault_file_path)
self.assertEqual(VaultFile(vault=self.vault, branch=Branch.Keep,
path=self.tmp_file_a).source, self.tmp_file_a)
def test_constructor(self):
# Test Location
self.assertEqual(self.vault.location, self._path /
T.Path("parent_dir/child_dir_one/.vault"))
# Test Ownerships
self.assertEqual(next(self.vault.owners), 1)
self.assertEqual(self.vault.group, self.child_dir_one.stat().st_gid)
# Test Branch Creation
self.assertTrue(os.path.isdir(
self._path / T.Path("parent_dir/child_dir_one/.vault/keep")))
self.assertTrue(os.path.isdir(
self._path / T.Path("parent_dir/child_dir_one/.vault/archive")))
self.assertTrue(os.path.isdir(
self._path / T.Path("parent_dir/child_dir_one/.vault/.staged")))
def test_valid_root(self):
vault = DummyVault()
with self.assertRaises(exception.InvalidRoot):
vault.root = T.Path("foo")
with self.assertRaises(exception.InvalidRoot):
vault.root = T.Path("/foo")
vault.root = T.Path("/")
self.assertEqual(vault.root, T.Path("/"))
with self.assertRaises(exception.RootIsImmutable):
vault.root = T.Path("/tmp")
class VaultFileKey(os.PathLike):
""" HGI vault file key properties """
# NOTE This is implemented in a separate class to keep that part of
# the logic outside VaultFile and to decouple it from the filesystem
_delimiter: T.ClassVar[str] = "-"
_prefix: T.Optional[T.Path] # inode prefix path, without the LSB
_suffix: str # LSB and encoded basename suffix name
def __init__(self,
path: T.Path,
inode: T.Optional[int] = None,
max_file_name_length: int = _default_max_name_length) -> None:
"""
Construct the key from a path and (optional) inode
@param path Path to construct from
@param inode inode ID to construct from (defaults to inode
of path)
@param max_file_name The maximum length of a filename. Defaults to
current directory, however can be passed
in for each path being added to the Vault
"""
# Use the path's inode, if one is not explicitly provided
if inode is None:
inode = file.inode_id(path)
# The byte-padded hexadecimal representation of the inode ID
if len(inode_hex := f"{inode:x}") % 2:
inode_hex = f"0{inode_hex}"
# Chunk the inode ID into 8-bit segments
chunks = [inode_hex[i:i + 2] for i in range(0, len(inode_hex), 2)]
# inode ID, without the least significant byte, if it exists
self._prefix = None
if len(chunks) > 1:
self._prefix = T.Path(*chunks[:-1])
# inode ID LSB, delimiter, and the base64 encoding of the path.
# If the relative file path is too long, we split it by max file name length
# and save each part as a directory until we get to a final file
encoded_path = base64.encode(path)
max_file_name_length -= 3
self._suffix = chunks[-1] + self._delimiter + str(
T.Path(*[
encoded_path[i:i + max_file_name_length]
for i in range(0, len(encoded_path), max_file_name_length)
]))
def test_change_location_of_vaulted_file_outside(self):
self.new_location_tmp_file_a = self.child_dir_two / "new_location_tmp_file_a"
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no_old = self.tmp_file_a.stat().st_ino
vault_file_key_path_old = VFK(T.Path("a"), inode_no_old).path
vault_file_path_old = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path_old
self.assertTrue(os.path.isfile(vault_file_path_old))
shutil.move(self.tmp_file_a, self.new_location_tmp_file_a)
self.assertRaises(exception.IncorrectVault, self.vault.remove,
Branch.Keep, self.new_location_tmp_file_a)
def test_add_already_existing(self):
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no = self.tmp_file_a.stat().st_ino
vault_file_key_path = VFK(T.Path("a"), inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path
self.assertTrue(os.path.isfile(vault_file_path))
# Add again
self.vault.add(Branch.Keep, self.tmp_file_a)
inode_no = self.tmp_file_a.stat().st_ino
vault_file_key_path = VFK(T.Path("a"), inode_no).path
vault_file_path = self._path / \
T.Path("parent_dir/child_dir_one/.vault/keep") / \
vault_file_key_path
self.assertTrue(os.path.isfile(vault_file_path))
def setUp(self) -> None:
self._tmp = TemporaryDirectory()
self._path = T.Path(self._tmp.name).resolve()
self._path.chmod(0o770)
Vault._find_root = lambda *_: self._path
Vault(self._path,
idm=DummyIDM(config,
num_grp_owners=int(config.min_group_owners)))
def test_umask(self):
tmp = T.Path(self._tmp.name)
with umask(0):
(zero := tmp / "zero").touch(S_IRWXA)
self.assertEqual(zero.stat().st_mode & S_IRWXA, S_IRWXA)
with umask(stat.S_IRWXG | stat.S_IRWXO):
(user := tmp / "user").touch(S_IRWXA)
self.assertEqual(user.stat().st_mode & S_IRWXA, stat.S_IRWXU)
def FromDBRecord(cls, record: T.NamedTuple,
idm: idm.base.IdentityManager) -> File:
""" Construct from database record """
file = cls(
device=record.device,
inode=record.inode,
path=T.Path(record.path),
key=T.Path(record.key) if record.key is not None else None,
mtime=time.to_utc(record.mtime),
# The db only records mtime, so we set a and c time to mtime
# for now.
atime=time.to_utc(record.mtime),
ctime=time.to_utc(record.mtime),
owner=idm.user(uid=record.owner),
group=idm.group(gid=record.group_id),
size=record.size)
file.db_id = record.id
return file
def _find_root(relative_to: T.Path) -> T.Path:
"""
The vault's location is the root of the homogroupic subtree that
contains relative_to; that's where we start and traverse up
"""
relative_to = relative_to.resolve()
root = relative_to.parent if not relative_to.is_dir() else relative_to
while root != T.Path("/") and root.group() == root.parent.group():
root = root.parent
return root
def test_add_long(self):
"""
A new directory tree (this/path/is/going/..) is added here, to test the case of long relative paths.
+- tmp
+- parent/
+- child_dir_one
+- a
+- b
+ this/
+ path/
+ ....
+-.vault/
+- keep/
+- archive/
...
+- child_dir_two
+- c
"""
# File with really long relative path
dummy_long = T.Path('this/path/is/going/to/be/much/much/much/much/much'
'/much/much/much/much/much/much/much/much/much/much/much/much/much'
'/much/much/much/much/much/much/much/much/much/much/much/much/much'
'/much/much/much/much/much/much/much/much/much/much/much/much/much'
'/longer/than/two/hundred/and/fifty/five/characters')
# child_dir_one is the root of our vault
self.long_subdirectory = self.child_dir_one / dummy_long
self.long_subdirectory.mkdir(parents=True, exist_ok=True)
self.tmp_file_d = self.long_subdirectory / "d"
self.tmp_file_d.touch()
# Subdirectories are made rwx for user so that os.walk is able to read
# into it.
for dirpath, dirname, filenames in os.walk(self.parent_dir):
for momo in dirname:
dname = T.Path(os.path.join(dirpath, momo))
dname.chmod(0o730)
for filename in filenames:
fname = T.Path(os.path.join(dirpath, filename))
fname.chmod(0o777)
self.vault.add(Branch.Limbo, self.tmp_file_d)
def test_keep_files_symlink(self, mock_add, mock_remove):
self._tmp = TemporaryDirectory()
path = T.Path(self._tmp.name).resolve()
# Form a directory hierarchy
filepath = path / "a"
symlink = path / "b"
filepath.touch()
os.symlink(filepath, symlink)
main(["__init__", "keep", str(symlink)])
mock_add.assert_called_with(Branch.Keep, [filepath])
mock_remove.assert_not_called()
self._tmp.cleanup()
def _preexisting(self, branch: Branch,
key: VaultFileKey) -> T.Optional[VaultFileKey]:
"""
Return an pre-existing key, if one exists, in the given branch
@param branch Branch to search
@param key Key to match
@return Pre-existing key (None, if not found)
"""
key_base, key_glob = key.search_criteria
search_base = branch_base = self.vault.location / branch
if key_base is not None:
search_base = search_base / key_base
try:
alt_suffix, *others = (
T.Path(dirname, f)
for dirname, _, subfiles in os.walk(search_base)
for f in subfiles
if fnmatch.fnmatch(T.Path(dirname, f), key_glob))
except ValueError:
# Alternate not found
return None
if len(others) != 0:
# If the glob finds multiple matches, that's bad!
raise VaultExc.VaultCorruption(
f"The vault in {self.vault.root} contains duplicates of {key.path} in the {branch} branch"
)
alternate = T.Path(alt_suffix)
if key_base is not None:
alternate = key_base / alternate
# The VFK must be relative to the branch
return VaultFileKey.Reconstruct(alternate.relative_to(branch_base))
