def test_containment(self): vault = DummyVault() with TemporaryDirectory() as root: # Set up vault like so: /path/to/tmp/${_DUMMY_VAULT}/ # + foo/ # | + foo # + bar/ # + bar root = T.Path(root) for branch in DummyBranch: bpath = branch.value path = root / _DUMMY_VAULT / bpath path.mkdir(parents=True) filename = path / bpath filename.touch() vault.root = root for branch in DummyBranch: bpath = branch.value self.assertEqual(vault.branch(bpath), branch) self.assertTrue(bpath in vault) not_in_vault = T.Path("path/to/nowhere") self.assertIsNone(vault.branch(not_in_vault)) self.assertFalse(not_in_vault in vault)
def test_sibling_to_work_dir(self): work_dir = T.Path("this/is/my/path") vault_relative_path = T.Path("this/is/my/file3") expected = T.Path("../file3") work_dir_rel = relativise(vault_relative_path, work_dir) self.assertEqual(expected, work_dir_rel)
def test_child_to_work_dir(self): work_dir = T.Path("some/path") vault_relative_path = T.Path("some/path/file1") expected = T.Path("file1") work_dir_rel = relativise(vault_relative_path, work_dir) self.assertEqual(expected, work_dir_rel)
class Branch(core.vault.base.Branch): """ HGI vault branches """ Keep = T.Path("keep") Archive = T.Path("archive") Staged = T.Path(".staged") Limbo = T.Path(".limbo") Stash = T.Path(".stash")
def test_constructor(self): self.assertEqual(VFK(_DUMMY, 0x1).path, T.Path(f"01-{_B64_DUMMY}")) self.assertEqual(VFK(_DUMMY, 0x12).path, T.Path(f"12-{_B64_DUMMY}")) self.assertEqual(VFK(_DUMMY, 0x123).path, T.Path(f"01/23-{_B64_DUMMY}")) self.assertEqual(VFK(_DUMMY, 0x1234).path, T.Path(f"12/34-{_B64_DUMMY}"))
def test_change_location_of_vaulted_file(self): self.child_of_child_dir_one = self.child_dir_one / "child_of_child_dir_one" self.child_of_child_dir_one.mkdir() self.child_of_child_dir_one.chmod(0o330) self.new_location_tmp_file_a = self.child_of_child_dir_one / "new_location_tmp_file_a" self.vault.add(Branch.Keep, self.tmp_file_a) inode_no_old = self.tmp_file_a.stat().st_ino vault_file_key_path_old = VFK(T.Path("a"), inode_no_old).path vault_file_path_old = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path_old self.assertTrue(os.path.isfile(vault_file_path_old)) shutil.move(self.tmp_file_a, self.new_location_tmp_file_a) self.vault.add(Branch.Keep, self.new_location_tmp_file_a) inode_no = self.new_location_tmp_file_a.stat().st_ino vault_file_key_path = VFK( T.Path("child_of_child_dir_one") / "new_location_tmp_file_a", inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path self.assertTrue(os.path.isfile(vault_file_path)) self.assertFalse(os.path.isfile(vault_file_path_old))
def test_child_to_work_dir(self): work_dir = T.Path("some/path") work_dir_rel = T.Path("file1") vault_path = T.Path("/this/is/vault/root") vault_relative_path = derelativise(work_dir_rel, work_dir, vault_path) expected = T.Path("some/path/file1") self.assertEqual(expected, vault_relative_path)
def test_sibling_to_work_dir(self): work_dir = T.Path("this/is/my/path") work_dir_rel = T.Path("../file3") vault_path = T.Path("/this/is/vault/root") vault_relative_path = derelativise(work_dir_rel, work_dir, vault_path) expected = T.Path("this/is/my/file3") self.assertEqual(expected, vault_relative_path)
def test_archive_stash_fofn(self, mock_add, mock_remove, mock_file): main(["__init__", "archive", "--stash", "--fofn", "mock_file"]) args = mock_add.call_args.args files = list(args[1]) branch = args[0] self.assertEqual(files, [T.Path("/file1"), T.Path("/file2")]) self.assertEqual(branch, Branch.Stash) mock_remove.assert_not_called()
def test_list(self): self.vault.add(Branch.Keep, self.tmp_file_a) inode_no = self.tmp_file_a.stat().st_ino vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ VFK(T.Path("a"), inode_no).path self.assertEqual(next(self.vault.list(Branch.Keep)), (self.tmp_file_a, vault_file_path))
def test_reconstructor_long(self): self.assertEqual(VFK_k(T.Path( f"01-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG) self.assertEqual(VFK_k(T.Path( f"12-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG) self.assertEqual(VFK_k(T.Path( f"01/23-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG) self.assertEqual(VFK_k(T.Path( f"12/34-{_B64_DUMMY_LONG_FIRST_PART}/{_B64_DUMMY_LONG_SECOND_PART}")).source, _DUMMY_LONG)
def test_remove_not_existing_file(self): inode_no = self.tmp_file_b.stat().st_ino vault_file_key_path = VFK(T.Path("a"), inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ VFK(T.Path("a"), inode_no).path self.assertFalse(os.path.isfile(vault_file_path)) self.vault.remove(Branch.Keep, self.tmp_file_a) self.assertFalse(os.path.isfile(vault_file_path))
def test_constructor_longest(self): self.assertEqual(VFK(_DUMMY_LONGEST, 0x1).path, T.Path( f"01-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}")) self.assertEqual(VFK(_DUMMY_LONGEST, 0x12).path, T.Path( f"12-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}")) self.assertEqual(VFK(_DUMMY_LONGEST, 0x123).path, T.Path( f"01/23-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}")) self.assertEqual(VFK(_DUMMY_LONGEST, 0x1234).path, T.Path( f"12/34-{_B64_DUMMY_LONGEST_FIRST_PART}/{_B64_DUMMY_LONGEST_SECOND_PART}/{_B64_DUMMY_LONGEST_THIRD_PART}"))
def setUp(self) -> None: """ The following tests will emulate the following directory structure +- tmp +- parent/ +- child_dir_one +- a +- b +-.vault/ +- keep +- archive ... +- child_dir_two +- c """ _dummy_idm = DummyIDM(config) self._tmp = TemporaryDirectory() self._path = path = T.Path(self._tmp.name).resolve() # Form a directory hierarchy self.parent_dir = path / "parent_dir" self.child_dir_one = self.parent_dir / "child_dir_one" self.child_dir_two = self.parent_dir / "child_dir_two" self.tmp_file_a = self.child_dir_one / "a" self.tmp_file_b = self.child_dir_one / "b" self.tmp_file_c = self.child_dir_two / "c" self.child_dir_one.mkdir(parents=True, exist_ok=True) self.child_dir_two.mkdir(parents=True, exist_ok=True) self.tmp_file_a.touch() self.tmp_file_b.touch() self.tmp_file_c.touch() # The following conditions should be checked upfront for each file and, if not satisfied, that action should fail for that file, logged appropriately: # Check that the permissions of the file are at least ug+rw; 660+ # Check that the user and group permissions of the file are equal;66* or 77* # Check that the file's parent directory permissions are at least # ug+wx. 330+ # Default file permissions can be unsuitable for archiving, like 644 # (rw-r--r--, where owner and group dont have same permissions. self.tmp_file_a.chmod(0o660) # rw, rw, _ self.tmp_file_b.chmod(0o644) # rw, r, r self.tmp_file_c.chmod(0o777) # rwx, rwx, rwx # Default parent dir permissions can be unsuitable for archiving, like # 755 - write permissions are missing. self.child_dir_one.chmod(0o730) # wx, wx, _ self.parent_dir.chmod(0o777) # rwx, rwx, rwx Vault._find_root = MagicMock( return_value=self._path / T.Path("parent_dir/child_dir_one")) self.vault = Vault(relative_to=self._path / T.Path("parent_dir/child_dir_one/a"), idm=_dummy_idm)
def list(self, branch: Branch) -> T.Iterator[T.Tuple[T.Path, T.Path]]: # NOTE The order in which the listing is generated is # unspecified (I suspect it will be by inode ID); it is up to # downstream to modify this, as required bpath = self.location / branch return ((self.root / VaultFileKey.Reconstruct( T.Path(dirname, file).relative_to(bpath)).source, T.Path(dirname, file)) for dirname, _, files in os.walk(bpath) for file in files)
def test_add(self): # Add child_dir_one/tmp_file_b to vault and check whether hard link # exists at desired location. self.vault.add(Branch.Keep, self.tmp_file_a) inode_no = self.tmp_file_a.stat().st_ino vault_file_key_path = VFK(T.Path("a"), inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path self.assertTrue(os.path.isfile(vault_file_path))
def setUp(self) -> None: """ The following tests will emulate the following directory structure +- tmp +- parent/ +- child_dir_one +- a +- b +- perms_mod +- perms_mod_dir +- d +-.vault/ +- keep +- archive ... +- child_dir_two +- c """ _dummy_idm = DummyIDM(config) self._tmp = TemporaryDirectory() self._path = path = T.Path(self._tmp.name).resolve() # Form a directory hierarchy self.parent_dir = path / "parent_dir" self.child_dir_one = self.parent_dir / "child_dir_one" self.child_dir_two = self.parent_dir / "child_dir_two" self.tmp_file_a = self.child_dir_one / "a" self.tmp_file_b = self.child_dir_one / "b" self.tmp_file_c = self.child_dir_two / "c" self.perms_mod = self.child_dir_one / "perms_mod" self.perms_mod_dir = self.child_dir_one / "perms_mod_dir" self.tmp_file_d = self.perms_mod_dir / "d" self.child_dir_one.mkdir(parents=True, exist_ok=True) self.child_dir_two.mkdir(parents=True, exist_ok=True) self.perms_mod_dir.mkdir(parents=True, exist_ok=True) self.tmp_file_a.touch() self.tmp_file_b.touch() self.tmp_file_c.touch() self.perms_mod.touch() self.tmp_file_d.touch() # The permissions of the file ought to be least ug+rw; 660+ # The user and group permissions of the file are equal;66* or 77* # Thefile's parent directory permissions are at least ug+wx. 330+ self.tmp_file_a.chmod(0o660) self.tmp_file_b.chmod(0o644) self.tmp_file_c.chmod(0o777) self.child_dir_one.chmod(0o330) self.parent_dir.chmod(0o777) self.perms_mod_dir.chmod(0o777) self.tmp_file_d.chmod(0o664) Vault._find_root = MagicMock( return_value=self._path / T.Path("parent_dir/child_dir_one")) self.vault = Vault(relative_to=self._path / T.Path("parent_dir/child_dir_one/a"), idm=_dummy_idm)
def test_constructor(self): inode_no = self.tmp_file_a.stat().st_ino vault_file_key_path = VFK(T.Path("a"), inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path # Test source and path self.assertEqual(VaultFile(vault=self.vault, branch=Branch.Keep, path=self.tmp_file_a).path, vault_file_path) self.assertEqual(VaultFile(vault=self.vault, branch=Branch.Keep, path=self.tmp_file_a).source, self.tmp_file_a)
def test_constructor(self): # Test Location self.assertEqual(self.vault.location, self._path / T.Path("parent_dir/child_dir_one/.vault")) # Test Ownerships self.assertEqual(next(self.vault.owners), 1) self.assertEqual(self.vault.group, self.child_dir_one.stat().st_gid) # Test Branch Creation self.assertTrue(os.path.isdir( self._path / T.Path("parent_dir/child_dir_one/.vault/keep"))) self.assertTrue(os.path.isdir( self._path / T.Path("parent_dir/child_dir_one/.vault/archive"))) self.assertTrue(os.path.isdir( self._path / T.Path("parent_dir/child_dir_one/.vault/.staged")))
def test_valid_root(self): vault = DummyVault() with self.assertRaises(exception.InvalidRoot): vault.root = T.Path("foo") with self.assertRaises(exception.InvalidRoot): vault.root = T.Path("/foo") vault.root = T.Path("/") self.assertEqual(vault.root, T.Path("/")) with self.assertRaises(exception.RootIsImmutable): vault.root = T.Path("/tmp")
class VaultFileKey(os.PathLike): """ HGI vault file key properties """ # NOTE This is implemented in a separate class to keep that part of # the logic outside VaultFile and to decouple it from the filesystem _delimiter: T.ClassVar[str] = "-" _prefix: T.Optional[T.Path] # inode prefix path, without the LSB _suffix: str # LSB and encoded basename suffix name def __init__(self, path: T.Path, inode: T.Optional[int] = None, max_file_name_length: int = _default_max_name_length) -> None: """ Construct the key from a path and (optional) inode @param path Path to construct from @param inode inode ID to construct from (defaults to inode of path) @param max_file_name The maximum length of a filename. Defaults to current directory, however can be passed in for each path being added to the Vault """ # Use the path's inode, if one is not explicitly provided if inode is None: inode = file.inode_id(path) # The byte-padded hexadecimal representation of the inode ID if len(inode_hex := f"{inode:x}") % 2: inode_hex = f"0{inode_hex}" # Chunk the inode ID into 8-bit segments chunks = [inode_hex[i:i + 2] for i in range(0, len(inode_hex), 2)] # inode ID, without the least significant byte, if it exists self._prefix = None if len(chunks) > 1: self._prefix = T.Path(*chunks[:-1]) # inode ID LSB, delimiter, and the base64 encoding of the path. # If the relative file path is too long, we split it by max file name length # and save each part as a directory until we get to a final file encoded_path = base64.encode(path) max_file_name_length -= 3 self._suffix = chunks[-1] + self._delimiter + str( T.Path(*[ encoded_path[i:i + max_file_name_length] for i in range(0, len(encoded_path), max_file_name_length) ]))
def test_change_location_of_vaulted_file_outside(self): self.new_location_tmp_file_a = self.child_dir_two / "new_location_tmp_file_a" self.vault.add(Branch.Keep, self.tmp_file_a) inode_no_old = self.tmp_file_a.stat().st_ino vault_file_key_path_old = VFK(T.Path("a"), inode_no_old).path vault_file_path_old = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path_old self.assertTrue(os.path.isfile(vault_file_path_old)) shutil.move(self.tmp_file_a, self.new_location_tmp_file_a) self.assertRaises(exception.IncorrectVault, self.vault.remove, Branch.Keep, self.new_location_tmp_file_a)
def test_add_already_existing(self): self.vault.add(Branch.Keep, self.tmp_file_a) inode_no = self.tmp_file_a.stat().st_ino vault_file_key_path = VFK(T.Path("a"), inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path self.assertTrue(os.path.isfile(vault_file_path)) # Add again self.vault.add(Branch.Keep, self.tmp_file_a) inode_no = self.tmp_file_a.stat().st_ino vault_file_key_path = VFK(T.Path("a"), inode_no).path vault_file_path = self._path / \ T.Path("parent_dir/child_dir_one/.vault/keep") / \ vault_file_key_path self.assertTrue(os.path.isfile(vault_file_path))
def setUp(self) -> None: self._tmp = TemporaryDirectory() self._path = T.Path(self._tmp.name).resolve() self._path.chmod(0o770) Vault._find_root = lambda *_: self._path Vault(self._path, idm=DummyIDM(config, num_grp_owners=int(config.min_group_owners)))
def test_umask(self): tmp = T.Path(self._tmp.name) with umask(0): (zero := tmp / "zero").touch(S_IRWXA) self.assertEqual(zero.stat().st_mode & S_IRWXA, S_IRWXA) with umask(stat.S_IRWXG | stat.S_IRWXO): (user := tmp / "user").touch(S_IRWXA) self.assertEqual(user.stat().st_mode & S_IRWXA, stat.S_IRWXU)
def FromDBRecord(cls, record: T.NamedTuple, idm: idm.base.IdentityManager) -> File: """ Construct from database record """ file = cls( device=record.device, inode=record.inode, path=T.Path(record.path), key=T.Path(record.key) if record.key is not None else None, mtime=time.to_utc(record.mtime), # The db only records mtime, so we set a and c time to mtime # for now. atime=time.to_utc(record.mtime), ctime=time.to_utc(record.mtime), owner=idm.user(uid=record.owner), group=idm.group(gid=record.group_id), size=record.size) file.db_id = record.id return file
def _find_root(relative_to: T.Path) -> T.Path: """ The vault's location is the root of the homogroupic subtree that contains relative_to; that's where we start and traverse up """ relative_to = relative_to.resolve() root = relative_to.parent if not relative_to.is_dir() else relative_to while root != T.Path("/") and root.group() == root.parent.group(): root = root.parent return root
def test_add_long(self): """ A new directory tree (this/path/is/going/..) is added here, to test the case of long relative paths. +- tmp +- parent/ +- child_dir_one +- a +- b + this/ + path/ + .... +-.vault/ +- keep/ +- archive/ ... +- child_dir_two +- c """ # File with really long relative path dummy_long = T.Path('this/path/is/going/to/be/much/much/much/much/much' '/much/much/much/much/much/much/much/much/much/much/much/much/much' '/much/much/much/much/much/much/much/much/much/much/much/much/much' '/much/much/much/much/much/much/much/much/much/much/much/much/much' '/longer/than/two/hundred/and/fifty/five/characters') # child_dir_one is the root of our vault self.long_subdirectory = self.child_dir_one / dummy_long self.long_subdirectory.mkdir(parents=True, exist_ok=True) self.tmp_file_d = self.long_subdirectory / "d" self.tmp_file_d.touch() # Subdirectories are made rwx for user so that os.walk is able to read # into it. for dirpath, dirname, filenames in os.walk(self.parent_dir): for momo in dirname: dname = T.Path(os.path.join(dirpath, momo)) dname.chmod(0o730) for filename in filenames: fname = T.Path(os.path.join(dirpath, filename)) fname.chmod(0o777) self.vault.add(Branch.Limbo, self.tmp_file_d)
def test_keep_files_symlink(self, mock_add, mock_remove): self._tmp = TemporaryDirectory() path = T.Path(self._tmp.name).resolve() # Form a directory hierarchy filepath = path / "a" symlink = path / "b" filepath.touch() os.symlink(filepath, symlink) main(["__init__", "keep", str(symlink)]) mock_add.assert_called_with(Branch.Keep, [filepath]) mock_remove.assert_not_called() self._tmp.cleanup()
def _preexisting(self, branch: Branch, key: VaultFileKey) -> T.Optional[VaultFileKey]: """ Return an pre-existing key, if one exists, in the given branch @param branch Branch to search @param key Key to match @return Pre-existing key (None, if not found) """ key_base, key_glob = key.search_criteria search_base = branch_base = self.vault.location / branch if key_base is not None: search_base = search_base / key_base try: alt_suffix, *others = ( T.Path(dirname, f) for dirname, _, subfiles in os.walk(search_base) for f in subfiles if fnmatch.fnmatch(T.Path(dirname, f), key_glob)) except ValueError: # Alternate not found return None if len(others) != 0: # If the glob finds multiple matches, that's bad! raise VaultExc.VaultCorruption( f"The vault in {self.vault.root} contains duplicates of {key.path} in the {branch} branch" ) alternate = T.Path(alt_suffix) if key_base is not None: alternate = key_base / alternate # The VFK must be relative to the branch return VaultFileKey.Reconstruct(alternate.relative_to(branch_base))