def test_farmfs_blob_broken(tmp_path, capsys): root = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r1 == 0 a = Path('a', root) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r2 = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r2 == 0 a_blob = a.readlink() a_blob.unlink() r3 = farmfs_ui(['fsck', '--broken'], root) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\ta\n" assert captured.err == '' assert r3 == 1 # Test relative pathing. d = Path('d', root) d.mkdir() r4 = farmfs_ui(['fsck', '--broken'], d) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\t../a\n" assert captured.err == '' assert r3 == 1
def test_unlink_clean(tmp_path): tmp = Path(str(tmp_path)) d1 = tmp.join("d1") d1.mkdir() d2 = d1.join("d2") d2.mkdir() d3 = d2.join("d3") d3.mkdir() f1 = d3.join("f1") f2 = d3.join("f2") with f1.open("w") as fd: fd.write("f1") with f2.open("w") as fd: fd.write("f2") assert d1.exists() assert d2.exists() assert d3.exists() assert f1.exists() assert f2.exists() f2.unlink(clean=d2) assert d1.exists() assert d2.exists() assert d3.exists() assert f1.exists() assert not f2.exists() f1.unlink(clean=d2) assert d1.exists() assert d2.exists() assert not d3.exists() assert not f1.exists() assert not f2.exists()
def __init__(self, root): assert isinstance(root, Path) self.root = root self.mdd = _metadata_path(root) self.keydb = KeyDB(_keys_path(root)) self.udd = Path(self.keydb.read('udd')) self.bs = FileBlobstore(self.udd) self.snapdb = KeyDBFactory(KeyDBWindow("snaps", self.keydb), encode_snapshot, partial(decode_snapshot, self.bs.reverser)) self.remotedb = KeyDBFactory(KeyDBWindow("remotes", self.keydb), encode_volume, decode_volume) exclude_file = Path('.farmignore', self.root) ignored = [safetype(self.mdd)] try: with exclude_file.open('rb') as exclude_fd: for raw_pattern in exclude_fd.readlines(): pattern = ingest(raw_pattern.strip()) excluded = safetype(Path(pattern, root)) ignored.append(excluded) except IOError as e: if e.errno == NoSuchFile: pass else: raise e self.is_ignored = partial(skip_ignored, ignored)
def test_ensure_symlink(tmp_path): tmp = Path(str(tmp_path)) dne = tmp.join("dne") sb = tmp.join("sb") ensure_symlink(sb, dne) assert sb.exists() assert sb.islink() assert sb.readlink() == dne
def __cmp__(self, other): assert other is None or isinstance(other, SnapshotItem) if other is None: return -1 # Legacy snaps have leading '/' and modern ones are realative to ROOT. # Adding a './' before allows us to work around th issue. self_path = Path("./" + self._path, ROOT) other_path = Path("./" + other._path, ROOT) return self_path.__cmp__(other_path)
class KeyDBWrapper: def __init__(self, root): self.root = Path(root) def __enter__(self): ensure_absent(self.root) self.root.mkdir() return KeyDB(self.root) def __exit__(self, type, value, traceback): ensure_absent(self.root)
def test_ensure_absent(tmp_path): # Test broken symlink tmp = Path(str(tmp_path)) dne = tmp.join("dne") sb = tmp.join("sb") ensure_symlink(sb, dne) assert sb.islink() ensure_absent(sb) assert not sb.islink() # Test link is removed, but underlying file is left intact. tmp = Path(str(tmp_path)) f = tmp.join("dne") with f.open('w') as fd: fd.write('hi') sb = tmp.join("sb") ensure_symlink(sb, f) assert sb.islink() assert f.isfile() ensure_absent(sb) assert not sb.islink() assert f.isfile() # Test dir d = tmp.join('d') f1 = d.join('f1') f2 = d.join('f2') d.mkdir() with f1.open('w') as fd: fd.write('f1') with f2.open('w') as fd: fd.write('f2') assert d.exists() and d.isdir() ensure_absent(d) assert not d.exists() and not d.isdir()
class KeyDBWrapper: def __init__(self, datadir): self.root = Path(datadir) def __enter__(self): ensure_absent(self.root) self.root.mkdir() return KeyDB(self.root) def __exit__(self, type, value, traceback): ensure_absent(self.root)
def test_ensure_copy(tmp_path): tmp = Path(str(tmp_path)) s = tmp.join('s') with s.open('w') as fd: fd.write('f') d = tmp.join('d') assert s.exists() and s.isfile() assert not d.exists() and not d.isfile() ensure_copy(d, s) assert s.exists() and s.isfile() assert d.exists() and d.isfile()
def produce_mismatches(segments): """ Helper function to produce pairs of paths which have lexographical/path order mismatches""" paths = list( filter( lambda p: search("//", p) is None, map( lambda p: sep + p, map(lambda s: reduce(lambda x, y: x + y, s), permutations(segments, len(segments)))))) combos = list(combinations(paths, 2)) is_mismatch = uncurry(lambda x, y: bool(x < y) != bool(Path(x) < Path(y))) mismatches = list(filter(is_mismatch, combos)) return mismatches
def test_exists(tmp_path): tmp = Path(str(tmp_path)) # Test a path that doesn't exist dne = tmp.join("dne") assert not dne.exists() # Test a regular file. f = tmp.join("f") with f.open("w") as fd: fd.write("some text") assert f.exists() # Test a directory d = tmp.join("d") d.mkdir() assert d.exists() # Test a symlink to a regular file f_slnk = tmp.join("f_lnk") f_slnk.symlink(f) assert f_slnk.exists() # Test a symlink to a directory. d_slnk = tmp.join("d_lnk") d_slnk.symlink(d) assert d_slnk.exists() # Test a broken symlink. b_slnk = tmp.join("b_lnk") b_slnk.symlink(dne) assert b_slnk.exists()
def test_farmfs_ignore_corruption(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 with vol.join(".farmignore").open("w") as ignore: ignore.write("a") r = farmfs_ui(['fsck', '--frozen-ignored'], vol) captured = capsys.readouterr() assert captured.out == 'Ignored file frozen a\n' assert captured.err == "" assert r == 4
def test_readlink(tmp_path): tmp = Path(str(tmp_path)) # Test a path that doesn't exist dne = tmp.join("dne") with pytest.raises(OSError) as e: dne.readlink() assert e.value.errno == FileDoesNotExist # Test a regular file. f = tmp.join("f") with f.open("w") as fd: fd.write("some text") with pytest.raises(OSError) as e: f.readlink() is None assert e.value.errno == InvalidArgument # Test a directory d = tmp.join("d") d.mkdir() with pytest.raises(OSError) as e: d.readlink() is None assert e.value.errno == InvalidArgument # Test a symlink to a regular file f_slnk = tmp.join("f_lnk") f_slnk.symlink(f) assert f_slnk.readlink() == f # Test a symlink to a directory. d_slnk = tmp.join("d_lnk") d_slnk.symlink(d) assert d_slnk.readlink() == d # Test a broken symlink. b_slnk = tmp.join("b_lnk") b_slnk.symlink(dne) assert b_slnk.readlink() == dne
def test_KeyDBFactory_copy(tmp_Path): with KeyDBWrapper(Path("db1", tmp_Path)) as db1: window1 = KeyDBWindow("window", db1) factory1 = KeyDBFactory(window1, str, lambda data, name: safetype(data)) assert factory1.list() == [] factory1.write("five", 5) with KeyDBWrapper(Path("db2", tmp_Path)) as db2: window2 = KeyDBWindow("other", db2) factory2 = KeyDBFactory(window2, str, lambda data, name: safetype(data)) factory2.copy("five", window1) value = factory2.read("five") assert value == safetype(5)
def test_ensure_dir(tmp_path): # Test dir already exists. tmp = Path(str(tmp_path)) d1 = tmp.join('d1') d1.mkdir() assert d1.isdir() ensure_dir(d1) assert d1.isdir() # Test dir creation d2 = tmp.join('d2') assert not d2.exists() ensure_dir(d2) assert d2.exists() and d2.isdir() # test removes file d3 = tmp.join('d3') dne = tmp.join('dne') d3.symlink(dne) assert d3.exists() and d3.islink() ensure_dir(d3) assert d3.exists() and d3.isdir() # test removes symlink d4 = tmp.join('d4') f = tmp.join('f') with f.open('w') as fd: fd.write('f') d4.symlink(f) assert d4.exists() and not d4.isdir() and d4.islink() ensure_dir(d4) assert d4.exists() and d4.isdir() and not d4.islink()
def test_farmfs_status(tmp_path, capsys): tmp = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], tmp) captured = capsys.readouterr() assert r1 == 0 a = Path('a', tmp) with a.open('w') as a_fd: a_fd.write('a') r2 = farmfs_ui(['status'], tmp) captured = capsys.readouterr() assert captured.out == "a\n" assert captured.err == "" assert r2 == 0 # Test relative status report. d = Path('d', tmp) d.mkdir() r3 = farmfs_ui(['status'], d) captured = capsys.readouterr() assert captured.out == "../a\n" assert captured.err == "" assert r3 == 0 # Freeze a r4 = farmfs_ui(['freeze'], tmp) captured = capsys.readouterr() assert r4 == 0 # assert captured.out == "" assert captured.err == "" r5 = farmfs_ui(['status'], tmp) captured = capsys.readouterr() assert captured.out == "" assert captured.err == "" assert r5 == 0
def test_checksum_non_files(tmp_path): tmp = Path(str(tmp_path)) # Test dir d = tmp.join("d") d.mkdir() with pytest.raises(IOError) as e: d.checksum() assert e.value.errno == IsADirectory # Test symlink to dir d_slnk = tmp.join("dslnk") d_slnk.symlink(d) with pytest.raises(IOError) as e: d_slnk.checksum() assert e.value.errno == IsADirectory # Setup files f = tmp.join("f") with f.open("w") as fd: fd.write("") # Test file symlink f_slnk = tmp.join("fslnk") f_slnk.symlink(f) f_slnk.checksum() == u"d41d8cd98f00b204e9800998ecf8427e" # Test broken symlink b_slnk = tmp.join("bslnk") b_slnk.symlink(tmp.join("dne")) with pytest.raises(IOError) as e: b_slnk.checksum() assert e.value.errno == FileDoesNotExist
def test_rename_dir(tmp_path): tmp = Path(str(tmp_path)) d = tmp.join("d") d.mkdir() d.join("inside").mkdir() d2 = tmp.join("d2") assert d.exists() assert d.isdir() assert not d2.exists() assert not d2.isdir() d.rename(d2) assert not d.exists() assert not d.isdir() assert d2.exists() assert d2.isdir()
def test_rename_file(tmp_path): tmp = Path(str(tmp_path)) f = tmp.join("f") with f.open("w") as fd: fd.write("some text") f2 = tmp.join("f2") assert f.exists() assert f.isfile() assert not f2.exists() assert not f2.isfile() f.rename(f2) assert not f.exists() assert not f.isfile() assert f2.exists() assert f2.isfile()
def test_farmfs_blob_permission(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 a_blob = a.readlink() a_blob.chmod(0o777) r = farmfs_ui(['fsck', '--blob-permissions'], vol) captured = capsys.readouterr() assert captured.out == 'writable blob: ' + a_csum + '\n' assert captured.err == "" assert r == 8
def test_rename_symlink(tmp_path): tmp = Path(str(tmp_path)) d = tmp.join("d") d.mkdir() s = tmp.join("s") s.symlink(d) s2 = tmp.join("s2") assert s.exists() assert s.islink() assert not s2.exists() assert not s2.islink() s.rename(s2) assert not s.exists() assert not s.islink() assert s2.exists() assert s2.islink()
def test_tree_diff_order(): name_a = u"/a/+b" name_b = u"/a+/b" path_a = Path(name_a) path_b = Path(name_b) link_a = makeLink(path_a, u"00000000000000000000000000000000") link_b = makeLink(path_b, u"00000000000000000000000000000000") left = KeySnapshot([link_a], u"left", None) right = KeySnapshot([link_b], u"right", None) diff = tree_diff(left, right) paths = list(map(lambda change: change.path(ROOT), diff)) assert paths == [path_a, path_b]
def test_farmfs_status(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') r = farmfs_ui(['status'], vol) captured = capsys.readouterr() assert captured.out == "a\n" assert captured.err == "" assert r == 0 # Test relative status report. d = Path('d', vol) d.mkdir() r = farmfs_ui(['status'], d) captured = capsys.readouterr() assert captured.out == "../a\n" assert captured.err == "" assert r == 0 # Freeze a r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 # assert captured.out == "" assert captured.err == "" r = farmfs_ui(['status'], vol) captured = capsys.readouterr() assert captured.out == "" assert captured.err == "" assert r == 0
def test_create_path(): p1 = Path("/") p2 = Path("/a") p2 = Path("/a/b") p3 = Path(p1) p4 = Path("a", p1) with pytest.raises(AssertionError): p5 = Path("/a/b", p2) with pytest.raises(ValueError): p6 = Path(None) with pytest.raises(ValueError): p7 = Path(None, p1) with pytest.raises(AssertionError): p8 = Path("a", "b")
def test_rewrite_links(tmp_path, capsys): tmp = Path(str(tmp_path)) vol1 = tmp.join("vol1") vol2 = tmp.join("vol2") a = Path('a', vol1) # Make the Farm r = farmfs_ui(['mkfs'], vol1) captured = capsys.readouterr() assert r == 0 # Make a with a.open('w') as fd: fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol1) captured = capsys.readouterr() assert r == 0 # Move from vol1 to vol2 vol1.rename(vol2) # Reinit the fs. This will fix the udd directory pointer. r = farmfs_ui(['mkfs'], vol2) captured = capsys.readouterr() assert r == 0 # Rewrite the links r = dbg_ui(['rewrite-links', '.'], vol2) captured = capsys.readouterr() vol2a = vol2.join('a') vol2a_blob = str(vol2a.readlink()) assert r == 0 assert captured.out == "Relinked a to " + vol2a_blob + "\n" assert captured.err == ""
def test_farmfs_blob_corruption(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 a_blob = a.readlink() a_blob.unlink() with a_blob.open('w') as a_fd: a_fd.write('b') ensure_readonly(a_blob) r = farmfs_ui(['fsck', '--checksums'], vol) captured = capsys.readouterr() assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n' assert captured.err == "" assert r == 2
def test_farmfs_similarity(vol, capsys): a_path = Path("a", vol) a_path.mkdir() b_path = Path("b", vol) b_path.mkdir() for i in [1, 2, 3]: with Path(str(i), a_path).open('w') as fd: fd.write(str(i)) for i in [1, 2, 4, 5]: with Path(str(i), b_path).open('w') as fd: fd.write(str(i)) # Freeze r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 r = farmfs_ui(['similarity', "a", "b"], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "left\tboth\tright\tjaccard_similarity\n1\t2\t2\t0.4\n"
def test_blob(vol, capsys): a = Path('a', vol) b = Path('b', vol) # Make a,b,b2; freeze, snap, delete with a.open('w') as fd: fd.write('a') a_csum = str(a.checksum()) with b.open('w') as fd: fd.write('b') b_csum = str(b.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 # get blob paths r = dbg_ui(['blob', a_csum, b_csum], vol) captured = capsys.readouterr() assert r == 0 a_rel = a.readlink().relative_to(vol) b_rel = b.readlink().relative_to(vol) assert captured.out == a_csum + " " + a_rel + "\n" + b_csum + " " + b_rel + "\n" assert captured.err == ""
def test_unlink(tmp_path): tmp = Path(str(tmp_path)) dne = tmp.join("dne") f = tmp.join("f") with f.open("w") as fd: fd.write("some text") d = tmp.join("d") d.mkdir() f_slnk = tmp.join("f_lnk") f_slnk.symlink(f) d_slnk = tmp.join("d_lnk") d_slnk.symlink(d) b_slnk = tmp.join("b_lnk") b_slnk.symlink(dne) # test unlink f_slnk assert f.exists() assert f_slnk.exists() f_slnk.unlink() assert not f_slnk.exists() assert f.exists() # test unlink d_slnk assert d.exists() assert d_slnk.exists() d_slnk.unlink() assert not d_slnk.exists() assert d.exists() # test unlink b_slnk assert b_slnk.exists() b_slnk.unlink() assert not b_slnk.exists() # test unlink dne assert not dne.exists() dne.unlink() assert not dne.exists() # test unlink file assert f.exists() f.unlink() assert not f.exists() # test unlink dir assert d.exists() with pytest.raises(OSError) as e: d.unlink() assert e.value.errno == NotPermitted assert d.exists()
def test_farmdbg_reverse(vol, capsys, a, b, c): a_path = Path(a, vol) with a_path.open('w') as a_fd: a_fd.write('a') a_csum = str(a_path.checksum()) bc_path = Path(b, vol).join(c) ensure_copy(bc_path, a_path) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() r = farmfs_ui(['snap', 'make', 'mysnap'], vol) assert r == 0 r = dbg_ui(['walk', 'root'], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == ".\tdir\t\n%s\tlink\t%s\n%s\tdir\t\n%s/%s\tlink\t%s\n" % ( a, a_csum, b, b, c, a_csum) assert captured.err == '' r = dbg_ui(['walk', 'userdata'], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == a_csum + '\n' assert captured.err == '' r = dbg_ui(['reverse', a_csum], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "<tree> " + a + "\n<tree> " + b + "/" + c + "\n" assert captured.err == '' r = dbg_ui(['reverse', '--all', a_csum], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "<tree> " + a + "\n<tree> " + b + "/" + c + "\nmysnap " + a + "\nmysnap " + b + "/" + c + "\n" assert captured.err == '' r = dbg_ui(['reverse', '--snap', 'mysnap', a_csum], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "mysnap " + a + "\nmysnap " + b + "/" + c + "\n" assert captured.err == ''
def test_ensure_file(tmp_path): tmp = Path(str(tmp_path)) # Test write file f1 = tmp.join('f1') assert not f1.exists() and not f1.isfile() with ensure_file(f1, 'w') as fd: fd.write('f') assert f1.exists() and f1.isfile() # Test write file in missing dir d2 = tmp.join('d2') f2 = d2.join(d2) assert not d2.exists() and not f2.exists() with ensure_file(f2, 'w') as fd: fd.write('f') assert d2.exists() and f2.exists() # Test write file when replacing a file with a dir. d3 = tmp.join('d3') with ensure_file(d3, 'w') as fd: fd.write('f') assert d3.isfile() f3 = d3.join(d3) with ensure_file(f3, 'w') as fd: fd.write('f') assert d3.exists() and d3.isdir() and f3.exists() and f3.isfile()
def __init__(self, root): self.root = Path(root)