def test_farmfs_blob_broken(tmp_path, capsys):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', root)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r2 = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r2 == 0
a_blob = a.readlink()
a_blob.unlink()
r3 = farmfs_ui(['fsck', '--broken'], root)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\ta\n"
assert captured.err == ''
assert r3 == 1
# Test relative pathing.
d = Path('d', root)
d.mkdir()
r4 = farmfs_ui(['fsck', '--broken'], d)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\t../a\n"
assert captured.err == ''
assert r3 == 1
def test_unlink_clean(tmp_path):
tmp = Path(str(tmp_path))
d1 = tmp.join("d1")
d1.mkdir()
d2 = d1.join("d2")
d2.mkdir()
d3 = d2.join("d3")
d3.mkdir()
f1 = d3.join("f1")
f2 = d3.join("f2")
with f1.open("w") as fd:
fd.write("f1")
with f2.open("w") as fd:
fd.write("f2")
assert d1.exists()
assert d2.exists()
assert d3.exists()
assert f1.exists()
assert f2.exists()
f2.unlink(clean=d2)
assert d1.exists()
assert d2.exists()
assert d3.exists()
assert f1.exists()
assert not f2.exists()
f1.unlink(clean=d2)
assert d1.exists()
assert d2.exists()
assert not d3.exists()
assert not f1.exists()
assert not f2.exists()
def __init__(self, root):
assert isinstance(root, Path)
self.root = root
self.mdd = _metadata_path(root)
self.keydb = KeyDB(_keys_path(root))
self.udd = Path(self.keydb.read('udd'))
self.bs = FileBlobstore(self.udd)
self.snapdb = KeyDBFactory(KeyDBWindow("snaps", self.keydb),
encode_snapshot,
partial(decode_snapshot, self.bs.reverser))
self.remotedb = KeyDBFactory(KeyDBWindow("remotes", self.keydb),
encode_volume, decode_volume)
exclude_file = Path('.farmignore', self.root)
ignored = [safetype(self.mdd)]
try:
with exclude_file.open('rb') as exclude_fd:
for raw_pattern in exclude_fd.readlines():
pattern = ingest(raw_pattern.strip())
excluded = safetype(Path(pattern, root))
ignored.append(excluded)
except IOError as e:
if e.errno == NoSuchFile:
pass
else:
raise e
self.is_ignored = partial(skip_ignored, ignored)
def test_ensure_symlink(tmp_path):
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
sb = tmp.join("sb")
ensure_symlink(sb, dne)
assert sb.exists()
assert sb.islink()
assert sb.readlink() == dne
def __cmp__(self, other):
assert other is None or isinstance(other, SnapshotItem)
if other is None:
return -1
# Legacy snaps have leading '/' and modern ones are realative to ROOT.
# Adding a './' before allows us to work around th issue.
self_path = Path("./" + self._path, ROOT)
other_path = Path("./" + other._path, ROOT)
return self_path.__cmp__(other_path)
class KeyDBWrapper:
def __init__(self, root):
self.root = Path(root)
def __enter__(self):
ensure_absent(self.root)
self.root.mkdir()
return KeyDB(self.root)
def __exit__(self, type, value, traceback):
ensure_absent(self.root)
def test_ensure_absent(tmp_path):
# Test broken symlink
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
sb = tmp.join("sb")
ensure_symlink(sb, dne)
assert sb.islink()
ensure_absent(sb)
assert not sb.islink()
# Test link is removed, but underlying file is left intact.
tmp = Path(str(tmp_path))
f = tmp.join("dne")
with f.open('w') as fd:
fd.write('hi')
sb = tmp.join("sb")
ensure_symlink(sb, f)
assert sb.islink()
assert f.isfile()
ensure_absent(sb)
assert not sb.islink()
assert f.isfile()
# Test dir
d = tmp.join('d')
f1 = d.join('f1')
f2 = d.join('f2')
d.mkdir()
with f1.open('w') as fd:
fd.write('f1')
with f2.open('w') as fd:
fd.write('f2')
assert d.exists() and d.isdir()
ensure_absent(d)
assert not d.exists() and not d.isdir()
class KeyDBWrapper:
def __init__(self, datadir):
self.root = Path(datadir)
def __enter__(self):
ensure_absent(self.root)
self.root.mkdir()
return KeyDB(self.root)
def __exit__(self, type, value, traceback):
ensure_absent(self.root)
def test_ensure_copy(tmp_path):
tmp = Path(str(tmp_path))
s = tmp.join('s')
with s.open('w') as fd:
fd.write('f')
d = tmp.join('d')
assert s.exists() and s.isfile()
assert not d.exists() and not d.isfile()
ensure_copy(d, s)
assert s.exists() and s.isfile()
assert d.exists() and d.isfile()
def produce_mismatches(segments):
""" Helper function to produce pairs of paths which have lexographical/path order mismatches"""
paths = list(
filter(
lambda p: search("//", p) is None,
map(
lambda p: sep + p,
map(lambda s: reduce(lambda x, y: x + y, s),
permutations(segments, len(segments))))))
combos = list(combinations(paths, 2))
is_mismatch = uncurry(lambda x, y: bool(x < y) != bool(Path(x) < Path(y)))
mismatches = list(filter(is_mismatch, combos))
return mismatches
def test_exists(tmp_path):
tmp = Path(str(tmp_path))
# Test a path that doesn't exist
dne = tmp.join("dne")
assert not dne.exists()
# Test a regular file.
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
assert f.exists()
# Test a directory
d = tmp.join("d")
d.mkdir()
assert d.exists()
# Test a symlink to a regular file
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
assert f_slnk.exists()
# Test a symlink to a directory.
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
assert d_slnk.exists()
# Test a broken symlink.
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
assert b_slnk.exists()
def test_farmfs_ignore_corruption(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
with vol.join(".farmignore").open("w") as ignore:
ignore.write("a")
r = farmfs_ui(['fsck', '--frozen-ignored'], vol)
captured = capsys.readouterr()
assert captured.out == 'Ignored file frozen a\n'
assert captured.err == ""
assert r == 4
def test_readlink(tmp_path):
tmp = Path(str(tmp_path))
# Test a path that doesn't exist
dne = tmp.join("dne")
with pytest.raises(OSError) as e:
dne.readlink()
assert e.value.errno == FileDoesNotExist
# Test a regular file.
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
with pytest.raises(OSError) as e:
f.readlink() is None
assert e.value.errno == InvalidArgument
# Test a directory
d = tmp.join("d")
d.mkdir()
with pytest.raises(OSError) as e:
d.readlink() is None
assert e.value.errno == InvalidArgument
# Test a symlink to a regular file
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
assert f_slnk.readlink() == f
# Test a symlink to a directory.
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
assert d_slnk.readlink() == d
# Test a broken symlink.
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
assert b_slnk.readlink() == dne
def test_KeyDBFactory_copy(tmp_Path):
with KeyDBWrapper(Path("db1", tmp_Path)) as db1:
window1 = KeyDBWindow("window", db1)
factory1 = KeyDBFactory(window1, str,
lambda data, name: safetype(data))
assert factory1.list() == []
factory1.write("five", 5)
with KeyDBWrapper(Path("db2", tmp_Path)) as db2:
window2 = KeyDBWindow("other", db2)
factory2 = KeyDBFactory(window2, str,
lambda data, name: safetype(data))
factory2.copy("five", window1)
value = factory2.read("five")
assert value == safetype(5)
def test_ensure_dir(tmp_path):
# Test dir already exists.
tmp = Path(str(tmp_path))
d1 = tmp.join('d1')
d1.mkdir()
assert d1.isdir()
ensure_dir(d1)
assert d1.isdir()
# Test dir creation
d2 = tmp.join('d2')
assert not d2.exists()
ensure_dir(d2)
assert d2.exists() and d2.isdir()
# test removes file
d3 = tmp.join('d3')
dne = tmp.join('dne')
d3.symlink(dne)
assert d3.exists() and d3.islink()
ensure_dir(d3)
assert d3.exists() and d3.isdir()
# test removes symlink
d4 = tmp.join('d4')
f = tmp.join('f')
with f.open('w') as fd:
fd.write('f')
d4.symlink(f)
assert d4.exists() and not d4.isdir() and d4.islink()
ensure_dir(d4)
assert d4.exists() and d4.isdir() and not d4.islink()
def test_farmfs_status(tmp_path, capsys):
tmp = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], tmp)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', tmp)
with a.open('w') as a_fd:
a_fd.write('a')
r2 = farmfs_ui(['status'], tmp)
captured = capsys.readouterr()
assert captured.out == "a\n"
assert captured.err == ""
assert r2 == 0
# Test relative status report.
d = Path('d', tmp)
d.mkdir()
r3 = farmfs_ui(['status'], d)
captured = capsys.readouterr()
assert captured.out == "../a\n"
assert captured.err == ""
assert r3 == 0
# Freeze a
r4 = farmfs_ui(['freeze'], tmp)
captured = capsys.readouterr()
assert r4 == 0
# assert captured.out == ""
assert captured.err == ""
r5 = farmfs_ui(['status'], tmp)
captured = capsys.readouterr()
assert captured.out == ""
assert captured.err == ""
assert r5 == 0
def test_checksum_non_files(tmp_path):
tmp = Path(str(tmp_path))
# Test dir
d = tmp.join("d")
d.mkdir()
with pytest.raises(IOError) as e:
d.checksum()
assert e.value.errno == IsADirectory
# Test symlink to dir
d_slnk = tmp.join("dslnk")
d_slnk.symlink(d)
with pytest.raises(IOError) as e:
d_slnk.checksum()
assert e.value.errno == IsADirectory
# Setup files
f = tmp.join("f")
with f.open("w") as fd:
fd.write("")
# Test file symlink
f_slnk = tmp.join("fslnk")
f_slnk.symlink(f)
f_slnk.checksum() == u"d41d8cd98f00b204e9800998ecf8427e"
# Test broken symlink
b_slnk = tmp.join("bslnk")
b_slnk.symlink(tmp.join("dne"))
with pytest.raises(IOError) as e:
b_slnk.checksum()
assert e.value.errno == FileDoesNotExist
def test_rename_dir(tmp_path):
tmp = Path(str(tmp_path))
d = tmp.join("d")
d.mkdir()
d.join("inside").mkdir()
d2 = tmp.join("d2")
assert d.exists()
assert d.isdir()
assert not d2.exists()
assert not d2.isdir()
d.rename(d2)
assert not d.exists()
assert not d.isdir()
assert d2.exists()
assert d2.isdir()
def test_rename_file(tmp_path):
tmp = Path(str(tmp_path))
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
f2 = tmp.join("f2")
assert f.exists()
assert f.isfile()
assert not f2.exists()
assert not f2.isfile()
f.rename(f2)
assert not f.exists()
assert not f.isfile()
assert f2.exists()
assert f2.isfile()
def test_farmfs_blob_permission(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
a_blob = a.readlink()
a_blob.chmod(0o777)
r = farmfs_ui(['fsck', '--blob-permissions'], vol)
captured = capsys.readouterr()
assert captured.out == 'writable blob: ' + a_csum + '\n'
assert captured.err == ""
assert r == 8
def test_rename_symlink(tmp_path):
tmp = Path(str(tmp_path))
d = tmp.join("d")
d.mkdir()
s = tmp.join("s")
s.symlink(d)
s2 = tmp.join("s2")
assert s.exists()
assert s.islink()
assert not s2.exists()
assert not s2.islink()
s.rename(s2)
assert not s.exists()
assert not s.islink()
assert s2.exists()
assert s2.islink()
def test_tree_diff_order():
name_a = u"/a/+b"
name_b = u"/a+/b"
path_a = Path(name_a)
path_b = Path(name_b)
link_a = makeLink(path_a, u"00000000000000000000000000000000")
link_b = makeLink(path_b, u"00000000000000000000000000000000")
left = KeySnapshot([link_a], u"left", None)
right = KeySnapshot([link_b], u"right", None)
diff = tree_diff(left, right)
paths = list(map(lambda change: change.path(ROOT), diff))
assert paths == [path_a, path_b]
def test_farmfs_status(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
r = farmfs_ui(['status'], vol)
captured = capsys.readouterr()
assert captured.out == "a\n"
assert captured.err == ""
assert r == 0
# Test relative status report.
d = Path('d', vol)
d.mkdir()
r = farmfs_ui(['status'], d)
captured = capsys.readouterr()
assert captured.out == "../a\n"
assert captured.err == ""
assert r == 0
# Freeze a
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
# assert captured.out == ""
assert captured.err == ""
r = farmfs_ui(['status'], vol)
captured = capsys.readouterr()
assert captured.out == ""
assert captured.err == ""
assert r == 0
def test_create_path():
p1 = Path("/")
p2 = Path("/a")
p2 = Path("/a/b")
p3 = Path(p1)
p4 = Path("a", p1)
with pytest.raises(AssertionError):
p5 = Path("/a/b", p2)
with pytest.raises(ValueError):
p6 = Path(None)
with pytest.raises(ValueError):
p7 = Path(None, p1)
with pytest.raises(AssertionError):
p8 = Path("a", "b")
def test_rewrite_links(tmp_path, capsys):
tmp = Path(str(tmp_path))
vol1 = tmp.join("vol1")
vol2 = tmp.join("vol2")
a = Path('a', vol1)
# Make the Farm
r = farmfs_ui(['mkfs'], vol1)
captured = capsys.readouterr()
assert r == 0
# Make a
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol1)
captured = capsys.readouterr()
assert r == 0
# Move from vol1 to vol2
vol1.rename(vol2)
# Reinit the fs. This will fix the udd directory pointer.
r = farmfs_ui(['mkfs'], vol2)
captured = capsys.readouterr()
assert r == 0
# Rewrite the links
r = dbg_ui(['rewrite-links', '.'], vol2)
captured = capsys.readouterr()
vol2a = vol2.join('a')
vol2a_blob = str(vol2a.readlink())
assert r == 0
assert captured.out == "Relinked a to " + vol2a_blob + "\n"
assert captured.err == ""
def test_farmfs_blob_corruption(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
a_blob = a.readlink()
a_blob.unlink()
with a_blob.open('w') as a_fd:
a_fd.write('b')
ensure_readonly(a_blob)
r = farmfs_ui(['fsck', '--checksums'], vol)
captured = capsys.readouterr()
assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n'
assert captured.err == ""
assert r == 2
def test_farmfs_similarity(vol, capsys):
a_path = Path("a", vol)
a_path.mkdir()
b_path = Path("b", vol)
b_path.mkdir()
for i in [1, 2, 3]:
with Path(str(i), a_path).open('w') as fd:
fd.write(str(i))
for i in [1, 2, 4, 5]:
with Path(str(i), b_path).open('w') as fd:
fd.write(str(i))
# Freeze
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
r = farmfs_ui(['similarity', "a", "b"], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "left\tboth\tright\tjaccard_similarity\n1\t2\t2\t0.4\n"
def test_blob(vol, capsys):
a = Path('a', vol)
b = Path('b', vol)
# Make a,b,b2; freeze, snap, delete
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
with b.open('w') as fd:
fd.write('b')
b_csum = str(b.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
# get blob paths
r = dbg_ui(['blob', a_csum, b_csum], vol)
captured = capsys.readouterr()
assert r == 0
a_rel = a.readlink().relative_to(vol)
b_rel = b.readlink().relative_to(vol)
assert captured.out == a_csum + " " + a_rel + "\n" + b_csum + " " + b_rel + "\n"
assert captured.err == ""
def test_unlink(tmp_path):
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
d = tmp.join("d")
d.mkdir()
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
# test unlink f_slnk
assert f.exists()
assert f_slnk.exists()
f_slnk.unlink()
assert not f_slnk.exists()
assert f.exists()
# test unlink d_slnk
assert d.exists()
assert d_slnk.exists()
d_slnk.unlink()
assert not d_slnk.exists()
assert d.exists()
# test unlink b_slnk
assert b_slnk.exists()
b_slnk.unlink()
assert not b_slnk.exists()
# test unlink dne
assert not dne.exists()
dne.unlink()
assert not dne.exists()
# test unlink file
assert f.exists()
f.unlink()
assert not f.exists()
# test unlink dir
assert d.exists()
with pytest.raises(OSError) as e:
d.unlink()
assert e.value.errno == NotPermitted
assert d.exists()
def test_farmdbg_reverse(vol, capsys, a, b, c):
a_path = Path(a, vol)
with a_path.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a_path.checksum())
bc_path = Path(b, vol).join(c)
ensure_copy(bc_path, a_path)
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
r = farmfs_ui(['snap', 'make', 'mysnap'], vol)
assert r == 0
r = dbg_ui(['walk', 'root'], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == ".\tdir\t\n%s\tlink\t%s\n%s\tdir\t\n%s/%s\tlink\t%s\n" % (
a, a_csum, b, b, c, a_csum)
assert captured.err == ''
r = dbg_ui(['walk', 'userdata'], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == a_csum + '\n'
assert captured.err == ''
r = dbg_ui(['reverse', a_csum], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "<tree> " + a + "\n<tree> " + b + "/" + c + "\n"
assert captured.err == ''
r = dbg_ui(['reverse', '--all', a_csum], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "<tree> " + a + "\n<tree> " + b + "/" + c + "\nmysnap " + a + "\nmysnap " + b + "/" + c + "\n"
assert captured.err == ''
r = dbg_ui(['reverse', '--snap', 'mysnap', a_csum], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "mysnap " + a + "\nmysnap " + b + "/" + c + "\n"
assert captured.err == ''
def test_ensure_file(tmp_path):
tmp = Path(str(tmp_path))
# Test write file
f1 = tmp.join('f1')
assert not f1.exists() and not f1.isfile()
with ensure_file(f1, 'w') as fd:
fd.write('f')
assert f1.exists() and f1.isfile()
# Test write file in missing dir
d2 = tmp.join('d2')
f2 = d2.join(d2)
assert not d2.exists() and not f2.exists()
with ensure_file(f2, 'w') as fd:
fd.write('f')
assert d2.exists() and f2.exists()
# Test write file when replacing a file with a dir.
d3 = tmp.join('d3')
with ensure_file(d3, 'w') as fd:
fd.write('f')
assert d3.isfile()
f3 = d3.join(d3)
with ensure_file(f3, 'w') as fd:
fd.write('f')
assert d3.exists() and d3.isdir() and f3.exists() and f3.isfile()
def __init__(self, root): self.root = Path(root)
