def test_blob(tmp_path, capsys):
root = Path(str(tmp_path))
a = Path('a', root)
b = Path('b', root)
# Make the Farm
r = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r == 0
# Make a,b,b2; freeze, snap, delete
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
with b.open('w') as fd:
fd.write('b')
b_csum = str(b.checksum())
r = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r == 0
# get blob paths
r = dbg_ui(['blob', a_csum, b_csum], root)
captured = capsys.readouterr()
assert r == 0
a_rel = a.readlink().relative_to(root)
b_rel = b.readlink().relative_to(root)
assert captured.out == a_csum + " " + a_rel + "\n" + b_csum + " " + b_rel + "\n"
assert captured.err == ""
def test_farmfs_blob_broken(tmp_path, capsys):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', root)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r2 = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r2 == 0
a_blob = a.readlink()
a_blob.unlink()
r3 = farmfs_ui(['fsck', '--broken'], root)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\ta\n"
assert captured.err == ''
assert r3 == 1
# Test relative pathing.
d = Path('d', root)
d.mkdir()
r4 = farmfs_ui(['fsck', '--broken'], d)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\t../a\n"
assert captured.err == ''
assert r3 == 1
def test_farmfs_freeze_snap_thaw(tmp_path, parent, child, snap, content, read,
write):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
assert r1 == 0
parent_path = Path(parent, root)
child_path = Path(child, parent_path)
parent_path.mkdir()
with child_path.open(write) as child_fd:
child_fd.write(content)
assert parent_path.isdir()
assert child_path.isfile()
r2 = farmfs_ui(['freeze'], root)
assert r2 == 0
assert parent_path.isdir()
assert child_path.islink()
blob = child_path.readlink()
assert blob.isfile()
userdata = Path('.farmfs/userdata', root)
assert userdata in list(blob.parents())
with blob.open(read) as check_fd:
check_content = check_fd.read()
assert check_content == content
r3 = farmfs_ui(['snap', 'make', snap], root)
assert r3 == 0
snap_path = root.join(".farmfs/snap").join(snap)
snap_path.exists()
child_path.unlink()
assert not child_path.exists()
assert blob.isfile()
r4 = farmfs_ui(['snap', 'restore', snap], root)
assert r4 == 0
assert child_path.islink()
assert blob.isfile()
assert child_path.readlink() == blob
r5 = farmfs_ui(['thaw', parent], root)
assert r5 == 0
assert child_path.isfile()
r6 = farmfs_ui(['freeze', child], parent_path)
assert r6 == 0
child_path.islink()
def test_farmfs_freeze_snap_thaw(vol, parent, child, snap, content, read,
write):
parent_path = Path(parent, vol)
child_path = Path(child, parent_path)
parent_path.mkdir()
with child_path.open(write) as child_fd:
child_fd.write(content)
assert parent_path.isdir()
assert child_path.isfile()
r = farmfs_ui(['freeze'], vol)
assert r == 0
assert parent_path.isdir()
assert child_path.islink()
blob = child_path.readlink()
assert blob.isfile()
userdata = Path('.farmfs/userdata', vol)
assert userdata in list(blob.parents())
with blob.open(read) as check_fd:
check_content = check_fd.read()
assert check_content == content
r = farmfs_ui(['snap', 'make', snap], vol)
assert r == 0
snap_path = vol.join(".farmfs/snap").join(snap)
snap_path.exists()
child_path.unlink()
assert not child_path.exists()
assert blob.isfile()
r = farmfs_ui(['snap', 'restore', snap], vol)
assert r == 0
assert child_path.islink()
assert blob.isfile()
assert child_path.readlink() == blob
r = farmfs_ui(['thaw', parent], vol)
assert r == 0
assert child_path.isfile()
r = farmfs_ui(['freeze', child], parent_path)
assert r == 0
child_path.islink()
def test_farmfs_blob_permission(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
a_blob = a.readlink()
a_blob.chmod(0o777)
r = farmfs_ui(['fsck', '--blob-permissions'], vol)
captured = capsys.readouterr()
assert captured.out == 'writable blob: ' + a_csum + '\n'
assert captured.err == ""
assert r == 8
def test_farmfs_blob_corruption(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
a_blob = a.readlink()
a_blob.unlink()
with a_blob.open('w') as a_fd:
a_fd.write('b')
ensure_readonly(a_blob)
r = farmfs_ui(['fsck', '--checksums'], vol)
captured = capsys.readouterr()
assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n'
assert captured.err == ""
assert r == 2
def test_farmfs_blob_permission(tmp_path, capsys):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', root)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r2 = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r2 == 0
a_blob = a.readlink()
a_blob.chmod(0o777)
r3 = farmfs_ui(['fsck', '--blob-permissions'], root)
captured = capsys.readouterr()
assert captured.out == 'writable blob: ' + a_csum + '\n'
assert captured.err == ""
assert r3 == 8
def test_farmfs_blob_corruption(tmp_path, capsys):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', root)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r2 = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r2 == 0
a_blob = a.readlink()
a_blob.unlink()
with a_blob.open('w') as a_fd:
a_fd.write('b')
ensure_readonly(a_blob)
r3 = farmfs_ui(['fsck', '--checksums'], root)
captured = capsys.readouterr()
assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n'
assert captured.err == ""
assert r3 == 2
def test_farmfs_blob_broken(vol, capsys):
a = Path('a', vol)
with a.open('w') as a_fd:
a_fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
a_blob = a.readlink()
a_blob.unlink()
r = farmfs_ui(['fsck', '--broken'], vol)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\ta\n"
assert captured.err == ''
assert r == 1
# Test relative pathing.
d = Path('d', vol)
d.mkdir()
r = farmfs_ui(['fsck', '--broken'], d)
captured = capsys.readouterr()
assert captured.out == a_csum + "\n\t<tree>\t../a\n"
assert captured.err == ''
assert r == 1
def test_s3_upload(tmp_path, capsys):
tmp = Path(str(tmp_path))
vol = tmp.join("vol")
a = Path('a', vol)
# Make the Farm
r = farmfs_ui(['mkfs'], vol)
captured = capsys.readouterr()
assert r == 0
# Make a
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
# upload to s3
bucket = 's3libtestbucket'
prefix = str(uuid.uuid1())
# Assert s3 bucket/prefix is empty
r = dbg_ui(['s3', 'list', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == ""
assert captured.err == ""
# Upload the contents.
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == \
'Fetching remote blobs\n' + \
'Remote Blobs: 0\n' + \
'Fetching local blobs\n' + \
'Local Blobs: 1\n' + \
'Uploading 1 blobs to s3\n' + \
'Successfully uploaded\n'
assert captured.err == ""
# Upload again
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == \
'Fetching remote blobs\n' + \
'Remote Blobs: 1\n' + \
'Fetching local blobs\n' + \
'Local Blobs: 1\n' + \
'Uploading 0 blobs to s3\n' + \
'Successfully uploaded\n'
assert captured.err == ""
# verify checksums
r = dbg_ui(['s3', 'check', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "All S3 blobs etags match\n"
assert captured.err == ""
# verify corrupt checksum
a_blob = a.readlink()
a_blob.unlink()
with a_blob.open('w') as fd:
fd.write('b')
b_csum = str(a.checksum())
ensure_readonly(a_blob)
prefix2 = str(uuid.uuid1())
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix2], vol)
captured = capsys.readouterr()
assert r == 0
r = dbg_ui(['s3', 'check', bucket, prefix2], vol)
captured = capsys.readouterr()
assert r == 2
assert captured.out == a_csum + " " + b_csum + "\n"
assert captured.err == ""
def test_fix_link(tmp_path, capsys):
test_dir = Path(str(tmp_path))
# Make roots
vol1 = Path("vol1", test_dir)
vol1.mkdir()
vol2 = Path("vol2", test_dir)
vol2.mkdir()
# Setup vol1
r = farmfs_ui(['mkfs'], vol1)
captured = capsys.readouterr()
assert r == 0
a = Path('a', vol1)
b = Path('b', vol1)
c = Path('c', vol1)
cd = Path('c/d', vol1)
# Make a,b; freeze, snap, delete
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
with b.open('w') as fd:
fd.write('b')
r = farmfs_ui(['freeze'], vol1)
captured = capsys.readouterr()
assert r == 0
# Setup vol2
r = farmfs_ui(['mkfs'], vol2)
captured = capsys.readouterr()
assert r == 0
e = Path('e', vol2)
with e.open('w') as fd:
fd.write('e')
e_csum = str(e.checksum())
r = farmfs_ui(['freeze'], vol2)
captured = capsys.readouterr()
assert r == 0
# Setup remote
r = farmfs_ui(['remote', 'add', 'vol2', '../vol2'], vol1)
captured = capsys.readouterr()
assert r == 0
# Check file type for a
r = dbg_ui(['fix', 'link', a_csum, 'b'], vol1)
captured = capsys.readouterr()
assert r == 0
assert captured.err == ""
assert captured.out == ""
assert a.readlink() == b.readlink()
# Try to fix link to a missing blob, e
with pytest.raises(ValueError):
r = dbg_ui(['fix', 'link', e_csum, 'e'], vol1)
captured = capsys.readouterr()
assert r == 0
assert captured.err == ""
assert captured.out == "blob " + e_csum + " doesn't exist\n"
# Pull e from remote.
r = dbg_ui(['fix', 'link', '--remote', 'vol2', e_csum, 'e'], vol1)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "blob " + e_csum + " doesn't exist\n"
assert captured.err == ""
# Try to fix a link to a missing target.
r = dbg_ui(['fix', 'link', a_csum, 'c'], vol1)
captured = capsys.readouterr()
assert r == 0
assert captured.err == ""
assert captured.out == ""
assert a.readlink() == c.readlink()
# Try to fix a link to a missing target, in a dir which is blobked by a link
r = dbg_ui(['fix', 'link', a_csum, 'c/d'], vol1)
captured = capsys.readouterr()
assert r == 0
assert captured.err == ""
assert captured.out == ""
assert a.readlink() == cd.readlink()
def test_gc(tmp_path, capsys):
root = Path(str(tmp_path))
sk = Path('sk', root)
sd = Path('sd', root)
tk = Path('tk', root)
td = Path('td', root)
# Make the Farm
r = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r == 0
# Make sk, freeze, snap, delete
with sk.open('w') as fd:
fd.write('sk')
r = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r == 0
sk_blob = sk.readlink()
r = farmfs_ui(['snap', 'make', 'snk'], root)
captured = capsys.readouterr()
assert r == 0
sk.unlink()
# Make sd, freeze, snap, delete, delete snap
with sd.open('w') as fd:
fd.write('sd')
sd_csum = str(sd.checksum())
r = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r == 0
sd_blob = sd.readlink()
r = farmfs_ui(['snap', 'make', 'snd'], root)
captured = capsys.readouterr()
assert r == 0
sd.unlink()
r = farmfs_ui(['snap', 'delete', 'snd'], root)
captured = capsys.readouterr()
assert r == 0
# Make tk and td, freeze, delete td
with tk.open('w') as fd:
fd.write('tk')
with td.open('w') as fd:
fd.write('td')
r = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r == 0
tk_blob = tk.readlink()
td_blob = td.readlink()
td_csum = str(td.checksum())
td.unlink()
# GC --noop
assert sk_blob.exists()
assert sd_blob.exists()
assert tk_blob.exists()
assert td_blob.exists()
r = farmfs_ui(['gc', '--noop'], root)
captured = capsys.readouterr()
assert captured.out == 'Removing ' + sd_csum + '\nRemoving ' + td_csum + '\n'
assert captured.err == ''
assert r == 0
assert sk_blob.exists()
assert sd_blob.exists()
assert tk_blob.exists()
assert td_blob.exists()
# GC
r = farmfs_ui(['gc'], root)
captured = capsys.readouterr()
assert captured.out == 'Removing ' + sd_csum + '\nRemoving ' + td_csum + '\n'
assert captured.err == ''
assert r == 0
assert sk_blob.exists()
assert not sd_blob.exists()
assert tk_blob.exists()
assert not td_blob.exists()
