def test_blob(tmp_path, capsys): root = Path(str(tmp_path)) a = Path('a', root) b = Path('b', root) # Make the Farm r = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r == 0 # Make a,b,b2; freeze, snap, delete with a.open('w') as fd: fd.write('a') a_csum = str(a.checksum()) with b.open('w') as fd: fd.write('b') b_csum = str(b.checksum()) r = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r == 0 # get blob paths r = dbg_ui(['blob', a_csum, b_csum], root) captured = capsys.readouterr() assert r == 0 a_rel = a.readlink().relative_to(root) b_rel = b.readlink().relative_to(root) assert captured.out == a_csum + " " + a_rel + "\n" + b_csum + " " + b_rel + "\n" assert captured.err == ""
def test_farmfs_blob_broken(tmp_path, capsys): root = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r1 == 0 a = Path('a', root) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r2 = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r2 == 0 a_blob = a.readlink() a_blob.unlink() r3 = farmfs_ui(['fsck', '--broken'], root) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\ta\n" assert captured.err == '' assert r3 == 1 # Test relative pathing. d = Path('d', root) d.mkdir() r4 = farmfs_ui(['fsck', '--broken'], d) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\t../a\n" assert captured.err == '' assert r3 == 1
def test_farmfs_freeze_snap_thaw(tmp_path, parent, child, snap, content, read, write): root = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], root) assert r1 == 0 parent_path = Path(parent, root) child_path = Path(child, parent_path) parent_path.mkdir() with child_path.open(write) as child_fd: child_fd.write(content) assert parent_path.isdir() assert child_path.isfile() r2 = farmfs_ui(['freeze'], root) assert r2 == 0 assert parent_path.isdir() assert child_path.islink() blob = child_path.readlink() assert blob.isfile() userdata = Path('.farmfs/userdata', root) assert userdata in list(blob.parents()) with blob.open(read) as check_fd: check_content = check_fd.read() assert check_content == content r3 = farmfs_ui(['snap', 'make', snap], root) assert r3 == 0 snap_path = root.join(".farmfs/snap").join(snap) snap_path.exists() child_path.unlink() assert not child_path.exists() assert blob.isfile() r4 = farmfs_ui(['snap', 'restore', snap], root) assert r4 == 0 assert child_path.islink() assert blob.isfile() assert child_path.readlink() == blob r5 = farmfs_ui(['thaw', parent], root) assert r5 == 0 assert child_path.isfile() r6 = farmfs_ui(['freeze', child], parent_path) assert r6 == 0 child_path.islink()
def test_farmfs_freeze_snap_thaw(vol, parent, child, snap, content, read, write): parent_path = Path(parent, vol) child_path = Path(child, parent_path) parent_path.mkdir() with child_path.open(write) as child_fd: child_fd.write(content) assert parent_path.isdir() assert child_path.isfile() r = farmfs_ui(['freeze'], vol) assert r == 0 assert parent_path.isdir() assert child_path.islink() blob = child_path.readlink() assert blob.isfile() userdata = Path('.farmfs/userdata', vol) assert userdata in list(blob.parents()) with blob.open(read) as check_fd: check_content = check_fd.read() assert check_content == content r = farmfs_ui(['snap', 'make', snap], vol) assert r == 0 snap_path = vol.join(".farmfs/snap").join(snap) snap_path.exists() child_path.unlink() assert not child_path.exists() assert blob.isfile() r = farmfs_ui(['snap', 'restore', snap], vol) assert r == 0 assert child_path.islink() assert blob.isfile() assert child_path.readlink() == blob r = farmfs_ui(['thaw', parent], vol) assert r == 0 assert child_path.isfile() r = farmfs_ui(['freeze', child], parent_path) assert r == 0 child_path.islink()
def test_farmfs_blob_permission(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 a_blob = a.readlink() a_blob.chmod(0o777) r = farmfs_ui(['fsck', '--blob-permissions'], vol) captured = capsys.readouterr() assert captured.out == 'writable blob: ' + a_csum + '\n' assert captured.err == "" assert r == 8
def test_farmfs_blob_corruption(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 a_blob = a.readlink() a_blob.unlink() with a_blob.open('w') as a_fd: a_fd.write('b') ensure_readonly(a_blob) r = farmfs_ui(['fsck', '--checksums'], vol) captured = capsys.readouterr() assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n' assert captured.err == "" assert r == 2
def test_farmfs_blob_permission(tmp_path, capsys): root = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r1 == 0 a = Path('a', root) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r2 = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r2 == 0 a_blob = a.readlink() a_blob.chmod(0o777) r3 = farmfs_ui(['fsck', '--blob-permissions'], root) captured = capsys.readouterr() assert captured.out == 'writable blob: ' + a_csum + '\n' assert captured.err == "" assert r3 == 8
def test_farmfs_blob_corruption(tmp_path, capsys): root = Path(str(tmp_path)) r1 = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r1 == 0 a = Path('a', root) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r2 = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r2 == 0 a_blob = a.readlink() a_blob.unlink() with a_blob.open('w') as a_fd: a_fd.write('b') ensure_readonly(a_blob) r3 = farmfs_ui(['fsck', '--checksums'], root) captured = capsys.readouterr() assert captured.out == 'CORRUPTION checksum mismatch in blob ' + a_csum + '\n' assert captured.err == "" assert r3 == 2
def test_farmfs_blob_broken(vol, capsys): a = Path('a', vol) with a.open('w') as a_fd: a_fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 a_blob = a.readlink() a_blob.unlink() r = farmfs_ui(['fsck', '--broken'], vol) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\ta\n" assert captured.err == '' assert r == 1 # Test relative pathing. d = Path('d', vol) d.mkdir() r = farmfs_ui(['fsck', '--broken'], d) captured = capsys.readouterr() assert captured.out == a_csum + "\n\t<tree>\t../a\n" assert captured.err == '' assert r == 1
def test_s3_upload(tmp_path, capsys): tmp = Path(str(tmp_path)) vol = tmp.join("vol") a = Path('a', vol) # Make the Farm r = farmfs_ui(['mkfs'], vol) captured = capsys.readouterr() assert r == 0 # Make a with a.open('w') as fd: fd.write('a') a_csum = str(a.checksum()) r = farmfs_ui(['freeze'], vol) captured = capsys.readouterr() assert r == 0 # upload to s3 bucket = 's3libtestbucket' prefix = str(uuid.uuid1()) # Assert s3 bucket/prefix is empty r = dbg_ui(['s3', 'list', bucket, prefix], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "" assert captured.err == "" # Upload the contents. r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == \ 'Fetching remote blobs\n' + \ 'Remote Blobs: 0\n' + \ 'Fetching local blobs\n' + \ 'Local Blobs: 1\n' + \ 'Uploading 1 blobs to s3\n' + \ 'Successfully uploaded\n' assert captured.err == "" # Upload again r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == \ 'Fetching remote blobs\n' + \ 'Remote Blobs: 1\n' + \ 'Fetching local blobs\n' + \ 'Local Blobs: 1\n' + \ 'Uploading 0 blobs to s3\n' + \ 'Successfully uploaded\n' assert captured.err == "" # verify checksums r = dbg_ui(['s3', 'check', bucket, prefix], vol) captured = capsys.readouterr() assert r == 0 assert captured.out == "All S3 blobs etags match\n" assert captured.err == "" # verify corrupt checksum a_blob = a.readlink() a_blob.unlink() with a_blob.open('w') as fd: fd.write('b') b_csum = str(a.checksum()) ensure_readonly(a_blob) prefix2 = str(uuid.uuid1()) r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix2], vol) captured = capsys.readouterr() assert r == 0 r = dbg_ui(['s3', 'check', bucket, prefix2], vol) captured = capsys.readouterr() assert r == 2 assert captured.out == a_csum + " " + b_csum + "\n" assert captured.err == ""
def test_fix_link(tmp_path, capsys): test_dir = Path(str(tmp_path)) # Make roots vol1 = Path("vol1", test_dir) vol1.mkdir() vol2 = Path("vol2", test_dir) vol2.mkdir() # Setup vol1 r = farmfs_ui(['mkfs'], vol1) captured = capsys.readouterr() assert r == 0 a = Path('a', vol1) b = Path('b', vol1) c = Path('c', vol1) cd = Path('c/d', vol1) # Make a,b; freeze, snap, delete with a.open('w') as fd: fd.write('a') a_csum = str(a.checksum()) with b.open('w') as fd: fd.write('b') r = farmfs_ui(['freeze'], vol1) captured = capsys.readouterr() assert r == 0 # Setup vol2 r = farmfs_ui(['mkfs'], vol2) captured = capsys.readouterr() assert r == 0 e = Path('e', vol2) with e.open('w') as fd: fd.write('e') e_csum = str(e.checksum()) r = farmfs_ui(['freeze'], vol2) captured = capsys.readouterr() assert r == 0 # Setup remote r = farmfs_ui(['remote', 'add', 'vol2', '../vol2'], vol1) captured = capsys.readouterr() assert r == 0 # Check file type for a r = dbg_ui(['fix', 'link', a_csum, 'b'], vol1) captured = capsys.readouterr() assert r == 0 assert captured.err == "" assert captured.out == "" assert a.readlink() == b.readlink() # Try to fix link to a missing blob, e with pytest.raises(ValueError): r = dbg_ui(['fix', 'link', e_csum, 'e'], vol1) captured = capsys.readouterr() assert r == 0 assert captured.err == "" assert captured.out == "blob " + e_csum + " doesn't exist\n" # Pull e from remote. r = dbg_ui(['fix', 'link', '--remote', 'vol2', e_csum, 'e'], vol1) captured = capsys.readouterr() assert r == 0 assert captured.out == "blob " + e_csum + " doesn't exist\n" assert captured.err == "" # Try to fix a link to a missing target. r = dbg_ui(['fix', 'link', a_csum, 'c'], vol1) captured = capsys.readouterr() assert r == 0 assert captured.err == "" assert captured.out == "" assert a.readlink() == c.readlink() # Try to fix a link to a missing target, in a dir which is blobked by a link r = dbg_ui(['fix', 'link', a_csum, 'c/d'], vol1) captured = capsys.readouterr() assert r == 0 assert captured.err == "" assert captured.out == "" assert a.readlink() == cd.readlink()
def test_gc(tmp_path, capsys): root = Path(str(tmp_path)) sk = Path('sk', root) sd = Path('sd', root) tk = Path('tk', root) td = Path('td', root) # Make the Farm r = farmfs_ui(['mkfs'], root) captured = capsys.readouterr() assert r == 0 # Make sk, freeze, snap, delete with sk.open('w') as fd: fd.write('sk') r = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r == 0 sk_blob = sk.readlink() r = farmfs_ui(['snap', 'make', 'snk'], root) captured = capsys.readouterr() assert r == 0 sk.unlink() # Make sd, freeze, snap, delete, delete snap with sd.open('w') as fd: fd.write('sd') sd_csum = str(sd.checksum()) r = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r == 0 sd_blob = sd.readlink() r = farmfs_ui(['snap', 'make', 'snd'], root) captured = capsys.readouterr() assert r == 0 sd.unlink() r = farmfs_ui(['snap', 'delete', 'snd'], root) captured = capsys.readouterr() assert r == 0 # Make tk and td, freeze, delete td with tk.open('w') as fd: fd.write('tk') with td.open('w') as fd: fd.write('td') r = farmfs_ui(['freeze'], root) captured = capsys.readouterr() assert r == 0 tk_blob = tk.readlink() td_blob = td.readlink() td_csum = str(td.checksum()) td.unlink() # GC --noop assert sk_blob.exists() assert sd_blob.exists() assert tk_blob.exists() assert td_blob.exists() r = farmfs_ui(['gc', '--noop'], root) captured = capsys.readouterr() assert captured.out == 'Removing ' + sd_csum + '\nRemoving ' + td_csum + '\n' assert captured.err == '' assert r == 0 assert sk_blob.exists() assert sd_blob.exists() assert tk_blob.exists() assert td_blob.exists() # GC r = farmfs_ui(['gc'], root) captured = capsys.readouterr() assert captured.out == 'Removing ' + sd_csum + '\nRemoving ' + td_csum + '\n' assert captured.err == '' assert r == 0 assert sk_blob.exists() assert not sd_blob.exists() assert tk_blob.exists() assert not td_blob.exists()