def test_ensure_absent(tmp_path):
# Test broken symlink
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
sb = tmp.join("sb")
ensure_symlink(sb, dne)
assert sb.islink()
ensure_absent(sb)
assert not sb.islink()
# Test link is removed, but underlying file is left intact.
tmp = Path(str(tmp_path))
f = tmp.join("dne")
with f.open('w') as fd:
fd.write('hi')
sb = tmp.join("sb")
ensure_symlink(sb, f)
assert sb.islink()
assert f.isfile()
ensure_absent(sb)
assert not sb.islink()
assert f.isfile()
# Test dir
d = tmp.join('d')
f1 = d.join('f1')
f2 = d.join('f2')
d.mkdir()
with f1.open('w') as fd:
fd.write('f1')
with f2.open('w') as fd:
fd.write('f2')
assert d.exists() and d.isdir()
ensure_absent(d)
assert not d.exists() and not d.isdir()
def test_ensure_symlink(tmp_path):
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
sb = tmp.join("sb")
ensure_symlink(sb, dne)
assert sb.exists()
assert sb.islink()
assert sb.readlink() == dne
def test_ensure_copy(tmp_path):
tmp = Path(str(tmp_path))
s = tmp.join('s')
with s.open('w') as fd:
fd.write('f')
d = tmp.join('d')
assert s.exists() and s.isfile()
assert not d.exists() and not d.isfile()
ensure_copy(d, s)
assert s.exists() and s.isfile()
assert d.exists() and d.isfile()
def test_ensure_link(tmp_path):
tmp = Path(str(tmp_path))
# test link to missing entity.
with pytest.raises(AssertionError):
dne = tmp.join("dne")
lb = tmp.join("lb")
ensure_link(lb, dne)
# test link to existing entry
f = tmp.join('f')
with f.open('w') as fd:
fd.write('f')
l1 = tmp.join('l1')
assert not l1.exists() and not l1.isfile()
ensure_link(l1, f)
assert l1.exists() and l1.isfile()
def test_rename_dir(tmp_path):
tmp = Path(str(tmp_path))
d = tmp.join("d")
d.mkdir()
d.join("inside").mkdir()
d2 = tmp.join("d2")
assert d.exists()
assert d.isdir()
assert not d2.exists()
assert not d2.isdir()
d.rename(d2)
assert not d.exists()
assert not d.isdir()
assert d2.exists()
assert d2.isdir()
def test_rename_file(tmp_path):
tmp = Path(str(tmp_path))
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
f2 = tmp.join("f2")
assert f.exists()
assert f.isfile()
assert not f2.exists()
assert not f2.isfile()
f.rename(f2)
assert not f.exists()
assert not f.isfile()
assert f2.exists()
assert f2.isfile()
def test_rewrite_links(tmp_path, capsys):
tmp = Path(str(tmp_path))
vol1 = tmp.join("vol1")
vol2 = tmp.join("vol2")
a = Path('a', vol1)
# Make the Farm
r = farmfs_ui(['mkfs'], vol1)
captured = capsys.readouterr()
assert r == 0
# Make a
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol1)
captured = capsys.readouterr()
assert r == 0
# Move from vol1 to vol2
vol1.rename(vol2)
# Reinit the fs. This will fix the udd directory pointer.
r = farmfs_ui(['mkfs'], vol2)
captured = capsys.readouterr()
assert r == 0
# Rewrite the links
r = dbg_ui(['rewrite-links', '.'], vol2)
captured = capsys.readouterr()
vol2a = vol2.join('a')
vol2a_blob = str(vol2a.readlink())
assert r == 0
assert captured.out == "Relinked a to " + vol2a_blob + "\n"
assert captured.err == ""
def test_unlink_clean(tmp_path):
tmp = Path(str(tmp_path))
d1 = tmp.join("d1")
d1.mkdir()
d2 = d1.join("d2")
d2.mkdir()
d3 = d2.join("d3")
d3.mkdir()
f1 = d3.join("f1")
f2 = d3.join("f2")
with f1.open("w") as fd:
fd.write("f1")
with f2.open("w") as fd:
fd.write("f2")
assert d1.exists()
assert d2.exists()
assert d3.exists()
assert f1.exists()
assert f2.exists()
f2.unlink(clean=d2)
assert d1.exists()
assert d2.exists()
assert d3.exists()
assert f1.exists()
assert not f2.exists()
f1.unlink(clean=d2)
assert d1.exists()
assert d2.exists()
assert not d3.exists()
assert not f1.exists()
assert not f2.exists()
def test_rename_symlink(tmp_path):
tmp = Path(str(tmp_path))
d = tmp.join("d")
d.mkdir()
s = tmp.join("s")
s.symlink(d)
s2 = tmp.join("s2")
assert s.exists()
assert s.islink()
assert not s2.exists()
assert not s2.islink()
s.rename(s2)
assert not s.exists()
assert not s.islink()
assert s2.exists()
assert s2.islink()
def test_readlink(tmp_path):
tmp = Path(str(tmp_path))
# Test a path that doesn't exist
dne = tmp.join("dne")
with pytest.raises(OSError) as e:
dne.readlink()
assert e.value.errno == FileDoesNotExist
# Test a regular file.
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
with pytest.raises(OSError) as e:
f.readlink() is None
assert e.value.errno == InvalidArgument
# Test a directory
d = tmp.join("d")
d.mkdir()
with pytest.raises(OSError) as e:
d.readlink() is None
assert e.value.errno == InvalidArgument
# Test a symlink to a regular file
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
assert f_slnk.readlink() == f
# Test a symlink to a directory.
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
assert d_slnk.readlink() == d
# Test a broken symlink.
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
assert b_slnk.readlink() == dne
def test_exists(tmp_path):
tmp = Path(str(tmp_path))
# Test a path that doesn't exist
dne = tmp.join("dne")
assert not dne.exists()
# Test a regular file.
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
assert f.exists()
# Test a directory
d = tmp.join("d")
d.mkdir()
assert d.exists()
# Test a symlink to a regular file
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
assert f_slnk.exists()
# Test a symlink to a directory.
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
assert d_slnk.exists()
# Test a broken symlink.
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
assert b_slnk.exists()
def test_checksum_non_files(tmp_path):
tmp = Path(str(tmp_path))
# Test dir
d = tmp.join("d")
d.mkdir()
with pytest.raises(IOError) as e:
d.checksum()
assert e.value.errno == IsADirectory
# Test symlink to dir
d_slnk = tmp.join("dslnk")
d_slnk.symlink(d)
with pytest.raises(IOError) as e:
d_slnk.checksum()
assert e.value.errno == IsADirectory
# Setup files
f = tmp.join("f")
with f.open("w") as fd:
fd.write("")
# Test file symlink
f_slnk = tmp.join("fslnk")
f_slnk.symlink(f)
f_slnk.checksum() == u"d41d8cd98f00b204e9800998ecf8427e"
# Test broken symlink
b_slnk = tmp.join("bslnk")
b_slnk.symlink(tmp.join("dne"))
with pytest.raises(IOError) as e:
b_slnk.checksum()
assert e.value.errno == FileDoesNotExist
def test_ensure_dir(tmp_path):
# Test dir already exists.
tmp = Path(str(tmp_path))
d1 = tmp.join('d1')
d1.mkdir()
assert d1.isdir()
ensure_dir(d1)
assert d1.isdir()
# Test dir creation
d2 = tmp.join('d2')
assert not d2.exists()
ensure_dir(d2)
assert d2.exists() and d2.isdir()
# test removes file
d3 = tmp.join('d3')
dne = tmp.join('dne')
d3.symlink(dne)
assert d3.exists() and d3.islink()
ensure_dir(d3)
assert d3.exists() and d3.isdir()
# test removes symlink
d4 = tmp.join('d4')
f = tmp.join('f')
with f.open('w') as fd:
fd.write('f')
d4.symlink(f)
assert d4.exists() and not d4.isdir() and d4.islink()
ensure_dir(d4)
assert d4.exists() and d4.isdir() and not d4.islink()
def test_ensure_file(tmp_path):
tmp = Path(str(tmp_path))
# Test write file
f1 = tmp.join('f1')
assert not f1.exists() and not f1.isfile()
with ensure_file(f1, 'w') as fd:
fd.write('f')
assert f1.exists() and f1.isfile()
# Test write file in missing dir
d2 = tmp.join('d2')
f2 = d2.join(d2)
assert not d2.exists() and not f2.exists()
with ensure_file(f2, 'w') as fd:
fd.write('f')
assert d2.exists() and f2.exists()
# Test write file when replacing a file with a dir.
d3 = tmp.join('d3')
with ensure_file(d3, 'w') as fd:
fd.write('f')
assert d3.isfile()
f3 = d3.join(d3)
with ensure_file(f3, 'w') as fd:
fd.write('f')
assert d3.exists() and d3.isdir() and f3.exists() and f3.isfile()
def test_farmfs_ignore_corruption(tmp_path, capsys):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
captured = capsys.readouterr()
assert r1 == 0
a = Path('a', root)
with a.open('w') as a_fd:
a_fd.write('a')
r2 = farmfs_ui(['freeze'], root)
captured = capsys.readouterr()
assert r2 == 0
with root.join(".farmignore").open("w") as ignore:
ignore.write("a")
r3 = farmfs_ui(['fsck', '--frozen-ignored'], root)
captured = capsys.readouterr()
assert captured.out == 'Ignored file frozen a\n'
assert captured.err == ""
assert r3 == 4
def test_create_dir(tmp_path):
a = Path(str(tmp_path)).join('a')
b = a.join('b')
assert a.isdir() == False
assert b.isdir() == False
# Cannot create with missing parents.
with pytest.raises(OSError) as e_info:
b.mkdir()
assert e_info.value.errno == FileDoesNotExist
assert a.isdir() == False
assert b.isdir() == False
# Create a
a.mkdir()
assert a.isdir() == True
assert b.isdir() == False
# idempotent
a.mkdir()
assert a.isdir() == True
assert b.isdir() == False
def test_unlink(tmp_path):
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
d = tmp.join("d")
d.mkdir()
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
# test unlink f_slnk
assert f.exists()
assert f_slnk.exists()
f_slnk.unlink()
assert not f_slnk.exists()
assert f.exists()
# test unlink d_slnk
assert d.exists()
assert d_slnk.exists()
d_slnk.unlink()
assert not d_slnk.exists()
assert d.exists()
# test unlink b_slnk
assert b_slnk.exists()
b_slnk.unlink()
assert not b_slnk.exists()
# test unlink dne
assert not dne.exists()
dne.unlink()
assert not dne.exists()
# test unlink file
assert f.exists()
f.unlink()
assert not f.exists()
# test unlink dir
assert d.exists()
with pytest.raises(OSError) as e:
d.unlink()
assert e.value.errno == NotPermitted
assert d.exists()
def test_farmfs_freeze_snap_thaw(tmp_path, parent, child, snap, content, read,
write):
root = Path(str(tmp_path))
r1 = farmfs_ui(['mkfs'], root)
assert r1 == 0
parent_path = Path(parent, root)
child_path = Path(child, parent_path)
parent_path.mkdir()
with child_path.open(write) as child_fd:
child_fd.write(content)
assert parent_path.isdir()
assert child_path.isfile()
r2 = farmfs_ui(['freeze'], root)
assert r2 == 0
assert parent_path.isdir()
assert child_path.islink()
blob = child_path.readlink()
assert blob.isfile()
userdata = Path('.farmfs/userdata', root)
assert userdata in list(blob.parents())
with blob.open(read) as check_fd:
check_content = check_fd.read()
assert check_content == content
r3 = farmfs_ui(['snap', 'make', snap], root)
assert r3 == 0
snap_path = root.join(".farmfs/snap").join(snap)
snap_path.exists()
child_path.unlink()
assert not child_path.exists()
assert blob.isfile()
r4 = farmfs_ui(['snap', 'restore', snap], root)
assert r4 == 0
assert child_path.islink()
assert blob.isfile()
assert child_path.readlink() == blob
r5 = farmfs_ui(['thaw', parent], root)
assert r5 == 0
assert child_path.isfile()
r6 = farmfs_ui(['freeze', child], parent_path)
assert r6 == 0
child_path.islink()
def test_checksum_file(tmp_path, input, expected):
tmp = Path(str(tmp_path))
fp = tmp.join("empty.txt")
with fp.open("wb") as fd:
fd.write(input)
assert fp.checksum() == expected
def test_s3_upload(tmp_path, capsys):
tmp = Path(str(tmp_path))
vol = tmp.join("vol")
a = Path('a', vol)
# Make the Farm
r = farmfs_ui(['mkfs'], vol)
captured = capsys.readouterr()
assert r == 0
# Make a
with a.open('w') as fd:
fd.write('a')
a_csum = str(a.checksum())
r = farmfs_ui(['freeze'], vol)
captured = capsys.readouterr()
assert r == 0
# upload to s3
bucket = 's3libtestbucket'
prefix = str(uuid.uuid1())
# Assert s3 bucket/prefix is empty
r = dbg_ui(['s3', 'list', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == ""
assert captured.err == ""
# Upload the contents.
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == \
'Fetching remote blobs\n' + \
'Remote Blobs: 0\n' + \
'Fetching local blobs\n' + \
'Local Blobs: 1\n' + \
'Uploading 1 blobs to s3\n' + \
'Successfully uploaded\n'
assert captured.err == ""
# Upload again
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == \
'Fetching remote blobs\n' + \
'Remote Blobs: 1\n' + \
'Fetching local blobs\n' + \
'Local Blobs: 1\n' + \
'Uploading 0 blobs to s3\n' + \
'Successfully uploaded\n'
assert captured.err == ""
# verify checksums
r = dbg_ui(['s3', 'check', bucket, prefix], vol)
captured = capsys.readouterr()
assert r == 0
assert captured.out == "All S3 blobs etags match\n"
assert captured.err == ""
# verify corrupt checksum
a_blob = a.readlink()
a_blob.unlink()
with a_blob.open('w') as fd:
fd.write('b')
b_csum = str(a.checksum())
ensure_readonly(a_blob)
prefix2 = str(uuid.uuid1())
r = dbg_ui(['s3', 'upload', '--quiet', bucket, prefix2], vol)
captured = capsys.readouterr()
assert r == 0
r = dbg_ui(['s3', 'check', bucket, prefix2], vol)
captured = capsys.readouterr()
assert r == 2
assert captured.out == a_csum + " " + b_csum + "\n"
assert captured.err == ""
def test_link(tmp_path):
tmp = Path(str(tmp_path))
dne = tmp.join("dne")
f = tmp.join("f")
with f.open("w") as fd:
fd.write("some text")
d = tmp.join("d")
d.mkdir()
f_slnk = tmp.join("f_lnk")
f_slnk.symlink(f)
d_slnk = tmp.join("d_lnk")
d_slnk.symlink(d)
b_slnk = tmp.join("b_lnk")
b_slnk.symlink(dne)
# Test with DNE name.
# test DNE to DNE
with pytest.raises(OSError) as e:
tmp.join("dne_dne").link(dne)
assert e.value.errno == FileDoesNotExist
# test DNE to dir
with pytest.raises(OSError) as e:
tmp.join("dne_d").link(d)
assert e.value.errno == NotPermitted
# test DNE to file
dne_f = tmp.join("dne_f")
dne_f.link(f)
with dne_f.open("r") as fd:
fd.read() == "some text"
# test DNE to file symlink
dne_fslnk = tmp.join("dne_fslnk")
dne_fslnk.link(f_slnk)
with dne_fslnk.open("r") as fd:
fd.read() == "some text"
# test DNE to dir symlink
dne_dslnk = tmp.join("dne_dslnk")
with pytest.raises(OSError) as e:
dne_dslnk.link(d_slnk)
# test DNE to broken symlink
dne_bslnk = tmp.join("dns_bslnk")
with pytest.raises(OSError) as e:
dne_bslnk.link(b_slnk)
assert e.value.errno == FileDoesNotExist
# Test with existing dir name
# test dir to DNE
dir_dne = tmp.join("dir_dne")
dir_dne.mkdir()
with pytest.raises(OSError) as e:
dir_dne.link(dne)
assert e.value.errno == FileDoesNotExist
# test dir to dir
dir_dir = tmp.join("dir_dir")
dir_dir.mkdir()
with pytest.raises(OSError) as e:
dir_dir.link(d)
assert e.value.errno == NotPermitted
# test dir to file
dir_f = tmp.join("dir_f")
dir_f.mkdir()
with pytest.raises(OSError) as e:
dir_f.link(f)
assert e.value.errno == FileExists
# test dir to file symlink
dir_fslnk = tmp.join("dir_fslnk")
dir_fslnk.mkdir()
with pytest.raises(OSError) as e:
dir_fslnk.link(f_slnk)
assert e.value.errno == FileExists
# test dir to dir symlink
# Skipped...
# test dir to broken symlink
# Skipped...
# Test with existing file name
# test file to DNE
f_dne = tmp.join("f_dne")
with f_dne.open("w") as fd:
fd.write("foo")
with pytest.raises(OSError) as e:
f_dne.link(dne)
assert e.value.errno == FileDoesNotExist
# test file to dir
f_dir = tmp.join("f_dir")
with f_dir.open("w") as fd:
fd.write("foo")
with pytest.raises(OSError) as e:
f_dir.link(d)
assert e.value.errno == NotPermitted
# test file to file
f_f = tmp.join("f_f")
with f_f.open("w") as fd:
fd.write("foo")
with pytest.raises(OSError) as e:
f_f.link(f)
assert e.value.errno == FileExists
# test file to file symlink
f_fslnk = tmp.join("f_fslnk")
f_fslnk.mkdir()
with pytest.raises(OSError) as e:
f_fslnk.link(f_slnk)
assert e.value.errno == FileExists
# test file to dir symlink
# Skipped...
# test file to broken symlink
# Skipped...
# Test with existing symlink
# test slnk to DNE
slnk_dne = tmp.join("slnk_dne")
slnk_dne.symlink(f)
with pytest.raises(OSError) as e:
slnk_dne.link(dne)
assert e.value.errno == FileDoesNotExist
# test slnk to dir
slnk_dir = tmp.join("slnk_dir")
slnk_dir.symlink(f)
with pytest.raises(OSError) as e:
slnk_dir.link(d)
assert e.value.errno == NotPermitted
# test slnk to file
slnk_f = tmp.join("slnk_f")
slnk_f.symlink(f)
with pytest.raises(OSError) as e:
slnk_f.link(f)
assert e.value.errno == FileExists
# test slnk to file symlink
slnk_fslnk = tmp.join("slnk_fslnk")
slnk_fslnk.symlink(f)
with pytest.raises(OSError) as e:
slnk_fslnk.link(f_slnk)
assert e.value.errno == FileExists
