def test_file_disappeared_2(self):
# We want to delete the file after the initial lstat() call,
# but before the file is opened for reading later on, to test this
# race condition. So we patch os.lstat to delete the file right after
# the lstat call.
file = self.create_file("dir/file1", "file contents")
scan.scan()
self.assertEqual(3, models.FSEntry.objects.count())
import os
real_lstat = os.lstat
def lstat(path):
stat_result = real_lstat(path)
if path == str(file):
file.unlink()
return stat_result
self.stack.enter_context(mock.patch(
"os.lstat",
lstat,
))
backup.backup()
self.assertEqual(2, models.FSEntry.objects.count())
self.assertEqual(
2,
models.Object.objects.count(),
)
self.assert_backupsets({self.backupdir: {'dir': {}}})
def test_calculate_children(self):
"""Checks that the Object.calculate_children() works as expected
Doesn't actually call restore()
"""
self.create_file("dir1/file1", "contents")
self.create_file("dir1/file2", "asdf")
self.create_file("dir2/file3", "aoeu")
self.create_file("dir2/file4", "zzzz")
scan.scan()
backup.backup()
self.assertEqual(11, models.Object.objects.count())
# Make sure the table is consistent, as the unit tests are run in a
# transaction and so foreign key constraints are not enforced
c = connection.cursor()
c.execute("PRAGMA foreign_key_check")
self.assertEqual(0, len(list(c)))
for obj in models.Object.objects.all():
self.assertSetEqual(set(b.hex() for b in obj.calculate_children()),
{c.objid.hex()
for c in obj.children.all()},
"Object {}'s children don't match".format(obj))
def test_restore_multiple_revisions(self):
self.create_file("file", "contents A")
scan.scan()
backup.backup()
self.create_file("file", "new contents")
scan.scan()
backup.backup()
snapshots = list(models.Snapshot.objects.order_by("date"))
self.assertEqual(2, len(snapshots))
self.assertEqual(6, models.Object.objects.count())
restoredir = pathlib.Path(self.restoredir)
restore.restore_item(snapshots[0].root, restoredir / "ss1", self.key)
restore.restore_item(snapshots[1].root, restoredir / "ss2", self.key)
file1 = restoredir / "ss1" / "file"
file2 = restoredir / "ss2" / "file"
self.assertEqual("contents A", file1.read_text())
self.assertEqual("new contents", file2.read_text())
def test_restore_large_file(self):
"""This file should take more than one block to save, so it tests
routines that must operate on multiple blocks.
"""
infile = self.create_file("bigfile", "")
block = b"\0" * 1024 * 1024
h = hashlib.md5()
with infile.open("wb") as f:
for _ in range(50):
h.update(block)
f.write(block)
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
outfile = pathlib.Path(self.restoredir, "bigfile")
h2 = hashlib.md5()
with outfile.open("rb") as f:
while True:
a = f.read(64 * 2**10)
if not a:
break
h2.update(a)
self.assertEqual(h.hexdigest(), h2.hexdigest())
def test_objects_comitted(self):
"""Do a backup and then assert the objects actually get committed to
the backing store"""
self.create_file("dir/file1", "file contents")
scan.scan()
backup.backup()
ds = datastore.default_datastore
for obj in models.Object.objects.all():
# Assert that an object file exists in the backing store and is
# named properly. In particular, make sure we're naming them with
# the hex representation of the objid
obj_filepath = pathlib.Path(
self.datadir,
"objects",
obj.objid.hex()[:3],
obj.objid.hex(),
)
self.assertTrue(obj_filepath.is_file())
cached_payload = obj.payload
remote_payload = ds.get_object(obj.objid)
if cached_payload is None:
# Only blob type objects don't have a cached payload
self.assertEqual(
"blob", umsgpack.unpack(util.BytesReader(remote_payload)))
else:
self.assertEqual(
cached_payload,
remote_payload,
)
def test_restore_time_dir(self):
dir_a = pathlib.Path(self.backupdir, "dir1")
dir_a.mkdir()
os.utime(dir_a, ns=(123456789, 987654321))
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
# The directory atime gets reset before we back it up, so just check
# that whatever value it had when it was backed up, that's what gets
# restored.
tree = ss.root.children.get()
info = list(models.Object.unpack_payload(tree.payload))[1]
atime = info['atime']
restore.restore_item(ss.root, self.restoredir, self.key)
dir1 = pathlib.Path(self.restoredir, "dir1")
stat_result = dir1.stat()
self.assertEqual(
987654321,
stat_result.st_mtime_ns,
)
self.assertEqual(
atime,
stat_result.st_atime_ns,
)
def test_invalid_utf8_filename(self):
"""Tests that a file with invalid utf-8 in the name can be backed up"""
name = os.fsdecode(b"\xff\xffhello\xff\xff")
self.create_file(name, "file contents")
scan.scan()
backup.backup()
self.assert_backupsets({self.backupdir: {name: "file contents"}})
def test_deleted_file(self):
file = self.create_file("dir/file1", "file contents")
scan.scan()
self.assertTrue(
models.FSEntry.objects.filter(path=os.fspath(file)).exists())
file.unlink()
scan.scan()
self.assertFalse(
models.FSEntry.objects.filter(path=os.fspath(file)).exists())
def test_root_merge(self):
file = self.create_file("dir1/dir2/file", "file contents")
models.FSEntry.objects.create(path=os.fspath(file.parent))
self.assertEqual(
2,
models.FSEntry.objects.filter(parent__isnull=True).count())
scan.scan()
self.assertEqual(
1,
models.FSEntry.objects.filter(parent__isnull=True).count())
def test_scan(self):
self.create_file("dir/file1", "file contents")
self.create_file("dir2/file2", "another file contents")
scan.scan()
self.assertEqual(5, models.FSEntry.objects.count())
entries = models.FSEntry.objects.all()
names = set(e.name for e in entries)
for name in ['file1', 'file2', 'dir', 'dir2']:
self.assertIn(name, names)
def test_replace_dir_with_file(self):
file = self.create_file("dir/file1", "file contents")
scan.scan()
file.unlink()
file.parent.rmdir()
file.parent.write_text("another file contents")
# Scan the parent first
models.FSEntry.objects.get(path=os.fspath(file.parent)).scan()
scan.scan()
self._replace_dir_with_file_asserts(file)
def test_file_disappeared(self):
file = self.create_file("dir/file1", "file contents")
scan.scan()
self.assertEqual(3, models.FSEntry.objects.count())
file.unlink()
backup.backup()
self.assertEqual(2, models.FSEntry.objects.count())
self.assertEqual(
2,
models.Object.objects.count(),
)
self.assert_backupsets({self.backupdir: {'dir': {}}})
def test_simple_restore(self):
self.create_file("file1", "contents1")
self.create_file("dir/file2", "contents2")
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
self.assert_restored_file("file1", "contents1")
self.assert_restored_file("dir/file2", "contents2")
def test_restore_invalid_utf8_filename(self):
name = os.fsdecode(b"\xFF\xFFHello\xFF\xFF")
self.assertRaises(UnicodeEncodeError, name.encode, "utf-8")
self.create_file(name, "contents")
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
def test_backup_identical_files(self):
self.create_file("file1", "file contents")
self.create_file("file2", "file contents")
scan.scan()
backup.backup()
self.assert_backupsets({
self.backupdir: {
'file1': 'file contents',
'file2': 'file contents',
}
})
# Inode objects differ, so 4 total objects uploaded
self.assertEqual(4, models.Object.objects.count())
def test_backup_hardlinked_files(self):
file = self.create_file("file1", "file contents")
os.link(file, file.parent / "file2")
scan.scan()
backup.backup()
self.assert_backupsets({
self.backupdir: {
'file1': 'file contents',
'file2': 'file contents',
}
})
# The inode objects should be identical, so 3 total objects uploaded
self.assertEqual(3, models.Object.objects.count())
def test_permission_denied_file(self):
file = self.create_file("dir/file1", "file contents")
scan.scan()
self.assertEqual(3, models.FSEntry.objects.count())
file.chmod(0o000)
backup.backup()
self.assertEqual(2, models.FSEntry.objects.count())
self.assertEqual(
2,
models.Object.objects.count(),
)
self.assert_backupsets({self.backupdir: {'dir': {}}})
def test_dir_no_permission(self):
file = self.create_file("dir/file1", "file contents")
file.parent.chmod(0o000)
scan.scan()
self.assertTrue(
models.FSEntry.objects.filter(
path=os.fspath(file.parent)).exists())
self.assertFalse(
models.FSEntry.objects.filter(path=os.fspath(file)).exists())
# Set permission back so the tests can be cleaned up
file.parent.chmod(0o777)
def test_restore_mode(self):
file_a = self.create_file("file1", "contents")
file_a.chmod(0o777)
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
file_b = pathlib.Path(self.restoredir, "file1")
stat_result = file_b.stat()
self.assertEqual(
0o777,
stat.S_IMODE(stat_result.st_mode),
)
def test_restore_single_file(self):
self.create_file("file", "contents")
scan.scan()
backup.backup()
root = models.Snapshot.objects.get().root
# Should just be one child
inode = root.children.get()
filename = pathlib.Path(self.restoredir, "my_file")
restore.restore_item(inode, filename, self.key)
self.assertEqual("contents", filename.read_text())
def test_restore_uid_gid(self):
file_a = self.create_file("file1", "contents")
try:
os.chown(file_a, 1, 1)
except PermissionError:
raise unittest.SkipTest("Process doesn't have chown permission")
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
file_b = pathlib.Path(self.restoredir, "file1")
stat_result = file_b.stat()
self.assertEqual(1, stat_result.st_uid)
self.assertEqual(1, stat_result.st_gid)
def test_backup(self):
self.create_file("dir/file1", "file contents")
self.create_file("dir/file2", "file contents 2")
scan.scan()
self.assertEqual(4, models.FSEntry.objects.count())
backup.backup()
self.assertTrue(
all(entry.obj is not None
for entry in models.FSEntry.objects.all()))
self.assertEqual(6, models.Object.objects.count())
self.assert_backupsets({
self.backupdir: {
'dir': {
'file1': 'file contents',
'file2': 'file contents 2',
}
}
})
def test_not_plaintext(self):
"""Tests that the plaintext of a file doesn't appear in the object
payload on disk"""
self.create_file("secret_file", "super secret contents")
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
tree = ss.root
inode = tree.children.get()
blob = inode.children.get()
self.assertIs(None, blob.payload)
path = pathlib.Path(self.datadir, "objects",
blob.objid.hex()[:3], blob.objid.hex())
self.assertTrue(path.exists())
contents = path.read_bytes()
self.assertFalse(b"super secret contents" in contents)
def test_restore_time(self):
file_a = self.create_file("file1", "contents")
os.utime(file_a, ns=(123456789, 987654321))
scan.scan()
backup.backup()
ss = models.Snapshot.objects.get()
restore.restore_item(ss.root, self.restoredir, self.key)
file_b = pathlib.Path(self.restoredir, "file1")
stat_result = file_b.stat()
self.assertEqual(
123456789,
stat_result.st_atime_ns,
)
self.assertEqual(
987654321,
stat_result.st_mtime_ns,
)
