class cache_tests(unittest.TestCase):
def setUp(self):
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
self.dbfile = tempfile.NamedTemporaryFile(delete=False)
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
self.cache = inode_cache.InodeCache(self.db, 0)
def tearDown(self):
self.cache.destroy()
os.unlink(self.dbfile.name)
def test_create(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
for key in attrs.keys():
self.assertEqual(attrs[key], getattr(inode, key))
self.assertTrue(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
def test_del(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
del self.cache[inode.id]
self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
self.assertRaises(KeyError, self.cache.__delitem__, inode.id)
def test_get(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
for (key, val) in attrs.iteritems():
self.assertEqual(getattr(inode, key), val)
# Create another inode
self.cache.create_inode(**attrs)
self.db.execute('DELETE FROM inodes WHERE id=?', (inode.id,))
# Entry should still be in cache
self.assertEqual(inode, self.cache[inode.id])
# Now it should be out of the cache
for _ in xrange(inode_cache.CACHE_SIZE + 1):
self.cache.create_inode(**attrs)
self.assertRaises(KeyError, self.cache.__getitem__, inode.id)
class fs_api_tests(unittest.TestCase):
def setUp(self):
self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
plain_backend = local.Backend('local://' + self.backend_dir, None, None)
self.backend_pool = BackendPool(lambda: ComprencBackend(b'schwubl', ('zlib', 6),
plain_backend))
self.backend = self.backend_pool.pop_conn()
self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
self.max_obj_size = 1024
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
self.dbfile = tempfile.NamedTemporaryFile(delete=False)
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
# Tested methods assume that they are called from
# file system request handler
llfuse.lock.acquire()
cache = BlockCache(self.backend_pool, self.db, self.cachedir + "/cache",
self.max_obj_size * 5)
self.block_cache = cache
self.server = fs.Operations(cache, self.db, self.max_obj_size,
InodeCache(self.db, 0))
self.server.init()
# Monkeypatch around the need for removal and upload threads
cache.to_remove = DummyQueue(cache)
class DummyDistributor:
def put(self, arg, timeout=None):
cache._do_upload(*arg)
return True
cache.to_upload = DummyDistributor()
# Keep track of unused filenames
self.name_cnt = 0
def tearDown(self):
self.server.inodes.destroy()
llfuse.lock.release()
self.block_cache.destroy()
shutil.rmtree(self.cachedir)
shutil.rmtree(self.backend_dir)
os.unlink(self.dbfile.name)
self.dbfile.close()
@staticmethod
def random_data(len_):
with open("/dev/urandom", "rb") as fd:
return fd.read(len_)
def fsck(self):
self.block_cache.clear()
self.server.inodes.flush()
fsck = Fsck(self.cachedir + '/cache', self.backend,
{ 'max_obj_size': self.max_obj_size }, self.db)
fsck.check()
self.assertFalse(fsck.found_errors)
def newname(self):
self.name_cnt += 1
return ("s3ql_%d" % self.name_cnt).encode()
def test_getattr_root(self):
self.assertTrue(stat.S_ISDIR(self.server.getattr(ROOT_INODE).mode))
self.fsck()
def test_create(self):
ctx = Ctx()
mode = self.dir_mode()
name = self.newname()
inode_p_old = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server._create(ROOT_INODE, name, mode, ctx)
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON name_id = names.id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
inode = self.server.getattr(id_)
self.assertEqual(inode.mode, mode)
self.assertEqual(inode.uid, ctx.uid)
self.assertEqual(inode.gid, ctx.gid)
self.assertEqual(inode.refcount, 1)
self.assertEqual(inode.size, 0)
inode_p_new = self.server.getattr(ROOT_INODE)
self.assertGreater(inode_p_new.mtime, inode_p_old.mtime)
self.assertGreater(inode_p_new.ctime, inode_p_old.ctime)
self.server.forget([(id_, 1)])
self.fsck()
def test_extstat(self):
# Test with zero contents
self.server.extstat()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.extstat()
# Test with data in file
fh = self.server.open(inode.id, os.O_RDWR)
self.server.write(fh, 0, b'foobar')
self.server.release(fh)
self.server.extstat()
self.server.forget([(inode.id, 1)])
self.fsck()
@staticmethod
def dir_mode():
return (randint(0, 0o7777) & ~stat.S_IFDIR) | stat.S_IFDIR
@staticmethod
def file_mode():
return (randint(0, 0o7777) & ~stat.S_IFREG) | stat.S_IFREG
def test_getxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.getxattr, inode.id, b'nonexistant-attr')
self.server.setxattr(inode.id, b'my-attr', b'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, b'my-attr'), b'strabumm!')
self.server.forget([(inode.id, 1)])
self.fsck()
def test_link(self):
name = self.newname()
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
safe_sleep(CLOCK_GRANULARITY)
inode_before = self.server.getattr(inode.id).copy()
self.server.link(inode.id, inode_p_new.id, name)
inode_after = self.server.lookup(inode_p_new.id, name)
inode_p_new_after = self.server.getattr(inode_p_new.id)
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, inode_p_new.id))
self.assertEqual(inode_before.id, id_)
self.assertEqual(inode_after.refcount, 2)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1), (inode_after.id, 1)])
self.fsck()
def test_listxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertListEqual([], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, b'key1', b'blub')
self.assertListEqual([b'key1'], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, b'key2', b'blub')
self.assertListEqual(sorted([b'key1', b'key2']),
sorted(self.server.listxattr(inode.id)))
self.server.forget([(inode.id, 1)])
self.fsck()
def test_read(self):
len_ = self.max_obj_size
data = self.random_data(len_)
off = self.max_obj_size // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, off, data)
inode_before = self.server.getattr(inode.id).copy()
safe_sleep(CLOCK_GRANULARITY)
self.assertTrue(self.server.read(fh, off, len_) == data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.atime, inode_before.atime)
self.assertTrue(self.server.read(fh, 0, len_) == b"\0" * off + data[:off])
self.assertTrue(self.server.read(fh, self.max_obj_size, len_) == data[off:])
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_readdir(self):
# Create a few entries
names = [ ('entry_%2d' % i).encode() for i in range(20) ]
for name in names:
(fh, inode) = self.server.create(ROOT_INODE, name,
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode.id, 1)])
# Delete some to make sure that we don't have continous rowids
remove_no = [0, 2, 3, 5, 9]
for i in remove_no:
self.server.unlink(ROOT_INODE, names[i])
del names[i]
# Read all
fh = self.server.opendir(ROOT_INODE)
self.assertListEqual(sorted(names + [b'lost+found']) ,
sorted(x[0] for x in self.server.readdir(fh, 0)))
self.server.releasedir(fh)
# Read in parts
fh = self.server.opendir(ROOT_INODE)
entries = list()
try:
next_ = 0
while True:
gen = self.server.readdir(fh, next_)
for _ in range(3):
(name, _, next_) = next(gen)
entries.append(name)
except StopIteration:
pass
self.assertListEqual(sorted(names + [b'lost+found']) ,
sorted(entries))
self.server.releasedir(fh)
self.fsck()
def test_forget(self):
name = self.newname()
# Test that entries are deleted when they're no longer referenced
(fh, inode) = self.server.create(ROOT_INODE, name,
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'foobar')
self.server.unlink(ROOT_INODE, name)
self.assertFalse(self.db.has_val('SELECT 1 FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(self.server.getattr(inode.id).id)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_removexattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.removexattr, inode.id, b'some name')
self.server.setxattr(inode.id, b'key1', b'blub')
self.server.removexattr(inode.id, b'key1')
self.assertListEqual([], self.server.listxattr(inode.id))
self.server.forget([(inode.id, 1)])
self.fsck()
def test_rename(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id == name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.fsck()
def test_replace_file(self):
oldname = self.newname()
newname = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, oldname, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, b'some data to deal with')
self.server.release(fh)
self.server.setxattr(inode.id, b'test_xattr', b'42*8')
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
(fh, inode2) = self.server.create(inode_p_new.id, newname, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, b'even more data to deal with')
self.server.release(fh)
self.server.setxattr(inode2.id, b'test_xattr', b'42*8')
self.server.forget([(inode2.id, 1)])
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id,)))
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.fsck()
def test_replace_dir(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
inode2 = self.server.mkdir(inode_p_new.id, newname, self.dir_mode(), Ctx())
self.server.forget([(inode2.id, 1)])
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id,)))
self.fsck()
def test_setattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(), 0o641,
os.O_RDWR, Ctx())
self.server.release(fh)
inode_old = self.server.getattr(inode.id).copy()
attr = llfuse.EntryAttributes()
attr.st_mode = self.file_mode()
attr.st_uid = randint(0, 2 ** 32)
attr.st_gid = None
attr.st_atime = randint(0, 2 ** 32) / 10 ** 6
attr.st_mtime = randint(0, 2 ** 32) / 10 ** 6
safe_sleep(CLOCK_GRANULARITY)
self.server.setattr(inode.id, attr)
inode_new = self.server.getattr(inode.id)
self.assertGreater(inode_new.ctime, inode_old.ctime)
for key in attr.__slots__:
if key in ('st_mode', 'st_uid', 'st_atime', 'st_mtime'):
self.assertEqual(getattr(attr, key),
getattr(inode_new, key), '%s mismatch' % key)
elif key != 'st_ctime':
self.assertEqual(getattr(inode_old, key),
getattr(inode_new, key), '%s mismatch' % key)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_truncate(self):
len_ = int(2.7 * self.max_obj_size)
data = self.random_data(len_)
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
attr.st_size = len_ // 2
self.server.setattr(inode.id, attr)
self.assertTrue(self.server.read(fh, 0, len_) == data[:len_ // 2])
attr.st_size = len_
self.server.setattr(inode.id, attr)
self.assertTrue(self.server.read(fh, 0, len_)
== data[:len_ // 2] + b'\0' * (len_ // 2))
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_truncate_0(self):
len1 = 158
len2 = 133
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, self.random_data(len1))
self.server.release(fh)
self.server.inodes.flush()
fh = self.server.open(inode.id, os.O_RDWR)
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, self.random_data(len2))
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_setxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.setxattr(inode.id, b'my-attr', b'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, b'my-attr'), b'strabumm!')
self.server.forget([(inode.id, 1)])
self.fsck()
def test_names(self):
name1 = self.newname()
name2 = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, name1, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode.id, 1)])
(fh, inode) = self.server.create(ROOT_INODE, name2, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.setxattr(inode.id, name1, b'strabumm!')
self.fsck()
self.server.removexattr(inode.id, name1)
self.fsck()
self.server.setxattr(inode.id, name1, b'strabumm karacho!!')
self.server.unlink(ROOT_INODE, name1)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_statfs(self):
# Test with zero contents
self.server.statfs()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.statfs()
# Test with data in file
fh = self.server.open(inode.id, os.O_RDWR)
self.server.write(fh, 0, b'foobar')
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.server.statfs()
def test_symlink(self):
target = self.newname()
name = self.newname()
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
inode = self.server.symlink(ROOT_INODE, name, target, Ctx())
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertEqual(target, self.server.readlink(inode.id))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_unlink(self):
name = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'some data to deal with')
self.server.release(fh)
# Add extended attributes
self.server.setxattr(inode.id, b'test_xattr', b'42*8')
self.server.forget([(inode.id, 1)])
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.unlink(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_rmdir(self):
name = self.newname()
inode = self.server.mkdir(ROOT_INODE, name, self.dir_mode(), Ctx())
self.server.forget([(inode.id, 1)])
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.rmdir(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_relink(self):
name = self.newname()
name2 = self.newname()
data = b'some data to deal with'
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.release(fh)
self.server.unlink(ROOT_INODE, name)
self.server.inodes.flush()
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.server.link(inode.id, ROOT_INODE, name2)
self.server.forget([(inode.id, 2)])
inode = self.server.lookup(ROOT_INODE, name2)
fh = self.server.open(inode.id, os.O_RDONLY)
self.assertTrue(self.server.read(fh, 0, len(data)) == data)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_write(self):
len_ = self.max_obj_size
data = self.random_data(len_)
off = self.max_obj_size // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
inode_before = self.server.getattr(inode.id).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.write(fh, off, data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.mtime, inode_before.mtime)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertEqual(inode_after.size, off + len_)
self.server.write(fh, 0, data)
inode_after = self.server.getattr(inode.id)
self.assertEqual(inode_after.size, off + len_)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_failsafe(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.cache.clear()
self.assertTrue(self.server.failsafe is False)
datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with self.assertRaises(FUSEError) as cm:
with catch_logmsg('^Backend returned malformed data for',
count=1, level=logging.ERROR):
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
self.assertTrue(self.server.failsafe)
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with self.assertRaises(FUSEError) as cm:
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
# Release and re-open, now we should be able to access again
self.server.release(fh)
self.server.forget([(inode.id, 1)])
# ..but not write access since we are in failsafe mode
with self.assertRaises(FUSEError) as cm:
self.server.open(inode.id, os.O_RDWR)
self.assertEqual(cm.exception.errno, errno.EPERM)
# ..ready only is fine.
fh = self.server.open(inode.id, os.O_RDONLY)
self.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
self.server.read(fh, 3, len_//2)
self.server.cache.clear()
with self.assertRaises(FUSEError) as cm:
with catch_logmsg('^Backend lost block',
count=1, level=logging.ERROR):
self.server.read(fh, 5, len_//2)
self.assertEqual(cm.exception.errno, errno.EIO)
# Don't call fsck, we're missing a block
def test_create_open(self):
name = self.newname()
# Create a new file
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode, 1)])
# Open it atomically
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode, 1)])
self.fsck()
def test_edit(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.release(fh)
self.block_cache.clear()
fh = self.server.open(inode.id, os.O_RDWR)
attr = llfuse.EntryAttributes()
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, data[50:])
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_copy_tree(self):
ext_attr_name = b'system.foo.brazl'
ext_attr_val = b'schulla dku woumm bramp'
src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), Ctx())
dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(), Ctx())
# Create file
(fh, f1_inode) = self.server.create(src_inode.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
self.server.setxattr(f1_inode.id, ext_attr_name, ext_attr_val)
# Create hardlink
(fh, f2_inode) = self.server.create(src_inode.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
f2_inode = self.server.link(f2_inode.id, src_inode.id, b'file2_hardlink')
# Create subdirectory
d1_inode = self.server.mkdir(src_inode.id, b'dir1', self.dir_mode(), Ctx())
d2_inode = self.server.mkdir(d1_inode.id, b'dir2', self.dir_mode(), Ctx())
# ..with a 3rd hardlink
f2_inode = self.server.link(f2_inode.id, d1_inode.id, b'file2_hardlink')
# Replicate
self.server.copy_tree(src_inode.id, dst_inode.id)
# Change files
fh = self.server.open(f1_inode.id, os.O_RDWR)
self.server.write(fh, 0, b'new file1 contents')
self.server.release(fh)
fh = self.server.open(f2_inode.id, os.O_RDWR)
self.server.write(fh, 0, b'new file2 contents')
self.server.release(fh)
# Get copy properties
f1_inode_c = self.server.lookup(dst_inode.id, b'file1')
f2_inode_c = self.server.lookup(dst_inode.id, b'file2')
f2h_inode_c = self.server.lookup(dst_inode.id, b'file2_hardlink')
d1_inode_c = self.server.lookup(dst_inode.id, b'dir1')
d2_inode_c = self.server.lookup(d1_inode_c.id, b'dir2')
f2_h_inode_c = self.server.lookup(d1_inode_c.id, b'file2_hardlink')
# Check file1
fh = self.server.open(f1_inode_c.id, os.O_RDWR)
self.assertEqual(self.server.read(fh, 0, 42), b'file1 contents')
self.server.release(fh)
self.assertNotEqual(f1_inode.id, f1_inode_c.id)
self.assertEqual(self.server.getxattr(f1_inode_c.id, ext_attr_name),
ext_attr_val)
# Check file2
fh = self.server.open(f2_inode_c.id, os.O_RDWR)
self.assertTrue(self.server.read(fh, 0, 42) == b'file2 contents')
self.server.release(fh)
self.assertEqual(f2_inode_c.id, f2h_inode_c.id)
self.assertEqual(f2_inode_c.refcount, 3)
self.assertNotEqual(f2_inode.id, f2_inode_c.id)
self.assertEqual(f2_h_inode_c.id, f2_inode_c.id)
# Check subdir1
self.assertNotEqual(d1_inode.id, d1_inode_c.id)
self.assertNotEqual(d2_inode.id, d2_inode_c.id)
self.server.forget(list(self.server.open_inodes.items()))
self.fsck()
def test_copy_tree_2(self):
src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), Ctx())
dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(), Ctx())
# Create file
(fh, inode) = self.server.create(src_inode.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'block 1 contents')
self.server.write(fh, self.max_obj_size, b'block 1 contents')
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.server.copy_tree(src_inode.id, dst_inode.id)
self.server.forget([(src_inode.id, 1), (dst_inode.id, 1)])
self.fsck()
def test_lock_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, b'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
# Another file
(fh, inode3) = self.server.create(ROOT_INODE, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
# Lock
self.server.lock_tree(inode1.id)
for i in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertTrue(self.server.inodes[i].locked)
# Remove
with self.assertRaises(FUSEError) as cm:
self.server._remove(inode1.id, b'file1', inode1a.id)
self.assertEqual(cm.exception.errno, errno.EPERM)
# Rename / Replace
with self.assertRaises(FUSEError) as cm:
self.server.rename(ROOT_INODE, b'file1', inode1.id, b'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.rename(inode1.id, b'file1', ROOT_INODE, b'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
# Open
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_RDWR)
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_WRONLY)
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(self.server.open(inode3.id, os.O_WRONLY))
# Write
fh = self.server.open(inode2a.id, os.O_RDONLY)
with self.assertRaises(FUSEError) as cm:
self.server.write(fh, 0, b'foo')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(fh)
# Create
with self.assertRaises(FUSEError) as cm:
self.server._create(inode2.id, b'dir1', self.dir_mode(), os.O_RDWR, Ctx())
self.assertEqual(cm.exception.errno, errno.EPERM)
# Setattr
with self.assertRaises(FUSEError) as cm:
self.server.setattr(inode2a.id, dict())
self.assertEqual(cm.exception.errno, errno.EPERM)
# xattr
with self.assertRaises(FUSEError) as cm:
self.server.setxattr(inode2.id, b'name', b'value')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.removexattr(inode2.id, b'name')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.forget(list(self.server.open_inodes.items()))
self.fsck()
def test_remove_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(), Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, b'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
# Remove
self.server.forget(list(self.server.open_inodes.items()))
self.server.remove_tree(ROOT_INODE, b'source')
for (id_p, name) in ((ROOT_INODE, b'source'),
(inode1.id, b'file1'),
(inode1.id, b'dir1'),
(inode2.id, b'file2')):
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, id_p)))
for id_ in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (id_,)))
self.fsck()
class fsck_tests(unittest.TestCase):
def setUp(self):
self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
self.backend = local.Backend(
Namespace(storage_url='local://' + self.backend_dir))
self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
self.max_obj_size = 1024
self.dbfile = tempfile.NamedTemporaryFile()
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
self.fsck = Fsck(self.cachedir, self.backend,
{'max_obj_size': self.max_obj_size}, self.db)
self.fsck.expect_errors = True
def tearDown(self):
shutil.rmtree(self.cachedir)
shutil.rmtree(self.backend_dir)
self.dbfile.close()
def assert_fsck(self, fn):
'''Check that fn detects and corrects an error'''
self.fsck.found_errors = False
fn()
self.assertTrue(self.fsck.found_errors)
self.fsck.found_errors = False
self.fsck.check()
self.assertFalse(self.fsck.found_errors)
def test_cache(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH,
os.getuid(), os.getgid(), time_ns(), time_ns(), time_ns(), 1, 8))
self._link(b'test-entry', inode)
# Create new block
fh = open(self.cachedir + '/%d-0' % inode, 'wb')
fh.write(b'somedata')
fh.close()
self.assert_fsck(self.fsck.check_cache)
self.assertEqual(self.backend['s3ql_data_1'], b'somedata')
# Existing block
self.db.execute('UPDATE inodes SET size=? WHERE id=?',
(self.max_obj_size + 8, inode))
with open(self.cachedir + '/%d-1' % inode, 'wb') as fh:
fh.write(b'somedata')
self.assert_fsck(self.fsck.check_cache)
# Old block preserved
with open(self.cachedir + '/%d-0' % inode, 'wb') as fh:
fh.write(b'somedat2')
self.assert_fsck(self.fsck.check_cache)
# Old block removed
with open(self.cachedir + '/%d-1' % inode, 'wb') as fh:
fh.write(b'somedat3')
self.assert_fsck(self.fsck.check_cache)
def test_lof1(self):
# Make lost+found a file
inode = self.db.get_val(
"SELECT inode FROM contents_v WHERE name=? AND parent_inode=?",
(b"lost+found", ROOT_INODE))
self.db.execute('DELETE FROM contents WHERE parent_inode=?', (inode, ))
self.db.execute('UPDATE inodes SET mode=?, size=? WHERE id=?',
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, inode))
def check():
self.fsck.check_lof()
self.fsck.check_inodes_refcount()
self.assert_fsck(check)
def test_lof2(self):
# Remove lost+found
name_id = self.db.get_val('SELECT id FROM names WHERE name=?',
(b'lost+found', ))
inode = self.db.get_val(
'SELECT inode FROM contents WHERE name_id=? AND '
'parent_inode=?', (name_id, ROOT_INODE))
self.db.execute('DELETE FROM inodes WHERE id=?', (inode, ))
self.db.execute(
'DELETE FROM contents WHERE name_id=? and parent_inode=?',
(name_id, ROOT_INODE))
self.db.execute('UPDATE names SET refcount = refcount-1 WHERE id=?',
(name_id, ))
self.assert_fsck(self.fsck.check_lof)
def test_wrong_inode_refcount(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 0))
self._link(b'name1', inode)
self._link(b'name2', inode)
self.assert_fsck(self.fsck.check_inodes_refcount)
def test_orphaned_inode(self):
self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 0))
self.assert_fsck(self.fsck.check_inodes_refcount)
def test_name_refcount(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 2, 0))
self._link(b'name1', inode)
self._link(b'name2', inode)
self.db.execute('UPDATE names SET refcount=refcount+1 WHERE name=?',
(b'name1', ))
self.assert_fsck(self.fsck.check_names_refcount)
def test_orphaned_name(self):
self._add_name(b'zupbrazl')
self.assert_fsck(self.fsck.check_names_refcount)
def test_contents_inode(self):
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foobar'), 124, ROOT_INODE))
self.assert_fsck(self.fsck.check_contents_inode)
def test_contents_inode_p(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 0))
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foobar'), inode, 123))
self.assert_fsck(self.fsck.check_contents_parent_inode)
def test_contents_name(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 0))
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(42, inode, ROOT_INODE))
self.assert_fsck(self.fsck.check_contents_name)
def _add_name(self, name):
'''Get id for *name* and increase refcount
Name is inserted in table if it does not yet exist.
'''
try:
name_id = self.db.get_val('SELECT id FROM names WHERE name=?',
(name, ))
except NoSuchRowError:
name_id = self.db.rowid(
'INSERT INTO names (name, refcount) VALUES(?,?)', (name, 1))
else:
self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?',
(name_id, ))
return name_id
def _link(self, name, inode, parent_inode=ROOT_INODE):
'''Link /*name* to *inode*'''
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(name), inode, parent_inode))
def test_inodes_size(self):
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 128))
self._link(b'test-entry', id_)
obj_id = self.db.rowid(
'INSERT INTO objects (refcount,size) VALUES(?,?)', (1, 36))
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 512, sha256(b'foo')))
self.backend['s3ql_data_%d' % obj_id] = b'foo'
# Case 1
self.db.execute('UPDATE inodes SET size=? WHERE id=?',
(self.max_obj_size + 120, id_))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 1, block_id))
self.assert_fsck(self.fsck.check_inodes_size)
# Case 2
self.db.execute('DELETE FROM inode_blocks WHERE inode=?', (id_, ))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 0, block_id))
self.db.execute('UPDATE inodes SET size=? WHERE id=?', (129, id_))
self.assert_fsck(self.fsck.check_inodes_size)
# Case 3
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 1, block_id))
self.db.execute('UPDATE inodes SET size=? WHERE id=?',
(self.max_obj_size + 120, id_))
self.db.execute(
'UPDATE blocks SET refcount = refcount + 1 WHERE id = ?',
(block_id, ))
self.assert_fsck(self.fsck.check_inodes_size)
def test_objects_id(self):
# Create an object that only exists in the backend
self.backend['s3ql_data_4364'] = b'Testdata'
self.assert_fsck(self.fsck.check_objects_id)
# Create an object that does not exist in the backend
self.db.execute(
'INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(34, 1, 27))
self.assert_fsck(self.fsck.check_objects_id)
def test_blocks_checksum(self):
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 8))
self._link(b'test-entry', id_)
# Assume that due to a crash we did not write the hash for the block
self.backend['s3ql_data_4364'] = b'Testdata'
self.db.execute(
'INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(4364, 1, 8))
block_id = self.db.execute(
'INSERT INTO blocks (obj_id, refcount, size) VALUES(?, ?, ?)',
(4364, 1, 8))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 0, block_id))
# Should pick up wrong hash and delete objects
self.fsck.found_errors = False
self.fsck.check_blocks_checksum()
assert self.fsck.found_errors
self.fsck.found_errors = False
self.fsck.check_blocks_checksum()
assert not self.fsck.found_errors
# Should save files in lost+found
self.fsck.found_errors = False
self.fsck.check()
assert self.fsck.found_errors
# Now everything should be good
self.fsck.found_errors = False
self.fsck.check()
assert not self.fsck.found_errors
assert not self.db.has_val(
'SELECT block_id FROM inode_blocks WHERE inode=?', (id_, ))
inode_p = self.db.get_val(
'SELECT parent_inode FROM contents_v WHERE inode=?', (id_, ))
lof_id = self.db.get_val(
"SELECT inode FROM contents_v WHERE name=? AND parent_inode=?",
(b"lost+found", ROOT_INODE))
assert inode_p == lof_id
def test_blocks_obj_id(self):
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, 48, 128))
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 128))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, block_id))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_blocks_obj_id)
def test_missing_obj(self):
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 32)')
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 128))
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 128))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, block_id))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_objects_id)
def test_inode_blocks_inode(self):
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 42)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 34))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(27, 0, block_id))
self.assert_fsck(self.fsck.check_inode_blocks_inode)
def test_inode_blocks_block_id(self):
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 128))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, 35))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_inode_blocks_block_id)
def test_symlinks_inode(self):
self.db.execute(
'INSERT INTO symlink_targets (inode, target) VALUES(?,?)',
(42, b'somewhere else'))
self.assert_fsck(self.fsck.check_symlinks_inode)
def test_ext_attrs_inode(self):
self.db.execute(
'INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)',
(self._add_name(b'some name'), 34, b'some value'))
self.assert_fsck(self.fsck.check_ext_attributes_inode)
def test_ext_attrs_name(self):
id_ = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1, 128))
self._link(b'test-entry', id_)
self.db.execute(
'INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)',
(34, id_, b'some value'))
self.assert_fsck(self.fsck.check_ext_attributes_name)
@staticmethod
def random_data(len_):
with open("/dev/urandom", "rb") as fd:
return fd.read(len_)
def test_loops(self):
# Create some directory inodes
inodes = [
self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR, 0, 0, time_ns(),
time_ns(), time_ns(), 1)) for dummy in range(3)
]
inodes.append(inodes[0])
last = inodes[0]
for inode in inodes[1:]:
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)',
(self._add_name(str(inode).encode()), inode, last))
last = inode
self.assert_fsck(self.fsck.check_loops)
def test_tmpfile(self):
# Ensure that path exists
objname = 's3ql_data_38375'
self.backend[objname] = b'bla'
del self.backend[objname]
path = self.backend._key_to_path(objname)
tmpname = '%s#%d-%d.tmp' % (path, os.getpid(), _thread.get_ident())
with open(tmpname, 'wb') as fh:
fh.write(b'Hello, world')
self.assert_fsck(self.fsck.check_objects_temp)
def test_obj_refcounts(self):
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 42)')
block_id_1 = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'foo')))
block_id_2 = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'bar')))
self.backend['s3ql_data_%d' % obj_id] = b'foo and bar'
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1, 2048))
self._link(b'test-entry', inode)
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id_1))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 2, block_id_2))
self.assert_fsck(self.fsck.check_objects_refcount)
def test_orphaned_obj(self):
self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 33)')
self.assert_fsck(self.fsck.check_objects_refcount)
def test_wrong_block_refcount(self):
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 23)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'')))
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1,
self.max_obj_size))
self._link(b'test-entry', inode)
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 0, block_id))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id))
self.assert_fsck(self.fsck.check_blocks_refcount)
def test_orphaned_block(self):
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 24)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size, hash) VALUES(?,?,?,?)',
(1, obj_id, 3, sha256(b'xyz')))
self.assert_fsck(self.fsck.check_blocks_refcount)
def test_unix_size(self):
inode = 42
self.db.execute(
"INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1, 0))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET size = 1 WHERE id=?', (inode, ))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_size_symlink(self):
inode = 42
target = b'some funny random string'
self.db.execute(
"INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(inode, stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1, len(target)))
self.db.execute(
'INSERT INTO symlink_targets (inode, target) VALUES(?,?)',
(inode, target))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET size = 0 WHERE id=?', (inode, ))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_target(self):
inode = 42
self.db.execute(
"INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?,?)",
(inode, stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute(
'INSERT INTO symlink_targets (inode, target) VALUES(?,?)',
(inode, 'foo'))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_nomode_reg(self):
perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP
stamp = time_ns()
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1))
self._link(b'test-entry', inode)
self.assert_fsck(self.fsck.check_unix)
newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?',
(inode, ))
self.assertEqual(stat.S_IMODE(newmode), perms)
self.assertEqual(stat.S_IFMT(newmode), stat.S_IFREG)
def test_unix_nomode_dir(self):
perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP
stamp = time_ns()
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1))
inode2 = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)", (perms | stat.S_IFREG, os.getuid(),
os.getgid(), stamp, stamp, stamp, 1))
self._link(b'test-entry', inode)
self._link(b'subentry', inode2, inode)
self.assert_fsck(self.fsck.check_unix)
newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?',
(inode, ))
self.assertEqual(stat.S_IMODE(newmode), perms)
self.assertEqual(stat.S_IFMT(newmode), stat.S_IFDIR)
def test_unix_symlink_no_target(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1))
self._link(b'test-entry', inode)
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_rdev(self):
inode = 42
self.db.execute(
"INSERT INTO inodes (id, mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?,?)",
(inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET rdev=? WHERE id=?', (42, inode))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_child(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute(
'INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foo'), ROOT_INODE, inode))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_blocks(self):
inode = self.db.rowid(
"INSERT INTO inodes (mode,uid,gid,mtime_ns,atime_ns,ctime_ns,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFSOCK | stat.S_IRUSR | stat.S_IWUSR, os.getuid(),
os.getgid(), time_ns(), time_ns(), time_ns(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
obj_id = self.db.rowid(
'INSERT INTO objects (refcount, size) VALUES(1, 32)')
block_id = self.db.rowid(
'INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 0))
self.db.execute(
'INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
class fs_api_tests(unittest.TestCase):
def setUp(self):
self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
plain_backend = local.Backend('local://' + self.backend_dir, None,
None)
self.backend_pool = BackendPool(
lambda: ComprencBackend(b'schwubl', ('zlib', 6), plain_backend))
self.backend = self.backend_pool.pop_conn()
self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
self.max_obj_size = 1024
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
self.dbfile = tempfile.NamedTemporaryFile(delete=False)
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
# Tested methods assume that they are called from
# file system request handler
llfuse.lock.acquire()
cache = BlockCache(self.backend_pool, self.db,
self.cachedir + "/cache", self.max_obj_size * 5)
self.block_cache = cache
self.server = fs.Operations(cache, self.db, self.max_obj_size,
InodeCache(self.db, 0))
self.server.init()
# Monkeypatch around the need for removal and upload threads
cache.to_remove = DummyQueue(cache)
class DummyDistributor:
def put(self, arg, timeout=None):
cache._do_upload(*arg)
return True
cache.to_upload = DummyDistributor()
# Keep track of unused filenames
self.name_cnt = 0
def tearDown(self):
self.server.inodes.destroy()
llfuse.lock.release()
self.block_cache.destroy()
shutil.rmtree(self.cachedir)
shutil.rmtree(self.backend_dir)
os.unlink(self.dbfile.name)
self.dbfile.close()
@staticmethod
def random_data(len_):
with open("/dev/urandom", "rb") as fd:
return fd.read(len_)
def fsck(self):
self.block_cache.clear()
self.server.inodes.flush()
fsck = Fsck(self.cachedir + '/cache', self.backend,
{'max_obj_size': self.max_obj_size}, self.db)
fsck.check()
self.assertFalse(fsck.found_errors)
def newname(self):
self.name_cnt += 1
return ("s3ql_%d" % self.name_cnt).encode()
def test_getattr_root(self):
self.assertTrue(stat.S_ISDIR(self.server.getattr(ROOT_INODE).mode))
self.fsck()
def test_create(self):
ctx = Ctx()
mode = self.dir_mode()
name = self.newname()
inode_p_old = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server._create(ROOT_INODE, name, mode, ctx)
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON name_id = names.id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
inode = self.server.getattr(id_)
self.assertEqual(inode.mode, mode)
self.assertEqual(inode.uid, ctx.uid)
self.assertEqual(inode.gid, ctx.gid)
self.assertEqual(inode.refcount, 1)
self.assertEqual(inode.size, 0)
inode_p_new = self.server.getattr(ROOT_INODE)
self.assertGreater(inode_p_new.mtime, inode_p_old.mtime)
self.assertGreater(inode_p_new.ctime, inode_p_old.ctime)
self.server.forget([(id_, 1)])
self.fsck()
def test_extstat(self):
# Test with zero contents
self.server.extstat()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.extstat()
# Test with data in file
fh = self.server.open(inode.id, os.O_RDWR)
self.server.write(fh, 0, b'foobar')
self.server.release(fh)
self.server.extstat()
self.server.forget([(inode.id, 1)])
self.fsck()
@staticmethod
def dir_mode():
return (randint(0, 0o7777) & ~stat.S_IFDIR) | stat.S_IFDIR
@staticmethod
def file_mode():
return (randint(0, 0o7777) & ~stat.S_IFREG) | stat.S_IFREG
def test_getxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.getxattr, inode.id,
b'nonexistant-attr')
self.server.setxattr(inode.id, b'my-attr', b'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, b'my-attr'),
b'strabumm!')
self.server.forget([(inode.id, 1)])
self.fsck()
def test_link(self):
name = self.newname()
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
safe_sleep(CLOCK_GRANULARITY)
inode_before = self.server.getattr(inode.id).copy()
self.server.link(inode.id, inode_p_new.id, name)
inode_after = self.server.lookup(inode_p_new.id, name)
inode_p_new_after = self.server.getattr(inode_p_new.id)
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, inode_p_new.id))
self.assertEqual(inode_before.id, id_)
self.assertEqual(inode_after.refcount, 2)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1),
(inode_after.id, 1)])
self.fsck()
def test_listxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertListEqual([], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, b'key1', b'blub')
self.assertListEqual([b'key1'], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, b'key2', b'blub')
self.assertListEqual(sorted([b'key1', b'key2']),
sorted(self.server.listxattr(inode.id)))
self.server.forget([(inode.id, 1)])
self.fsck()
def test_read(self):
len_ = self.max_obj_size
data = self.random_data(len_)
off = self.max_obj_size // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, off, data)
inode_before = self.server.getattr(inode.id).copy()
safe_sleep(CLOCK_GRANULARITY)
self.assertTrue(self.server.read(fh, off, len_) == data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.atime, inode_before.atime)
self.assertTrue(
self.server.read(fh, 0, len_) == b"\0" * off + data[:off])
self.assertTrue(
self.server.read(fh, self.max_obj_size, len_) == data[off:])
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_readdir(self):
# Create a few entries
names = [('entry_%2d' % i).encode() for i in range(20)]
for name in names:
(fh, inode) = self.server.create(ROOT_INODE,
name, self.file_mode(), os.O_RDWR,
Ctx())
self.server.release(fh)
self.server.forget([(inode.id, 1)])
# Delete some to make sure that we don't have continous rowids
remove_no = [0, 2, 3, 5, 9]
for i in remove_no:
self.server.unlink(ROOT_INODE, names[i])
del names[i]
# Read all
fh = self.server.opendir(ROOT_INODE)
self.assertListEqual(sorted(names + [b'lost+found']),
sorted(x[0] for x in self.server.readdir(fh, 0)))
self.server.releasedir(fh)
# Read in parts
fh = self.server.opendir(ROOT_INODE)
entries = list()
try:
next_ = 0
while True:
gen = self.server.readdir(fh, next_)
for _ in range(3):
(name, _, next_) = next(gen)
entries.append(name)
except StopIteration:
pass
self.assertListEqual(sorted(names + [b'lost+found']), sorted(entries))
self.server.releasedir(fh)
self.fsck()
def test_forget(self):
name = self.newname()
# Test that entries are deleted when they're no longer referenced
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, b'foobar')
self.server.unlink(ROOT_INODE, name)
self.assertFalse(
self.db.has_val(
'SELECT 1 FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(self.server.getattr(inode.id).id)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.assertFalse(
self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id, )))
self.fsck()
def test_removexattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.removexattr, inode.id,
b'some name')
self.server.setxattr(inode.id, b'key1', b'blub')
self.server.removexattr(inode.id, b'key1')
self.assertListEqual([], self.server.listxattr(inode.id))
self.server.forget([(inode.id, 1)])
self.fsck()
def test_rename(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON names.id == name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.fsck()
def test_replace_file(self):
oldname = self.newname()
newname = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, oldname, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, b'some data to deal with')
self.server.release(fh)
self.server.setxattr(inode.id, b'test_xattr', b'42*8')
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
(fh, inode2) = self.server.create(inode_p_new.id, newname,
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'even more data to deal with')
self.server.release(fh)
self.server.setxattr(inode2.id, b'test_xattr', b'42*8')
self.server.forget([(inode2.id, 1)])
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.assertFalse(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id, )))
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.fsck()
def test_replace_dir(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
inode2 = self.server.mkdir(inode_p_new.id, newname, self.dir_mode(),
Ctx())
self.server.forget([(inode2.id, 1)])
safe_sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.server.forget([(inode.id, 1), (inode_p_new.id, 1)])
self.assertFalse(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id, )))
self.fsck()
def test_setattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(), 0o641,
os.O_RDWR, Ctx())
self.server.release(fh)
inode_old = self.server.getattr(inode.id).copy()
attr = llfuse.EntryAttributes()
attr.st_mode = self.file_mode()
attr.st_uid = randint(0, 2**32)
attr.st_gid = randint(0, 2**32)
attr.st_atime = randint(0, 2**32) / 10**6
attr.st_mtime = randint(0, 2**32) / 10**6
safe_sleep(CLOCK_GRANULARITY)
self.server.setattr(inode.id, attr)
inode_new = self.server.getattr(inode.id)
self.assertGreater(inode_new.ctime, inode_old.ctime)
for key in attr.__slots__:
if getattr(attr, key) is not None:
self.assertEqual(getattr(attr, key), getattr(inode_new, key))
self.server.forget([(inode.id, 1)])
self.fsck()
def test_truncate(self):
len_ = int(2.7 * self.max_obj_size)
data = self.random_data(len_)
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
attr.st_size = len_ // 2
self.server.setattr(inode.id, attr)
self.assertTrue(self.server.read(fh, 0, len_) == data[:len_ // 2])
attr.st_size = len_
self.server.setattr(inode.id, attr)
self.assertTrue(
self.server.read(fh, 0, len_) == data[:len_ // 2] + b'\0' *
(len_ // 2))
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_truncate_0(self):
len1 = 158
len2 = 133
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, self.random_data(len1))
self.server.release(fh)
self.server.inodes.flush()
fh = self.server.open(inode.id, os.O_RDWR)
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, self.random_data(len2))
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_setxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.setxattr(inode.id, b'my-attr', b'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, b'my-attr'),
b'strabumm!')
self.server.forget([(inode.id, 1)])
self.fsck()
def test_names(self):
name1 = self.newname()
name2 = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, name1, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode.id, 1)])
(fh, inode) = self.server.create(ROOT_INODE, name2, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.setxattr(inode.id, name1, b'strabumm!')
self.fsck()
self.server.removexattr(inode.id, name1)
self.fsck()
self.server.setxattr(inode.id, name1, b'strabumm karacho!!')
self.server.unlink(ROOT_INODE, name1)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_statfs(self):
# Test with zero contents
self.server.statfs()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
self.server.statfs()
# Test with data in file
fh = self.server.open(inode.id, os.O_RDWR)
self.server.write(fh, 0, b'foobar')
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.server.statfs()
def test_symlink(self):
target = self.newname()
name = self.newname()
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
inode = self.server.symlink(ROOT_INODE, name, target, Ctx())
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertEqual(target, self.server.readlink(inode.id))
id_ = self.db.get_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_unlink(self):
name = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, b'some data to deal with')
self.server.release(fh)
# Add extended attributes
self.server.setxattr(inode.id, b'test_xattr', b'42*8')
self.server.forget([(inode.id, 1)])
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.unlink(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id, )))
self.fsck()
def test_rmdir(self):
name = self.newname()
inode = self.server.mkdir(ROOT_INODE, name, self.dir_mode(), Ctx())
self.server.forget([(inode.id, 1)])
inode_p_before = self.server.getattr(ROOT_INODE).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.rmdir(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id, )))
self.fsck()
def test_relink(self):
name = self.newname()
name2 = self.newname()
data = b'some data to deal with'
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.release(fh)
self.server.unlink(ROOT_INODE, name)
self.server.inodes.flush()
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id, )))
self.server.link(inode.id, ROOT_INODE, name2)
self.server.forget([(inode.id, 2)])
inode = self.server.lookup(ROOT_INODE, name2)
fh = self.server.open(inode.id, os.O_RDONLY)
self.assertTrue(self.server.read(fh, 0, len(data)) == data)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_write(self):
len_ = self.max_obj_size
data = self.random_data(len_)
off = self.max_obj_size // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
inode_before = self.server.getattr(inode.id).copy()
safe_sleep(CLOCK_GRANULARITY)
self.server.write(fh, off, data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.mtime, inode_before.mtime)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertEqual(inode_after.size, off + len_)
self.server.write(fh, 0, data)
inode_after = self.server.getattr(inode.id)
self.assertEqual(inode_after.size, off + len_)
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_failsafe(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.cache.clear()
self.assertTrue(self.server.failsafe is False)
datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with self.assertRaises(FUSEError) as cm:
with catch_logmsg('^Backend returned malformed data for',
count=1,
level=logging.ERROR):
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
self.assertTrue(self.server.failsafe)
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with self.assertRaises(FUSEError) as cm:
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
# Release and re-open, now we should be able to access again
self.server.release(fh)
self.server.forget([(inode.id, 1)])
# ..but not write access since we are in failsafe mode
with self.assertRaises(FUSEError) as cm:
self.server.open(inode.id, os.O_RDWR)
self.assertEqual(cm.exception.errno, errno.EPERM)
# ..ready only is fine.
fh = self.server.open(inode.id, os.O_RDONLY)
self.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
self.server.read(fh, 3, len_ // 2)
self.server.cache.clear()
with self.assertRaises(FUSEError) as cm:
with catch_logmsg('^Backend lost block',
count=1,
level=logging.ERROR):
self.server.read(fh, 5, len_ // 2)
self.assertEqual(cm.exception.errno, errno.EIO)
# Don't call fsck, we're missing a block
def test_create_open(self):
name = self.newname()
# Create a new file
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode, 1)])
# Open it atomically
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(),
os.O_RDWR, Ctx())
self.server.release(fh)
self.server.forget([(inode, 1)])
self.fsck()
def test_edit(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, data)
self.server.release(fh)
self.block_cache.clear()
fh = self.server.open(inode.id, os.O_RDWR)
attr = llfuse.EntryAttributes()
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, data[50:])
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.fsck()
def test_copy_tree(self):
ext_attr_name = b'system.foo.brazl'
ext_attr_val = b'schulla dku woumm bramp'
src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(),
Ctx())
dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(),
Ctx())
# Create file
(fh, f1_inode) = self.server.create(src_inode.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
self.server.setxattr(f1_inode.id, ext_attr_name, ext_attr_val)
# Create hardlink
(fh, f2_inode) = self.server.create(src_inode.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
f2_inode = self.server.link(f2_inode.id, src_inode.id,
b'file2_hardlink')
# Create subdirectory
d1_inode = self.server.mkdir(src_inode.id, b'dir1', self.dir_mode(),
Ctx())
d2_inode = self.server.mkdir(d1_inode.id, b'dir2', self.dir_mode(),
Ctx())
# ..with a 3rd hardlink
f2_inode = self.server.link(f2_inode.id, d1_inode.id,
b'file2_hardlink')
# Replicate
self.server.copy_tree(src_inode.id, dst_inode.id)
# Change files
fh = self.server.open(f1_inode.id, os.O_RDWR)
self.server.write(fh, 0, b'new file1 contents')
self.server.release(fh)
fh = self.server.open(f2_inode.id, os.O_RDWR)
self.server.write(fh, 0, b'new file2 contents')
self.server.release(fh)
# Get copy properties
f1_inode_c = self.server.lookup(dst_inode.id, b'file1')
f2_inode_c = self.server.lookup(dst_inode.id, b'file2')
f2h_inode_c = self.server.lookup(dst_inode.id, b'file2_hardlink')
d1_inode_c = self.server.lookup(dst_inode.id, b'dir1')
d2_inode_c = self.server.lookup(d1_inode_c.id, b'dir2')
f2_h_inode_c = self.server.lookup(d1_inode_c.id, b'file2_hardlink')
# Check file1
fh = self.server.open(f1_inode_c.id, os.O_RDWR)
self.assertEqual(self.server.read(fh, 0, 42), b'file1 contents')
self.server.release(fh)
self.assertNotEqual(f1_inode.id, f1_inode_c.id)
self.assertEqual(self.server.getxattr(f1_inode_c.id, ext_attr_name),
ext_attr_val)
# Check file2
fh = self.server.open(f2_inode_c.id, os.O_RDWR)
self.assertTrue(self.server.read(fh, 0, 42) == b'file2 contents')
self.server.release(fh)
self.assertEqual(f2_inode_c.id, f2h_inode_c.id)
self.assertEqual(f2_inode_c.refcount, 3)
self.assertNotEqual(f2_inode.id, f2_inode_c.id)
self.assertEqual(f2_h_inode_c.id, f2_inode_c.id)
# Check subdir1
self.assertNotEqual(d1_inode.id, d1_inode_c.id)
self.assertNotEqual(d2_inode.id, d2_inode_c.id)
self.server.forget(list(self.server.open_inodes.items()))
self.fsck()
def test_copy_tree_2(self):
src_inode = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(),
Ctx())
dst_inode = self.server.mkdir(ROOT_INODE, b'dest', self.dir_mode(),
Ctx())
# Create file
(fh, inode) = self.server.create(src_inode.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'block 1 contents')
self.server.write(fh, self.max_obj_size, b'block 1 contents')
self.server.release(fh)
self.server.forget([(inode.id, 1)])
self.server.copy_tree(src_inode.id, dst_inode.id)
self.server.forget([(src_inode.id, 1), (dst_inode.id, 1)])
self.fsck()
def test_lock_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(),
Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, b'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
# Another file
(fh, inode3) = self.server.create(ROOT_INODE, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.release(fh)
# Lock
self.server.lock_tree(inode1.id)
for i in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertTrue(self.server.inodes[i].locked)
# Remove
with self.assertRaises(FUSEError) as cm:
self.server._remove(inode1.id, b'file1', inode1a.id)
self.assertEqual(cm.exception.errno, errno.EPERM)
# Rename / Replace
with self.assertRaises(FUSEError) as cm:
self.server.rename(ROOT_INODE, b'file1', inode1.id, b'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.rename(inode1.id, b'file1', ROOT_INODE, b'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
# Open
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_RDWR)
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_WRONLY)
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(self.server.open(inode3.id, os.O_WRONLY))
# Write
fh = self.server.open(inode2a.id, os.O_RDONLY)
with self.assertRaises(FUSEError) as cm:
self.server.write(fh, 0, b'foo')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(fh)
# Create
with self.assertRaises(FUSEError) as cm:
self.server._create(inode2.id, b'dir1', self.dir_mode(), os.O_RDWR,
Ctx())
self.assertEqual(cm.exception.errno, errno.EPERM)
# Setattr
with self.assertRaises(FUSEError) as cm:
self.server.setattr(inode2a.id, dict())
self.assertEqual(cm.exception.errno, errno.EPERM)
# xattr
with self.assertRaises(FUSEError) as cm:
self.server.setxattr(inode2.id, b'name', b'value')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.removexattr(inode2.id, b'name')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.forget(list(self.server.open_inodes.items()))
self.fsck()
def test_remove_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, b'source', self.dir_mode(),
Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, b'file1',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, b'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, b'file2',
self.file_mode(), os.O_RDWR, Ctx())
self.server.write(fh, 0, b'file2 contents')
self.server.release(fh)
# Remove
self.server.forget(list(self.server.open_inodes.items()))
self.server.remove_tree(ROOT_INODE, b'source')
for (id_p, name) in ((ROOT_INODE, b'source'), (inode1.id, b'file1'),
(inode1.id, b'dir1'), (inode2.id, b'file2')):
self.assertFalse(
self.db.has_val(
'SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, id_p)))
for id_ in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertFalse(
self.db.has_val('SELECT id FROM inodes WHERE id=?', (id_, )))
self.fsck()
class cache_tests(unittest.TestCase):
def setUp(self):
# Destructors are not guaranteed to run, and we can't unlink
# the file immediately because apsw refers to it by name.
# Therefore, we unlink the file manually in tearDown()
self.dbfile = tempfile.NamedTemporaryFile(delete=False)
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
self.cache = inode_cache.InodeCache(self.db, 0)
def tearDown(self):
self.cache.destroy()
os.unlink(self.dbfile.name)
def test_create(self):
attrs = {
'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime_ns': time_ns(),
'ctime_ns': time_ns(),
'mtime_ns': time_ns()
}
inode = self.cache.create_inode(**attrs)
for key in list(attrs.keys()):
self.assertEqual(attrs[key], getattr(inode, key))
self.assertTrue(
self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id, )))
def test_del(self):
attrs = {
'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime_ns': time_ns(),
'ctime_ns': time_ns(),
'mtime_ns': time_ns()
}
inode = self.cache.create_inode(**attrs)
del self.cache[inode.id]
self.assertFalse(
self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id, )))
self.assertRaises(KeyError, self.cache.__delitem__, inode.id)
def test_get(self):
attrs = {
'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime_ns': time_ns(),
'ctime_ns': time_ns(),
'mtime_ns': time_ns()
}
inode = self.cache.create_inode(**attrs)
for (key, val) in attrs.items():
self.assertEqual(getattr(inode, key), val)
# Create another inode
self.cache.create_inode(**attrs)
self.db.execute('DELETE FROM inodes WHERE id=?', (inode.id, ))
# Entry should still be in cache
self.assertEqual(inode, self.cache[inode.id])
# Now it should be out of the cache
for _ in range(inode_cache.CACHE_SIZE + 1):
self.cache.create_inode(**attrs)
self.assertRaises(KeyError, self.cache.__getitem__, inode.id)
class fs_api_tests(TestCase):
def setUp(self):
self.bucket_dir = tempfile.mkdtemp()
self.bucket_pool = BucketPool(lambda: local.Bucket(self.bucket_dir, None, None))
self.bucket = self.bucket_pool.pop_conn()
self.cachedir = tempfile.mkdtemp() + "/"
self.blocksize = 1024
self.dbfile = tempfile.NamedTemporaryFile()
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
# Tested methods assume that they are called from
# file system request handler
llfuse.lock.acquire()
self.block_cache = BlockCache(self.bucket_pool, self.db, self.cachedir,
self.blocksize * 5)
self.server = fs.Operations(self.block_cache, self.db, self.blocksize)
self.server.init()
# Keep track of unused filenames
self.name_cnt = 0
def tearDown(self):
self.server.destroy()
self.block_cache.destroy()
if os.path.exists(self.cachedir):
shutil.rmtree(self.cachedir)
shutil.rmtree(self.bucket_dir)
llfuse.lock.release()
@staticmethod
def random_data(len_):
with open("/dev/urandom", "rb") as fd:
return fd.read(len_)
def fsck(self):
self.block_cache.clear()
self.server.inodes.flush()
fsck = Fsck(self.cachedir, self.bucket,
{ 'blocksize': self.blocksize }, self.db)
fsck.check()
self.assertFalse(fsck.found_errors)
def newname(self):
self.name_cnt += 1
return "s3ql_%d" % self.name_cnt
def test_getattr_root(self):
self.assertTrue(stat.S_ISDIR(self.server.getattr(ROOT_INODE).mode))
self.fsck()
def test_create(self):
ctx = Ctx()
mode = self.dir_mode()
name = self.newname()
inode_p_old = self.server.getattr(ROOT_INODE).copy()
time.sleep(CLOCK_GRANULARITY)
self.server._create(ROOT_INODE, name, mode, ctx)
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON name_id = names.id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
inode = self.server.getattr(id_)
self.assertEqual(inode.mode, mode)
self.assertEqual(inode.uid, ctx.uid)
self.assertEqual(inode.gid, ctx.gid)
self.assertEqual(inode.refcount, 1)
self.assertEqual(inode.size, 0)
inode_p_new = self.server.getattr(ROOT_INODE)
self.assertGreater(inode_p_new.mtime, inode_p_old.mtime)
self.assertGreater(inode_p_new.ctime, inode_p_old.ctime)
self.fsck()
def test_extstat(self):
# Test with zero contents
self.server.extstat()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.server.extstat()
# Test with data in file
fh = self.server.open(inode.id, os.O_RDWR)
self.server.write(fh, 0, 'foobar')
self.server.release(fh)
self.server.extstat()
self.fsck()
@staticmethod
def dir_mode():
return (randint(0, 07777) & ~stat.S_IFDIR) | stat.S_IFDIR
@staticmethod
def file_mode():
return (randint(0, 07777) & ~stat.S_IFREG) | stat.S_IFREG
def test_getxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.getxattr, inode.id, 'nonexistant-attr')
self.server.setxattr(inode.id, 'my-attr', 'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, 'my-attr'), 'strabumm!')
self.fsck()
def test_link(self):
name = self.newname()
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(),
self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
time.sleep(CLOCK_GRANULARITY)
inode_before = self.server.getattr(inode.id).copy()
self.server.link(inode.id, inode_p_new.id, name)
inode_after = self.server.lookup(inode_p_new.id, name)
inode_p_new_after = self.server.getattr(inode_p_new.id)
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, inode_p_new.id))
self.assertEqual(inode_before.id, id_)
self.assertEqual(inode_after.refcount, 2)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.fsck()
def test_listxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.assertListEqual([], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, 'key1', 'blub')
self.assertListEqual(['key1'], self.server.listxattr(inode.id))
self.server.setxattr(inode.id, 'key2', 'blub')
self.assertListEqual(sorted(['key1', 'key2']),
sorted(self.server.listxattr(inode.id)))
self.fsck()
def test_read(self):
len_ = self.blocksize
data = self.random_data(len_)
off = self.blocksize // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.write(fh, off, data)
inode_before = self.server.getattr(inode.id).copy()
time.sleep(CLOCK_GRANULARITY)
self.assertTrue(self.server.read(fh, off, len_) == data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.atime, inode_before.atime)
self.assertTrue(self.server.read(fh, 0, len_) == b"\0" * off + data[:off])
self.assertTrue(self.server.read(fh, self.blocksize, len_) == data[off:])
self.server.release(fh)
self.fsck()
def test_readdir(self):
# Create a few entries
names = [ 'entry_%2d' % i for i in range(20) ]
for name in names:
(fh, _) = self.server.create(ROOT_INODE, name,
self.file_mode(), Ctx())
self.server.release(fh)
# Delete some to make sure that we don't have continous rowids
remove_no = [0, 2, 3, 5, 9]
for i in remove_no:
self.server.unlink(ROOT_INODE, names[i])
del names[i]
# Read all
fh = self.server.opendir(ROOT_INODE)
self.assertListEqual(sorted(names + ['lost+found']) ,
sorted(x[0] for x in self.server.readdir(fh, 0)))
self.server.releasedir(fh)
# Read in parts
fh = self.server.opendir(ROOT_INODE)
entries = list()
try:
next_ = 0
while True:
gen = self.server.readdir(fh, next_)
for _ in range(3):
(name, _, next_) = next(gen)
entries.append(name)
except StopIteration:
pass
self.assertListEqual(sorted(names + ['lost+found']) ,
sorted(entries))
self.server.releasedir(fh)
self.fsck()
def test_release(self):
name = self.newname()
# Test that entries are deleted when they're no longer referenced
(fh, inode) = self.server.create(ROOT_INODE, name,
self.file_mode(), Ctx())
self.server.write(fh, 0, 'foobar')
self.server.unlink(ROOT_INODE, name)
self.assertFalse(self.db.has_val('SELECT 1 FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(self.server.getattr(inode.id).id)
self.server.release(fh)
self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_removexattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.assertRaises(FUSEError, self.server.removexattr, inode.id, 'some name')
self.server.setxattr(inode.id, 'key1', 'blub')
self.server.removexattr(inode.id, 'key1')
self.assertListEqual([], self.server.listxattr(inode.id))
self.fsck()
def test_rename(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
time.sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id == name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.fsck()
def test_replace_file(self):
oldname = self.newname()
newname = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, oldname, self.file_mode(), Ctx())
self.server.write(fh, 0, 'some data to deal with')
self.server.release(fh)
self.server.setxattr(inode.id, 'test_xattr', '42*8')
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
(fh, inode2) = self.server.create(inode_p_new.id, newname, self.file_mode(), Ctx())
self.server.write(fh, 0, 'even more data to deal with')
self.server.release(fh)
self.server.setxattr(inode2.id, 'test_xattr', '42*8')
time.sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id,)))
self.fsck()
def test_replace_dir(self):
oldname = self.newname()
newname = self.newname()
inode = self.server.mkdir(ROOT_INODE, oldname, self.dir_mode(), Ctx())
inode_p_new = self.server.mkdir(ROOT_INODE, self.newname(), self.dir_mode(), Ctx())
inode_p_new_before = self.server.getattr(inode_p_new.id).copy()
inode_p_old_before = self.server.getattr(ROOT_INODE).copy()
inode2 = self.server.mkdir(inode_p_new.id, newname, self.dir_mode(), Ctx())
time.sleep(CLOCK_GRANULARITY)
self.server.rename(ROOT_INODE, oldname, inode_p_new.id, newname)
inode_p_old_after = self.server.getattr(ROOT_INODE)
inode_p_new_after = self.server.getattr(inode_p_new.id)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (oldname, ROOT_INODE)))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (newname, inode_p_new.id))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_new_before.mtime, inode_p_new_after.mtime)
self.assertLess(inode_p_new_before.ctime, inode_p_new_after.ctime)
self.assertLess(inode_p_old_before.mtime, inode_p_old_after.mtime)
self.assertLess(inode_p_old_before.ctime, inode_p_old_after.ctime)
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode2.id,)))
self.fsck()
def test_setattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(), 0641, Ctx())
self.server.release(fh)
inode_old = self.server.getattr(inode.id).copy()
attr = llfuse.EntryAttributes()
attr.st_mode = self.file_mode()
attr.st_uid = randint(0, 2 ** 32)
attr.st_gid = randint(0, 2 ** 32)
attr.st_rdev = randint(0, 2 ** 32)
attr.st_atime = time.timezone + randint(0, 2 ** 32) / 10 ** 6
attr.st_mtime = time.timezone + randint(0, 2 ** 32) / 10 ** 6
time.sleep(CLOCK_GRANULARITY)
self.server.setattr(inode.id, attr)
inode_new = self.server.getattr(inode.id)
self.assertGreater(inode_new.ctime, inode_old.ctime)
for key in attr.__slots__:
if getattr(attr, key) is not None:
self.assertEquals(getattr(attr, key),
getattr(inode_new, key))
def test_truncate(self):
len_ = int(2.7 * self.blocksize)
data = self.random_data(len_)
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(), self.file_mode(), Ctx())
self.server.write(fh, 0, data)
attr.st_size = len_ // 2
self.server.setattr(inode.id, attr)
self.assertTrue(self.server.read(fh, 0, len_) == data[:len_ // 2])
attr.st_size = len_
self.server.setattr(inode.id, attr)
self.assertTrue(self.server.read(fh, 0, len_)
== data[:len_ // 2] + b'\0' * (len_ // 2))
self.server.release(fh)
self.fsck()
def test_truncate_0(self):
len1 = 158
len2 = 133
attr = llfuse.EntryAttributes()
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.write(fh, 0, self.random_data(len1))
self.server.release(fh)
self.server.inodes.flush()
fh = self.server.open(inode.id, os.O_RDWR)
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, self.random_data(len2))
self.server.release(fh)
self.fsck()
def test_setxattr(self):
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.server.setxattr(inode.id, 'my-attr', 'strabumm!')
self.assertEqual(self.server.getxattr(inode.id, 'my-attr'), 'strabumm!')
self.fsck()
def test_statfs(self):
# Test with zero contents
self.server.statfs()
# Test with empty file
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.release(fh)
self.server.statfs()
# Test with data in file
fh = self.server.open(inode.id, None)
self.server.write(fh, 0, 'foobar')
self.server.release(fh)
self.server.statfs()
def test_symlink(self):
target = self.newname()
name = self.newname()
inode_p_before = self.server.getattr(ROOT_INODE).copy()
time.sleep(CLOCK_GRANULARITY)
inode = self.server.symlink(ROOT_INODE, name, target, Ctx())
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertEqual(target, self.server.readlink(inode.id))
id_ = self.db.get_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE))
self.assertEqual(inode.id, id_)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
def test_unlink(self):
name = self.newname()
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), Ctx())
self.server.write(fh, 0, 'some data to deal with')
self.server.release(fh)
# Add extended attributes
self.server.setxattr(inode.id, 'test_xattr', '42*8')
inode_p_before = self.server.getattr(ROOT_INODE).copy()
time.sleep(CLOCK_GRANULARITY)
self.server.unlink(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_rmdir(self):
name = self.newname()
inode = self.server.mkdir(ROOT_INODE, name, self.dir_mode(), Ctx())
inode_p_before = self.server.getattr(ROOT_INODE).copy()
time.sleep(CLOCK_GRANULARITY)
self.server.rmdir(ROOT_INODE, name)
inode_p_after = self.server.getattr(ROOT_INODE)
self.assertLess(inode_p_before.mtime, inode_p_after.mtime)
self.assertLess(inode_p_before.ctime, inode_p_after.ctime)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.fsck()
def test_relink(self):
name = self.newname()
name2 = self.newname()
data = 'some data to deal with'
(fh, inode) = self.server.create(ROOT_INODE, name, self.file_mode(), Ctx())
self.server.write(fh, 0, data)
self.server.unlink(ROOT_INODE, name)
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, ROOT_INODE)))
self.assertTrue(self.db.has_val('SELECT id FROM inodes WHERE id=?', (inode.id,)))
self.server.link(inode.id, ROOT_INODE, name2)
self.server.release(fh)
fh = self.server.open(inode.id, os.O_RDONLY)
self.assertTrue(self.server.read(fh, 0, len(data)) == data)
self.server.release(fh)
self.fsck()
def test_write(self):
len_ = self.blocksize
data = self.random_data(len_)
off = self.blocksize // 2
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
inode_before = self.server.getattr(inode.id).copy()
time.sleep(CLOCK_GRANULARITY)
self.server.write(fh, off, data)
inode_after = self.server.getattr(inode.id)
self.assertGreater(inode_after.mtime, inode_before.mtime)
self.assertGreater(inode_after.ctime, inode_before.ctime)
self.assertEqual(inode_after.size, off + len_)
self.server.write(fh, 0, data)
inode_after = self.server.getattr(inode.id)
self.assertEqual(inode_after.size, off + len_)
self.server.release(fh)
self.fsck()
def test_edit(self):
len_ = self.blocksize
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), Ctx())
self.server.write(fh, 0, data)
self.server.release(fh)
self.block_cache.clear()
fh = self.server.open(inode.id, os.O_RDWR)
attr = llfuse.EntryAttributes()
attr.st_size = 0
self.server.setattr(inode.id, attr)
self.server.write(fh, 0, data[50:])
self.server.release(fh)
self.fsck()
def test_copy_tree(self):
src_inode = self.server.mkdir(ROOT_INODE, 'source', self.dir_mode(), Ctx())
dst_inode = self.server.mkdir(ROOT_INODE, 'dest', self.dir_mode(), Ctx())
# Create file
(fh, f1_inode) = self.server.create(src_inode.id, 'file1',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file1 contents')
self.server.release(fh)
# Create hardlink
(fh, f2_inode) = self.server.create(src_inode.id, 'file2',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file2 contents')
self.server.release(fh)
f2_inode = self.server.link(f2_inode.id, src_inode.id, 'file2_hardlink')
# Create subdirectory
d1_inode = self.server.mkdir(src_inode.id, 'dir1', self.dir_mode(), Ctx())
d2_inode = self.server.mkdir(d1_inode.id, 'dir2', self.dir_mode(), Ctx())
# ..with a 3rd hardlink
f2_inode = self.server.link(f2_inode.id, d1_inode.id, 'file2_hardlink')
# Replicate
self.server.copy_tree(src_inode.id, dst_inode.id)
# Change files
fh = self.server.open(f1_inode.id, os.O_RDWR)
self.server.write(fh, 0, 'new file1 contents')
self.server.release(fh)
fh = self.server.open(f2_inode.id, os.O_RDWR)
self.server.write(fh, 0, 'new file2 contents')
self.server.release(fh)
# Get copy properties
f1_inode_c = self.server.lookup(dst_inode.id, 'file1')
f2_inode_c = self.server.lookup(dst_inode.id, 'file2')
f2h_inode_c = self.server.lookup(dst_inode.id, 'file2_hardlink')
d1_inode_c = self.server.lookup(dst_inode.id, 'dir1')
d2_inode_c = self.server.lookup(d1_inode_c.id, 'dir2')
f2_h_inode_c = self.server.lookup(d1_inode_c.id, 'file2_hardlink')
# Check file1
fh = self.server.open(f1_inode_c.id, os.O_RDWR)
self.assertEqual(self.server.read(fh, 0, 42), 'file1 contents')
self.server.release(fh)
self.assertNotEqual(f1_inode.id, f1_inode_c.id)
# Check file2
fh = self.server.open(f2_inode_c.id, os.O_RDWR)
self.assertTrue(self.server.read(fh, 0, 42) == 'file2 contents')
self.server.release(fh)
self.assertEqual(f2_inode_c.id, f2h_inode_c.id)
self.assertEqual(f2_inode_c.refcount, 3)
self.assertNotEqual(f2_inode.id, f2_inode_c.id)
self.assertEqual(f2_h_inode_c.id, f2_inode_c.id)
# Check subdir1
self.assertNotEqual(d1_inode.id, d1_inode_c.id)
self.assertNotEqual(d2_inode.id, d2_inode_c.id)
self.fsck()
def test_lock_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, 'source', self.dir_mode(), Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, 'file1',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, 'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, 'file2',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file2 contents')
self.server.release(fh)
# Another file
(fh, inode3) = self.server.create(ROOT_INODE, 'file1',
self.file_mode(), Ctx())
self.server.release(fh)
# Lock
self.server.lock_tree(inode1.id)
for i in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertTrue(self.server.inodes[i].locked)
# Remove
with self.assertRaises(FUSEError) as cm:
self.server._remove(inode1.id, 'file1', inode1a.id)
self.assertEqual(cm.exception.errno, errno.EPERM)
# Rename / Replace
with self.assertRaises(FUSEError) as cm:
self.server.rename(ROOT_INODE, 'file1', inode1.id, 'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.rename(inode1.id, 'file1', ROOT_INODE, 'file2')
self.assertEqual(cm.exception.errno, errno.EPERM)
# Open
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_RDWR)
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.open(inode2a.id, os.O_WRONLY)
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(self.server.open(inode3.id, os.O_WRONLY))
# Write
fh = self.server.open(inode2a.id, os.O_RDONLY)
with self.assertRaises(FUSEError) as cm:
self.server.write(fh, 0, 'foo')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.server.release(fh)
# Create
with self.assertRaises(FUSEError) as cm:
self.server._create(inode2.id, 'dir1', self.dir_mode(), Ctx())
self.assertEqual(cm.exception.errno, errno.EPERM)
# Setattr
with self.assertRaises(FUSEError) as cm:
self.server.setattr(inode2a.id, dict())
self.assertEqual(cm.exception.errno, errno.EPERM)
# xattr
with self.assertRaises(FUSEError) as cm:
self.server.setxattr(inode2.id, 'name', 'value')
self.assertEqual(cm.exception.errno, errno.EPERM)
with self.assertRaises(FUSEError) as cm:
self.server.removexattr(inode2.id, 'name')
self.assertEqual(cm.exception.errno, errno.EPERM)
self.fsck()
def test_remove_tree(self):
inode1 = self.server.mkdir(ROOT_INODE, 'source', self.dir_mode(), Ctx())
# Create file
(fh, inode1a) = self.server.create(inode1.id, 'file1',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file1 contents')
self.server.release(fh)
# Create subdirectory
inode2 = self.server.mkdir(inode1.id, 'dir1', self.dir_mode(), Ctx())
(fh, inode2a) = self.server.create(inode2.id, 'file2',
self.file_mode(), Ctx())
self.server.write(fh, 0, 'file2 contents')
self.server.release(fh)
# Remove
self.server.remove_tree(ROOT_INODE, 'source')
for (id_p, name) in ((ROOT_INODE, 'source'),
(inode1.id, 'file1'),
(inode1.id, 'dir1'),
(inode2.id, 'file2')):
self.assertFalse(self.db.has_val('SELECT inode FROM contents JOIN names ON names.id = name_id '
'WHERE name=? AND parent_inode = ?', (name, id_p)))
for id_ in (inode1.id, inode1a.id, inode2.id, inode2a.id):
self.assertFalse(self.db.has_val('SELECT id FROM inodes WHERE id=?', (id_,)))
self.fsck()
class fsck_tests(unittest.TestCase):
def setUp(self):
self.backend_dir = tempfile.mkdtemp(prefix='s3ql-backend-')
self.backend = local.Backend('local://' + self.backend_dir, None, None)
self.cachedir = tempfile.mkdtemp(prefix='s3ql-cache-')
self.max_obj_size = 1024
self.dbfile = tempfile.NamedTemporaryFile()
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
self.fsck = Fsck(self.cachedir, self.backend,
{ 'max_obj_size': self.max_obj_size }, self.db)
self.fsck.expect_errors = True
def tearDown(self):
shutil.rmtree(self.cachedir)
shutil.rmtree(self.backend_dir)
self.dbfile.close()
def assert_fsck(self, fn):
'''Check that fn detects and corrects an error'''
self.fsck.found_errors = False
fn()
self.assertTrue(self.fsck.found_errors)
self.fsck.found_errors = False
self.fsck.check()
self.assertFalse(self.fsck.found_errors)
def test_cache(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
| stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(),
1, 8))
self._link(b'test-entry', inode)
# Create new block
fh = open(self.cachedir + '/%d-0' % inode, 'wb')
fh.write(b'somedata')
fh.close()
self.assert_fsck(self.fsck.check_cache)
self.assertEqual(self.backend['s3ql_data_1'], b'somedata')
# Existing block
self.db.execute('UPDATE inodes SET size=? WHERE id=?',
(self.max_obj_size + 8, inode))
with open(self.cachedir + '/%d-1' % inode, 'wb') as fh:
fh.write(b'somedata')
self.assert_fsck(self.fsck.check_cache)
# Old block preserved
with open(self.cachedir + '/%d-0' % inode, 'wb') as fh:
fh.write(b'somedat2')
self.assert_fsck(self.fsck.check_cache)
# Old block removed
with open(self.cachedir + '/%d-1' % inode, 'wb') as fh:
fh.write(b'somedat3')
self.assert_fsck(self.fsck.check_cache)
def test_lof1(self):
# Make lost+found a file
inode = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?",
(b"lost+found", ROOT_INODE))
self.db.execute('DELETE FROM contents WHERE parent_inode=?', (inode,))
self.db.execute('UPDATE inodes SET mode=?, size=? WHERE id=?',
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, 0, inode))
def check():
self.fsck.check_lof()
self.fsck.check_inodes_refcount()
self.assert_fsck(check)
def test_lof2(self):
# Remove lost+found
name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (b'lost+found',))
inode = self.db.get_val('SELECT inode FROM contents WHERE name_id=? AND '
'parent_inode=?', (name_id, ROOT_INODE))
self.db.execute('DELETE FROM inodes WHERE id=?', (inode,))
self.db.execute('DELETE FROM contents WHERE name_id=? and parent_inode=?',
(name_id, ROOT_INODE))
self.db.execute('UPDATE names SET refcount = refcount-1 WHERE id=?', (name_id,))
self.assert_fsck(self.fsck.check_lof)
def test_wrong_inode_refcount(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 0))
self._link(b'name1', inode)
self._link(b'name2', inode)
self.assert_fsck(self.fsck.check_inodes_refcount)
def test_orphaned_inode(self):
self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 0))
self.assert_fsck(self.fsck.check_inodes_refcount)
def test_name_refcount(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 2, 0))
self._link(b'name1', inode)
self._link(b'name2', inode)
self.db.execute('UPDATE names SET refcount=refcount+1 WHERE name=?', (b'name1',))
self.assert_fsck(self.fsck.check_names_refcount)
def test_orphaned_name(self):
self._add_name(b'zupbrazl')
self.assert_fsck(self.fsck.check_names_refcount)
def test_contents_inode(self):
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foobar'), 124, ROOT_INODE))
self.assert_fsck(self.fsck.check_contents_inode)
def test_contents_inode_p(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 0))
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foobar'), inode, 123))
self.assert_fsck(self.fsck.check_contents_parent_inode)
def test_contents_name(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 0))
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(42, inode, ROOT_INODE))
self.assert_fsck(self.fsck.check_contents_name)
def _add_name(self, name):
'''Get id for *name* and increase refcount
Name is inserted in table if it does not yet exist.
'''
try:
name_id = self.db.get_val('SELECT id FROM names WHERE name=?', (name,))
except NoSuchRowError:
name_id = self.db.rowid('INSERT INTO names (name, refcount) VALUES(?,?)',
(name, 1))
else:
self.db.execute('UPDATE names SET refcount=refcount+1 WHERE id=?', (name_id,))
return name_id
def _link(self, name, inode, parent_inode=ROOT_INODE):
'''Link /*name* to *inode*'''
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(name), inode, parent_inode))
def test_inodes_size(self):
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 128))
self._link(b'test-entry', id_)
obj_id = self.db.rowid('INSERT INTO objects (refcount,size) VALUES(?,?)', (1, 36))
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 512, sha256(b'foo')))
self.backend['s3ql_data_%d' % obj_id] = b'foo'
# Case 1
self.db.execute('UPDATE inodes SET size=? WHERE id=?', (self.max_obj_size + 120, id_))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 1, block_id))
self.assert_fsck(self.fsck.check_inodes_size)
# Case 2
self.db.execute('DELETE FROM inode_blocks WHERE inode=?', (id_,))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 0, block_id))
self.db.execute('UPDATE inodes SET size=? WHERE id=?', (129, id_))
self.assert_fsck(self.fsck.check_inodes_size)
# Case 3
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 1, block_id))
self.db.execute('UPDATE inodes SET size=? WHERE id=?',
(self.max_obj_size + 120, id_))
self.db.execute('UPDATE blocks SET refcount = refcount + 1 WHERE id = ?',
(block_id,))
self.assert_fsck(self.fsck.check_inodes_size)
def test_objects_id(self):
# Create an object that only exists in the backend
self.backend['s3ql_data_4364'] = b'Testdata'
self.assert_fsck(self.fsck.check_objects_id)
# Create an object that does not exist in the backend
self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)', (34, 1, 27))
self.assert_fsck(self.fsck.check_objects_id)
def test_blocks_checksum(self):
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 8))
self._link(b'test-entry', id_)
# Assume that due to a crash we did not write the hash for the block
self.backend['s3ql_data_4364'] = b'Testdata'
self.db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(4364, 1, 8))
block_id = self.db.execute('INSERT INTO blocks (obj_id, refcount, size) VALUES(?, ?, ?)',
(4364, 1, 8))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?, ?, ?)',
(id_, 0, block_id))
# Should pick up wrong hash and delete objects
self.fsck.found_errors = False
self.fsck.check_blocks_checksum()
assert self.fsck.found_errors
self.fsck.found_errors = False
self.fsck.check_blocks_checksum()
assert not self.fsck.found_errors
# Should save files in lost+found
self.fsck.found_errors = False
self.fsck.check()
assert self.fsck.found_errors
# Now everything should be good
self.fsck.found_errors = False
self.fsck.check()
assert not self.fsck.found_errors
assert not self.db.has_val('SELECT block_id FROM inode_blocks WHERE inode=?',
(id_,))
inode_p = self.db.get_val('SELECT parent_inode FROM contents_v WHERE inode=?', (id_,))
lof_id = self.db.get_val("SELECT inode FROM contents_v WHERE name=? AND parent_inode=?",
(b"lost+found", ROOT_INODE))
assert inode_p == lof_id
def test_blocks_obj_id(self):
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, 48, 128))
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 128))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, block_id))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_blocks_obj_id)
def test_missing_obj(self):
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 32)')
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 128))
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 128))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, block_id))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_objects_id)
def test_inode_blocks_inode(self):
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 42)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 34))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(27, 0, block_id))
self.assert_fsck(self.fsck.check_inode_blocks_inode)
def test_inode_blocks_block_id(self):
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 128))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(id_, 0, 35))
self._link(b'test-entry', id_)
self.assert_fsck(self.fsck.check_inode_blocks_block_id)
def test_symlinks_inode(self):
self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)',
(42, b'somewhere else'))
self.assert_fsck(self.fsck.check_symlinks_inode)
def test_ext_attrs_inode(self):
self.db.execute('INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)',
(self._add_name(b'some name'), 34, b'some value'))
self.assert_fsck(self.fsck.check_ext_attributes_inode)
def test_ext_attrs_name(self):
id_ = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1, 128))
self._link(b'test-entry', id_)
self.db.execute('INSERT INTO ext_attributes (name_id, inode, value) VALUES(?,?,?)',
(34, id_, b'some value'))
self.assert_fsck(self.fsck.check_ext_attributes_name)
@staticmethod
def random_data(len_):
with open("/dev/urandom", "rb") as fd:
return fd.read(len_)
def test_loops(self):
# Create some directory inodes
inodes = [ self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFDIR | stat.S_IRUSR | stat.S_IWUSR,
0, 0, time.time(), time.time(), time.time(), 1))
for dummy in range(3) ]
inodes.append(inodes[0])
last = inodes[0]
for inode in inodes[1:]:
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?, ?, ?)',
(self._add_name(str(inode).encode()), inode, last))
last = inode
self.assert_fsck(self.fsck.check_loops)
def test_tmpfile(self):
# Ensure that path exists
objname = 's3ql_data_38375'
self.backend[objname] = b'bla'
del self.backend[objname]
path = self.backend._key_to_path(objname)
tmpname = '%s#%d-%d.tmp' % (path, os.getpid(), _thread.get_ident())
with open(tmpname, 'wb') as fh:
fh.write(b'Hello, world')
self.assert_fsck(self.fsck.check_objects_temp)
def test_obj_refcounts(self):
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 42)')
block_id_1 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'foo')))
block_id_2 = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'bar')))
self.backend['s3ql_data_%d' % obj_id] = b'foo and bar'
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(),
1, 2048))
self._link(b'test-entry', inode)
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id_1))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 2, block_id_2))
self.assert_fsck(self.fsck.check_objects_refcount)
def test_orphaned_obj(self):
self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 33)')
self.assert_fsck(self.fsck.check_objects_refcount)
def test_wrong_block_refcount(self):
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 23)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) '
'VALUES(?,?,?,?)', (1, obj_id, 0, sha256(b'')))
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR, os.getuid(), os.getgid(),
time.time(), time.time(), time.time(), 1, self.max_obj_size))
self._link(b'test-entry', inode)
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 0, block_id))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id))
self.assert_fsck(self.fsck.check_blocks_refcount)
def test_orphaned_block(self):
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 24)')
self.backend['s3ql_data_%d' % obj_id] = b'foo'
self.db.rowid('INSERT INTO blocks (refcount, obj_id, size, hash) VALUES(?,?,?,?)',
(1, obj_id, 3, sha256(b'xyz')))
self.assert_fsck(self.fsck.check_blocks_refcount)
def test_unix_size(self):
inode = 42
self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1, 0))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET size = 1 WHERE id=?', (inode,))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_size_symlink(self):
inode = 42
target = b'some funny random string'
self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount,size) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(inode, stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1,
len(target)))
self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, target))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET size = 0 WHERE id=?', (inode,))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_target(self):
inode = 42
self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?,?)",
(inode, stat.S_IFCHR | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('INSERT INTO symlink_targets (inode, target) VALUES(?,?)', (inode, 'foo'))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_nomode_reg(self):
perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP
stamp = time.time()
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1))
self._link(b'test-entry', inode)
self.assert_fsck(self.fsck.check_unix)
newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?', (inode,))
self.assertEqual(stat.S_IMODE(newmode), perms)
self.assertEqual(stat.S_IFMT(newmode), stat.S_IFREG)
def test_unix_nomode_dir(self):
perms = stat.S_IRUSR | stat.S_IWUSR | stat.S_IROTH | stat.S_IRGRP
stamp = time.time()
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(perms, os.getuid(), os.getgid(), stamp, stamp, stamp, 1))
inode2 = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(perms | stat.S_IFREG, os.getuid(), os.getgid(), stamp,
stamp, stamp, 1))
self._link(b'test-entry', inode)
self._link(b'subentry', inode2, inode)
self.assert_fsck(self.fsck.check_unix)
newmode = self.db.get_val('SELECT mode FROM inodes WHERE id=?', (inode,))
self.assertEqual(stat.S_IMODE(newmode), perms)
self.assertEqual(stat.S_IFMT(newmode), stat.S_IFDIR)
def test_unix_symlink_no_target(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFLNK | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1))
self._link(b'test-entry', inode)
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_rdev(self):
inode = 42
self.db.execute("INSERT INTO inodes (id, mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?,?)",
(inode, stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('UPDATE inodes SET rdev=? WHERE id=?', (42, inode))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_child(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFREG | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
self.db.execute('INSERT INTO contents (name_id, inode, parent_inode) VALUES(?,?,?)',
(self._add_name(b'foo'), ROOT_INODE, inode))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
def test_unix_blocks(self):
inode = self.db.rowid("INSERT INTO inodes (mode,uid,gid,mtime,atime,ctime,refcount) "
"VALUES (?,?,?,?,?,?,?)",
(stat.S_IFSOCK | stat.S_IRUSR | stat.S_IWUSR,
os.getuid(), os.getgid(), time.time(), time.time(), time.time(), 1))
self._link(b'test-entry', inode)
self.fsck.found_errors = False
self.fsck.check_unix()
self.assertFalse(self.fsck.found_errors)
obj_id = self.db.rowid('INSERT INTO objects (refcount, size) VALUES(1, 32)')
block_id = self.db.rowid('INSERT INTO blocks (refcount, obj_id, size) VALUES(?,?,?)',
(1, obj_id, 0))
self.db.execute('INSERT INTO inode_blocks (inode, blockno, block_id) VALUES(?,?,?)',
(inode, 1, block_id))
self.fsck.check_unix()
self.assertTrue(self.fsck.found_errors)
class cache_tests(TestCase):
def setUp(self):
self.dbfile = tempfile.NamedTemporaryFile()
self.db = Connection(self.dbfile.name)
create_tables(self.db)
init_tables(self.db)
self.cache = inode_cache.InodeCache(self.db)
def tearDown(self):
self.cache.destroy()
def test_create(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
for key in attrs.keys():
self.assertEqual(attrs[key], getattr(inode, key))
self.assertTrue(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
def test_del(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
del self.cache[inode.id]
self.assertFalse(self.db.has_val('SELECT 1 FROM inodes WHERE id=?', (inode.id,)))
self.assertRaises(KeyError, self.cache.__delitem__, inode.id)
def test_get(self):
attrs = {'mode': 784,
'refcount': 3,
'uid': 7,
'gid': 2,
'size': 34674,
'rdev': 11,
'atime': time.time(),
'ctime': time.time(),
'mtime': time.time() }
inode = self.cache.create_inode(**attrs)
for (key, val) in attrs.iteritems():
self.assertEqual(getattr(inode, key), val)
# Create another inode
self.cache.create_inode(**attrs)
self.db.execute('DELETE FROM inodes WHERE id=?', (inode.id,))
# Entry should still be in cache
self.assertEqual(inode, self.cache[inode.id])
# Now it should be out of the cache
for _ in xrange(inode_cache.CACHE_SIZE + 1):
self.cache.create_inode(**attrs)
self.assertRaises(KeyError, self.cache.__getitem__, inode.id)
