def test_corrupted_meta(backend, monkeypatch):
key = 'brafasel'
value = b'hello there, let us see whats going on'
backend[key] = value
# Monkeypatch request handler to mess up metadata
handler_class = mock_server.S3CRequestHandler
def send_header(self,
keyword,
value,
count=[0],
send_header_real=handler_class.send_header):
if keyword == self.hdr_prefix + 'Meta-md5':
count[0] += 1
if count[0] <= 3:
value = value[::-1]
return send_header_real(self, keyword, value)
monkeypatch.setattr(handler_class, 'send_header', send_header)
with assert_logs('^MD5 mismatch in metadata for',
count=1,
level=logging.WARNING):
assert_raises(BadDigestError, backend.fetch, key)
enable_temp_fail(backend)
with assert_logs('^MD5 mismatch in metadata for',
count=2,
level=logging.WARNING):
assert backend[key] == value
def test_retrieve(backend, db):
plain_backend = backend
backend = ComprencBackend(b'schnorz', ('zlib', 6), plain_backend)
# Create a few objects in db
obj_ids = (22, 25, 30, 31)
for id_ in obj_ids:
db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(id_, 1, 27 * id_))
# Object one will be missing in backend
# Object two will have a checksum error in the data
key = 's3ql_data_%d' % obj_ids[1]
backend[key] = b'some data that will be broken on a data check'
(raw, meta) = plain_backend.fetch(key)
raw = bytearray(raw)
assert len(raw) > 20
raw[-10:-6] = b'forg'
plain_backend.store(key, raw, meta)
# Object three will have a checksum error in the metadata
key = 's3ql_data_%d' % obj_ids[2]
backend.store(key, b'some data that will be broken on a metadata check',
{ 'meta-key1': 'some textual data that just increases',
'meta-key2': 'the metadata size so that we can tamper with it' })
meta = plain_backend.lookup(key)
raw = bytearray(meta['data'])
assert len(raw) > 20
raw[-10:-6] = b'forg'
meta['data'] = raw
plain_backend.update_meta(key, meta)
# Object four will be ok
backend['s3ql_data_%d' % obj_ids[3]] = b'some data that is well'
# When using a single thread, we can fake the backend factory
def backend_factory():
return backend
missing_fh = io.StringIO()
corrupted_fh = io.StringIO()
with assert_logs('^Backend seems to have lost', count=1, level=logging.WARNING), \
assert_logs('^Object %d is corrupted', count=1, level=logging.WARNING):
verify.retrieve_objects(db, backend_factory, corrupted_fh, missing_fh,
thread_count=1, full=False)
assert missing_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[0]
assert corrupted_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[2]
missing_fh = io.StringIO()
corrupted_fh = io.StringIO()
with assert_logs('^Backend seems to have lost', count=1, level=logging.WARNING), \
assert_logs('^Object %d is corrupted', count=2, level=logging.WARNING):
verify.retrieve_objects(db, backend_factory, corrupted_fh, missing_fh,
thread_count=1, full=True)
assert missing_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[0]
assert corrupted_fh.getvalue() == ('s3ql_data_%d\n'*2) % obj_ids[1:3]
async def test_failsafe(ctx):
len_ = ctx.max_obj_size
data = random_data(len_)
(fi, inode) = await ctx.server.create(ROOT_INODE, newname(ctx),
file_mode(), os.O_RDWR, some_ctx)
fh = fi.fh
await ctx.server.write(fh, 0, data)
await ctx.cache.drop()
assert not ctx.server.failsafe
datafile = os.path.join(ctx.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with assert_raises(FUSEError) as cm:
with assert_logs('^Backend returned malformed data for',
count=1, level=logging.ERROR):
await ctx.server.read(fh, 0, len_)
assert cm.value.errno == errno.EIO
assert ctx.server.failsafe
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with assert_raises(FUSEError) as cm:
await ctx.server.read(fh, 0, len_)
assert cm.value.errno == errno.EIO
# Release and re-open, now we should be able to access again
await ctx.server.release(fh)
await ctx.server.forget([(inode.st_ino, 1)])
# ..but not write access since we are in failsafe mode
with assert_raises(FUSEError) as cm:
await ctx.server.open(inode.st_ino, os.O_RDWR, some_ctx)
assert cm.value.errno == errno.EPERM
# ..ready only is fine.
fi = await ctx.server.open(inode.st_ino, os.O_RDONLY, some_ctx)
fh = fi.fh
await ctx.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
await ctx.server.read(fh, 3, len_//2)
await ctx.cache.drop()
with assert_raises(FUSEError) as cm:
with assert_logs('^Backend lost block',
count=1, level=logging.ERROR):
await ctx.server.read(fh, 5, len_//2)
assert cm.value.errno == errno.EIO
def test_failsafe(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, some_ctx)
self.server.write(fh, 0, data)
self.server.cache.clear()
self.assertTrue(self.server.failsafe is False)
datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with self.assertRaises(FUSEError) as cm:
with assert_logs('^Backend returned malformed data for',
count=1,
level=logging.ERROR):
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
self.assertTrue(self.server.failsafe)
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with self.assertRaises(FUSEError) as cm:
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
# Release and re-open, now we should be able to access again
self.server.release(fh)
self.server.forget([(inode.st_ino, 1)])
# ..but not write access since we are in failsafe mode
with self.assertRaises(FUSEError) as cm:
self.server.open(inode.st_ino, os.O_RDWR, some_ctx)
self.assertEqual(cm.exception.errno, errno.EPERM)
# ..ready only is fine.
fh = self.server.open(inode.st_ino, os.O_RDONLY, some_ctx)
self.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
self.server.read(fh, 3, len_ // 2)
self.server.cache.clear()
with self.assertRaises(FUSEError) as cm:
with assert_logs('^Backend lost block',
count=1,
level=logging.ERROR):
self.server.read(fh, 5, len_ // 2)
self.assertEqual(cm.exception.errno, errno.EIO)
def test_failsafe(ctx):
len_ = ctx.max_obj_size
data = random_data(len_)
(fh, inode) = ctx.server.create(ROOT_INODE, newname(ctx),
file_mode(), os.O_RDWR, some_ctx)
ctx.server.write(fh, 0, data)
ctx.server.cache.drop()
assert not ctx.server.failsafe
datafile = os.path.join(ctx.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with assert_raises(FUSEError) as cm:
with assert_logs('^Backend returned malformed data for',
count=1, level=logging.ERROR):
ctx.server.read(fh, 0, len_)
assert cm.value.errno == errno.EIO
assert ctx.server.failsafe
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with assert_raises(FUSEError) as cm:
ctx.server.read(fh, 0, len_)
assert cm.value.errno == errno.EIO
# Release and re-open, now we should be able to access again
ctx.server.release(fh)
ctx.server.forget([(inode.st_ino, 1)])
# ..but not write access since we are in failsafe mode
with assert_raises(FUSEError) as cm:
ctx.server.open(inode.st_ino, os.O_RDWR, some_ctx)
assert cm.value.errno == errno.EPERM
# ..ready only is fine.
fh = ctx.server.open(inode.st_ino, os.O_RDONLY, some_ctx)
ctx.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
ctx.server.read(fh, 3, len_//2)
ctx.server.cache.drop()
with assert_raises(FUSEError) as cm:
with assert_logs('^Backend lost block',
count=1, level=logging.ERROR):
ctx.server.read(fh, 5, len_//2)
assert cm.value.errno == errno.EIO
def test_failsafe(self):
len_ = self.max_obj_size
data = self.random_data(len_)
(fh, inode) = self.server.create(ROOT_INODE, self.newname(),
self.file_mode(), os.O_RDWR, some_ctx)
self.server.write(fh, 0, data)
self.server.cache.clear()
self.assertTrue(self.server.failsafe is False)
datafile = os.path.join(self.backend_dir, 's3ql_data_', 's3ql_data_1')
shutil.copy(datafile, datafile + '.bak')
# Modify contents
with open(datafile, 'rb+') as rfh:
rfh.seek(560)
rfh.write(b'blrub!')
with self.assertRaises(FUSEError) as cm:
with assert_logs('^Backend returned malformed data for',
count=1, level=logging.ERROR):
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
self.assertTrue(self.server.failsafe)
# Restore contents, but should be marked as damaged now
os.rename(datafile + '.bak', datafile)
with self.assertRaises(FUSEError) as cm:
self.server.read(fh, 0, len_)
self.assertEqual(cm.exception.errno, errno.EIO)
# Release and re-open, now we should be able to access again
self.server.release(fh)
self.server.forget([(inode.st_ino, 1)])
# ..but not write access since we are in failsafe mode
with self.assertRaises(FUSEError) as cm:
self.server.open(inode.st_ino, os.O_RDWR, some_ctx)
self.assertEqual(cm.exception.errno, errno.EPERM)
# ..ready only is fine.
fh = self.server.open(inode.st_ino, os.O_RDONLY, some_ctx)
self.server.read(fh, 0, len_)
# Remove completely, should give error after cache flush
os.unlink(datafile)
self.server.read(fh, 3, len_//2)
self.server.cache.clear()
with self.assertRaises(FUSEError) as cm:
with assert_logs('^Backend lost block',
count=1, level=logging.ERROR):
self.server.read(fh, 5, len_//2)
self.assertEqual(cm.exception.errno, errno.EIO)
def test_conn_abort(backend, monkeypatch):
'''Close connection while sending data'''
data = b'hello there, let us see whats going on'
key = 'borg'
backend[key] = data
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
def send_data(self, data, count=[0]):
count[0] += 1
if count[0] >= 3:
self.wfile.write(data)
else:
self.wfile.write(data[:len(data) // 2])
self.close_connection = True
monkeypatch.setattr(handler_class, 'send_data', send_data)
with pytest.raises(ConnectionClosed):
with assert_logs("^Object closed prematurely, can't check MD5",
count=1,
level=logging.WARNING):
backend.fetch(key)
enable_temp_fail(backend)
assert backend[key] == data
def test_conn_abort(backend, monkeypatch):
'''Close connection while sending data'''
data = b'hello there, let us see whats going on'
key = 'borg'
backend[key] = data
# Monkeypatch request handler
handler_class = mock_server.S3CRequestHandler
def send_data(self, data, count=[0]):
count[0] += 1
if count[0] >= 3:
self.wfile.write(data)
else:
self.wfile.write(data[:len(data)//2])
self.close_connection = True
monkeypatch.setattr(handler_class, 'send_data', send_data)
with pytest.raises(ConnectionClosed):
with assert_logs("^Object closed prematurely, can't check MD5",
count=1, level=logging.WARNING):
backend.fetch(key)
enable_temp_fail(backend)
assert backend[key] == data
def test_missing(backend, db, full):
# Create two objects, one will be missing
obj_ids = (22, 25)
for id_ in obj_ids:
db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(id_, 1, 27 * id_))
key = 's3ql_data_%d' % obj_ids[0]
backend[key] = b'just some data that no-one really cares about'
# When using a single thread, we can fake the backend factory
backend_factory = lambda: backend
missing_fh = io.StringIO()
corrupted_fh = io.StringIO()
with assert_logs('^Backend seems to have lost',
count=1,
level=logging.WARNING):
verify.retrieve_objects(db,
backend_factory,
corrupted_fh,
missing_fh,
thread_count=1,
full=full)
assert missing_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[1]
assert corrupted_fh.getvalue() == ''
async def test_thread_hang(ctx):
# Make sure that we don't deadlock if uploads threads or removal
# threads have died and we try to expire or terminate
# Monkeypatch to avoid error messages about uncaught exceptions
# in other threads
upload_exc = False
removal_exc = False
def _upload_loop(*a, fn=ctx.cache._upload_loop):
try:
return fn(*a)
except NotADirectoryError:
nonlocal upload_exc
upload_exc = True
def _removal_loop_multi(*a, fn=ctx.cache._removal_loop_multi):
try:
return fn(*a)
except NotADirectoryError:
nonlocal removal_exc
removal_exc = True
ctx.cache._upload_loop = _upload_loop
ctx.cache._removal_loop_multi = _removal_loop_multi
# Start threads
ctx.cache.init(threads=3)
# Create first object (we'll try to remove that)
async with ctx.cache.get(ctx.inode, 0) as fh:
fh.write(b'bar wurfz!')
await ctx.cache.start_flush()
await ctx.cache.wait()
# Make sure that upload and removal will fail
os.rename(ctx.backend_dir, ctx.backend_dir + '-tmp')
open(ctx.backend_dir, 'w').close()
# Create second object (we'll try to upload that)
async with ctx.cache.get(ctx.inode, 1) as fh:
fh.write(b'bar wurfz number two!')
# Schedule a removal
await ctx.cache.remove(ctx.inode, 0)
try:
# Try to clean-up (implicitly calls expire)
with assert_logs('Unable to flush cache, no upload threads left alive',
level=logging.ERROR, count=1):
await ctx.cache.destroy(keep_cache=True)
assert upload_exc
assert removal_exc
finally:
# Fix backend dir
os.unlink(ctx.backend_dir)
os.rename(ctx.backend_dir + '-tmp', ctx.backend_dir)
# Remove objects from cache and make final destroy
# call into no-op.
await ctx.cache.remove(ctx.inode, 1)
ctx.cache.destroy = None
def test_corrupted_meta(backend, monkeypatch):
key = 'brafasel'
value = b'hello there, let us see whats going on'
backend[key] = value
# Monkeypatch request handler to mess up metadata
handler_class = mock_server.S3CRequestHandler
def send_header(self, keyword ,value, count=[0],
send_header_real=handler_class.send_header):
if keyword == self.hdr_prefix + 'Meta-md5':
count[0] += 1
if count[0] <= 3:
value = value[::-1]
return send_header_real(self, keyword, value)
monkeypatch.setattr(handler_class, 'send_header', send_header)
with assert_logs('^MD5 mismatch in metadata for', count=1, level=logging.WARNING):
assert_raises(BadDigestError, backend.fetch, key)
enable_temp_fail(backend)
with assert_logs('^MD5 mismatch in metadata for', count=2, level=logging.WARNING):
assert backend[key] == value
def test_corrupted_body(backend, db, full):
obj_ids = (35, 40)
for id_ in obj_ids:
db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(id_, 1, 27 * id_))
# Object one will be fine
key = 's3ql_data_%d' % obj_ids[0]
backend[key] = b'just some data that no-one really cares about'
# Object two will have a checksum error in the body
key = 's3ql_data_%d' % obj_ids[1]
backend[key] = b'some data that will be broken on a data check'
(raw, meta) = backend.backend.fetch(key)
raw = bytearray(raw)
assert len(raw) > 20
raw[-10:-6] = b'forg'
backend.backend.store(key, raw, meta)
# When using a single thread, we can fake the backend factory
backend_factory = lambda: backend
missing_fh = io.StringIO()
corrupted_fh = io.StringIO()
if full:
with assert_logs('^Object %d is corrupted',
count=1,
level=logging.WARNING):
verify.retrieve_objects(db,
backend_factory,
corrupted_fh,
missing_fh,
thread_count=1,
full=full)
assert missing_fh.getvalue() == ''
assert corrupted_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[1]
else:
# Should not show up when looking just at HEAD
verify.retrieve_objects(db,
backend_factory,
corrupted_fh,
missing_fh,
thread_count=1,
full=full)
assert missing_fh.getvalue() == ''
assert corrupted_fh.getvalue() == ''
def test_corrupted_head(backend, db, full):
obj_ids = (30, 31)
for id_ in obj_ids:
db.execute('INSERT INTO objects (id, refcount, size) VALUES(?, ?, ?)',
(id_, 1, 27 * id_))
# Object one will be fine
key = 's3ql_data_%d' % obj_ids[0]
backend[key] = b'just some data that no-one really cares about'
# Object two will have a checksum error in the metadata
key = 's3ql_data_%d' % obj_ids[1]
backend.store(
key, b'some data that will be broken on a metadata check', {
'meta-key1': 'some textual data that just increases',
'meta-key2': 'the metadata size so that we can tamper with it'
})
meta = backend.backend.lookup(key)
raw = bytearray(meta['data'])
assert len(raw) > 20
raw[-10:-6] = b'forg'
meta['data'] = raw
backend.backend.update_meta(key, meta)
# When using a single thread, we can fake the backend factory
backend_factory = lambda: backend
missing_fh = io.StringIO()
corrupted_fh = io.StringIO()
with assert_logs('^Object %d is corrupted', count=1,
level=logging.WARNING):
verify.retrieve_objects(db,
backend_factory,
corrupted_fh,
missing_fh,
thread_count=1,
full=full)
assert missing_fh.getvalue() == ''
assert corrupted_fh.getvalue() == 's3ql_data_%d\n' % obj_ids[1]
def test_thread_hang(self):
# Make sure that we don't deadlock if uploads threads or removal
# threads have died and we try to expire or terminate
# Monkeypatch to avoid error messages about uncaught exceptions
# in other threads
upload_exc = False
removal_exc = False
def _upload_loop(*a, fn=self.cache._upload_loop):
try:
return fn(*a)
except NotADirectoryError:
nonlocal upload_exc
upload_exc = True
def _removal_loop(*a, fn=self.cache._removal_loop):
try:
return fn(*a)
except NotADirectoryError:
nonlocal removal_exc
removal_exc = True
self.cache._upload_loop = _upload_loop
self.cache._removal_loop = _removal_loop
# Start threads
self.cache.init(threads=3)
# Create first object (we'll try to remove that)
with self.cache.get(self.inode, 0) as fh:
fh.write(b'bar wurfz!')
self.cache.commit()
self.cache.wait()
# Make sure that upload and removal will fail
os.rename(self.backend_dir, self.backend_dir + '-tmp')
open(self.backend_dir, 'w').close()
# Create second object (we'll try to upload that)
with self.cache.get(self.inode, 1) as fh:
fh.write(b'bar wurfz number two!')
# Schedule a removal
self.cache.remove(self.inode, 0)
try:
# Try to clean-up (implicitly calls expire)
with llfuse.lock_released, \
assert_logs('Unable to flush cache, no upload threads left alive',
level=logging.ERROR, count=1):
with pytest.raises(OSError) as exc_info:
self.cache.destroy()
assert exc_info.value.errno == errno.ENOTEMPTY
assert upload_exc
assert removal_exc
finally:
# Fix backend dir
os.unlink(self.backend_dir)
os.rename(self.backend_dir + '-tmp', self.backend_dir)
# Remove objects from cache and make final destroy
# call into no-op.
self.cache.remove(self.inode, 1)
self.cache.destroy = lambda: None
def test_thread_hang(self):
# Make sure that we don't deadlock if uploads threads or removal
# threads have died and we try to expire or terminate
# Monkeypatch to avoid error messages about uncaught exceptions
# in other threads
upload_exc = False
removal_exc = False
def _upload_loop(*a, fn=self.cache._upload_loop):
try:
return fn(*a)
except NotADirectoryError:
nonlocal upload_exc
upload_exc = True
def _removal_loop(*a, fn=self.cache._removal_loop):
try:
return fn(*a)
except NotADirectoryError:
nonlocal removal_exc
removal_exc = True
self.cache._upload_loop = _upload_loop
self.cache._removal_loop = _removal_loop
# Start threads
self.cache.init(threads=3)
# Create first object (we'll try to remove that)
with self.cache.get(self.inode, 0) as fh:
fh.write(b'bar wurfz!')
self.cache.start_flush()
self.cache.wait()
# Make sure that upload and removal will fail
os.rename(self.backend_dir, self.backend_dir + '-tmp')
open(self.backend_dir, 'w').close()
# Create second object (we'll try to upload that)
with self.cache.get(self.inode, 1) as fh:
fh.write(b'bar wurfz number two!')
# Schedule a removal
self.cache.remove(self.inode, 0)
try:
# Try to clean-up (implicitly calls expire)
with assert_logs(
'Unable to drop cache, no upload threads left alive',
level=logging.ERROR,
count=1):
with pytest.raises(OSError) as exc_info:
self.cache.destroy()
assert exc_info.value.errno == errno.ENOTEMPTY
assert upload_exc
assert removal_exc
finally:
# Fix backend dir
os.unlink(self.backend_dir)
os.rename(self.backend_dir + '-tmp', self.backend_dir)
# Remove objects from cache and make final destroy
# call into no-op.
self.cache.remove(self.inode, 1)
self.cache.destroy = lambda: None
def test_logging():
inst = NthAttempt(6)
with assert_logs(r'^Encountered %s \(%s\), retrying ',
count=2,
level=logging.WARNING):
inst.do_stuff()
def test_logging():
inst = NthAttempt(6)
with assert_logs(r'^Encountered %s \(%s\), retrying ',
count=2, level=logging.WARNING):
inst.do_stuff()
