def test_cache_upload(self):
self.reg_output(r'^WARNING: Writing dirty block', count=1)
self.reg_output(r'^WARNING: Remote metadata is outdated', count=1)
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
# Kill mount
self.flush_cache()
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
# Update cache files
cache_file = pjoin(self.cache_dir, _escape(self.storage_url) + '-cache', '4-0')
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
self.fsck(expect_retcode=128)
assert not os.path.exists(cache_file)
# Ensure that we get updated data
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA[::-1]
self.umount()
def test_cache_flush(self, with_fsck):
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
ino = os.fstat(fh.fileno()).st_ino
self.umount()
# Taint cache file, so we'll get an error if it's re-used
cache_file = pjoin(self.cache_dir, _escape(self.storage_url)
+ '-cache', '%d-0' % (ino,))
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
# Mount elsewhere
bak = self.cache_dir
self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-')
try:
self.mount()
with open(pjoin(self.mnt_dir, 'testfile2'), 'wb') as fh:
fh.write(b'hello')
self.umount()
finally:
shutil.rmtree(self.cache_dir)
self.cache_dir = bak
# Make sure that cache is ignored
if with_fsck:
self.fsck(args=['--keep-cache'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA
self.umount()
def test_cache_upload(self):
self.reg_output(r'^WARNING: Writing dirty block', count=1)
self.reg_output(r'^WARNING: Remote metadata is outdated', count=1)
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
# Kill mount
self.flush_cache()
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
# Update cache files
cache_file = pjoin(self.cache_dir,
_escape(self.storage_url) + '-cache', '4-0')
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
self.fsck(expect_retcode=128)
assert not os.path.exists(cache_file)
# Ensure that we get updated data
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA[::-1]
self.umount()
def test(self):
skip_without_rsync()
ref_dir = tempfile.mkdtemp(prefix='s3ql-ref-')
try:
populate_dir(ref_dir)
# Make file system and fake high inode number
self.mkfs()
cachepath = pjoin(self.cache_dir, _escape(self.storage_url))
db = Connection(cachepath + '.db')
db.execute('UPDATE sqlite_sequence SET seq=? WHERE name=?',
(2**31 + 10, 'inodes'))
db.close()
# Copy source data
self.mount()
subprocess.check_call(
['rsync', '-aHAX', ref_dir + '/', self.mnt_dir + '/'])
self.umount()
# Check that inode watermark is high
db = Connection(cachepath + '.db')
assert db.get_val('SELECT seq FROM sqlite_sequence WHERE name=?',
('inodes', )) > 2**31 + 10
assert db.get_val('SELECT MAX(id) FROM inodes') > 2**31 + 10
db.close()
# Renumber inodes
self.fsck()
# Check if renumbering was done
db = Connection(cachepath + '.db')
assert db.get_val('SELECT seq FROM sqlite_sequence WHERE name=?',
('inodes', )) < 2**31
assert db.get_val('SELECT MAX(id) FROM inodes') < 2**31
db.close()
# Compare
self.mount()
try:
out = check_output([
'rsync', '-anciHAX', '--delete', '--exclude',
'/lost+found', ref_dir + '/', self.mnt_dir + '/'
],
universal_newlines=True,
stderr=subprocess.STDOUT)
except CalledProcessError as exc:
pytest.fail('rsync failed with ' + exc.output)
if out:
pytest.fail('Copy not equal to original, rsync says:\n' + out)
self.umount()
finally:
shutil.rmtree(ref_dir)
def test_cache_flush_unclean(self):
self.reg_output(r'^WARNING: Renaming outdated cache directory',
count=1)
self.reg_output(r'^WARNING: You should delete this directory', count=1)
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
# Kill mount
self.flush_cache()
self.upload_meta()
# Metadata upload is not blocking, so give it some time to
# complete
time.sleep(1)
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
# Update cache files
cache_file = pjoin(self.cache_dir,
_escape(self.storage_url) + '-cache', '4-0')
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
# Mount elsewhere
bak = self.cache_dir
self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-')
try:
self.fsck(expect_retcode=0, args=['--force-remote'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile2'), 'wb') as fh:
fh.write(b'hello')
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
finally:
shutil.rmtree(self.cache_dir)
self.cache_dir = bak
# Make sure that cache is ignored
self.fsck(expect_retcode=0, args=['--force-remote', '--keep-cache'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA
self.umount()
def test(self):
skip_without_rsync()
ref_dir = tempfile.mkdtemp(prefix='s3ql-ref-')
try:
self.populate_dir(ref_dir)
# Copy source data
self.mkfs()
# Force 64bit inodes
cachepath = os.path.join(self.cache_dir, _escape(self.storage_url))
db = Connection(cachepath + '.db')
db.execute('UPDATE sqlite_sequence SET seq=? WHERE name=?',
(2**36 + 10, 'inodes'))
db.close()
self.mount()
subprocess.check_call(
['rsync', '-aHAX', ref_dir + '/', self.mnt_dir + '/'])
self.umount()
self.fsck()
# Delete cache, run fsck and compare
shutil.rmtree(self.cache_dir)
self.cache_dir = tempfile.mkdtemp('s3ql-cache-')
self.fsck()
self.mount()
try:
out = check_output([
'rsync', '-anciHAX', '--delete', '--exclude',
'/lost+found', ref_dir + '/', self.mnt_dir + '/'
],
universal_newlines=True,
stderr=subprocess.STDOUT)
except CalledProcessError as exc:
pytest.fail('rsync failed with ' + exc.output)
if out:
pytest.fail('Copy not equal to original, rsync says:\n' + out)
self.umount()
# Delete cache and mount
shutil.rmtree(self.cache_dir)
self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-')
self.mount()
self.umount()
finally:
shutil.rmtree(ref_dir)
def test_cache_flush_unclean(self):
self.reg_output(r'^WARNING: Renaming outdated cache directory', count=1)
self.reg_output(r'^WARNING: You should delete this directory', count=1)
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
# Kill mount
self.flush_cache()
self.upload_meta()
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
# Update cache files
cache_file = pjoin(self.cache_dir, _escape(self.storage_url)
+ '-cache', '4-0')
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
# Mount elsewhere
bak = self.cache_dir
self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-')
try:
self.fsck(expect_retcode=0,
args=['--force-remote'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile2'), 'wb') as fh:
fh.write(b'hello')
self.mount_process.kill()
self.mount_process.wait()
self.umount_fuse()
finally:
shutil.rmtree(self.cache_dir)
self.cache_dir = bak
# Make sure that cache is ignored
self.fsck(expect_retcode=0,
args=['--force-remote', '--keep-cache'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA
self.umount()
def test_cache_flush(self, with_fsck):
# Write test data
self.mkfs()
self.mount(extra_args=['--keep-cache'])
with open(pjoin(self.mnt_dir, 'testfile'), 'wb') as fh:
fh.write(TEST_DATA)
ino = os.fstat(fh.fileno()).st_ino
self.umount()
# Taint cache file, so we'll get an error if it's re-used
cache_file = pjoin(self.cache_dir,
_escape(self.storage_url) + '-cache',
'%d-0' % (ino, ))
with open(cache_file, 'rb+') as fh:
fh.write(TEST_DATA[::-1])
# Mount elsewhere
bak = self.cache_dir
self.cache_dir = tempfile.mkdtemp(prefix='s3ql-cache-')
try:
self.mount()
with open(pjoin(self.mnt_dir, 'testfile2'), 'wb') as fh:
fh.write(b'hello')
self.umount()
finally:
shutil.rmtree(self.cache_dir)
self.cache_dir = bak
# Make sure that cache is ignored
if with_fsck:
self.fsck(args=['--keep-cache'])
self.mount()
with open(pjoin(self.mnt_dir, 'testfile'), 'rb') as fh:
assert fh.read() == TEST_DATA
self.umount()