def test_broken_metadata_with_backup(self):
"""test that each time a metadata file is updated a .old is kept"""
self.shelf['bad_file'] = {'value': 'old'}
path = self.shelf.key_file('bad_file')
self.assertFalse(path_exists(path+'.old'))
self.assertEqual({'value': 'old'}, self.shelf['bad_file'])
# force the creation of the .old file
self.shelf['bad_file'] = {'value': 'new'}
self.assertTrue(path_exists(path+'.old'))
# check that the new value is there
self.assertEqual({'value': 'new'}, self.shelf['bad_file'])
# write the current md file fwith 0 bytes
open_file(path, 'w').close()
# test that the old value is retrieved
self.assertEqual({'value': 'old'}, self.shelf['bad_file'])
self.shelf['broken_pickle'] = {'value': 'old'}
path = self.shelf.key_file('broken_pickle')
# check that .old don't exist
self.assertFalse(path_exists(path+'.old'))
# force the creation of the .old file
self.shelf['broken_pickle'] = {'value': 'new'}
# check that .old exists
self.assertTrue(path_exists(path+'.old'))
# check that the new value is there
self.assertEqual({'value': 'new'}, self.shelf['broken_pickle'])
# write random bytes to the md file
with open_file(path, 'w') as f:
f.write(BROKEN_PICKLE)
# check that the old value is retrieved
self.assertEqual({'value': 'old'}, self.shelf['broken_pickle'])
def test_remove_dir(self):
"""Test the remove dir."""
testdir = os.path.join(self.basedir, 'foodir')
os.mkdir(testdir)
assert path_exists(testdir)
remove_dir(testdir)
self.assertFalse(path_exists(testdir))
def test_broken_metadata_with_backup(self):
"""overrides parent test as we have the value in the cache."""
self.shelf['bad_file'] = {'value': 'old'}
path = self.shelf.key_file('bad_file')
self.assertFalse(path_exists(path+'.old'))
self.assertEqual({'value': 'old'}, self.shelf['bad_file'])
# force the creation of the .old file
self.shelf['bad_file'] = {'value': 'new'}
self.assertTrue(path_exists(path+'.old'))
# check that the new value is there
self.assertEqual({'value': 'new'}, self.shelf['bad_file'])
# write the current md file fwith 0 bytes
open_file(path, 'w').close()
# HERE IS THE DIFFERENCE with the parent tests
# test that the new value is retrieved from the cache!
self.assertEqual({'value': 'new'}, self.shelf['bad_file'])
self.shelf['broken_pickle'] = {'value': 'old'}
path = self.shelf.key_file('broken_pickle')
# check that .old don't exist
self.assertFalse(path_exists(path+'.old'))
# force the creation of the .old file
self.shelf['broken_pickle'] = {'value': 'new'}
# check that .old exists
self.assertTrue(path_exists(path+'.old'))
# check that the new value is there
self.assertEqual({'value': 'new'}, self.shelf['broken_pickle'])
# write random bytes to the md file
with open_file(path, 'w') as f:
f.write(BROKEN_PICKLE)
# HERE IS THE DIFFERENCE with the parent tests
# test that the new value is retrieved from the cache!
self.assertEqual({'value': 'new'}, self.shelf['broken_pickle'])
def get_config_files():
""" return the path to the config files or and empty list.
The search path is based on the paths returned by load_config_paths
but it's returned in reverse order (e.g: /etc/xdg first).
"""
config_files = []
for xdg_config_dir in load_config_paths('ubuntuone'):
config_file = os.path.join(xdg_config_dir, CONFIG_FILE)
if path_exists(config_file):
config_files.append(config_file)
config_logs = os.path.join(xdg_config_dir, CONFIG_LOGS)
if path_exists(config_logs):
config_files.append(config_logs)
# reverse the list as load_config_paths returns the user dir first
config_files.reverse()
# if we are running from a branch, get the config files from it too
config_file = os.path.join(os.path.dirname(__file__), os.path.pardir,
os.path.pardir, 'data', CONFIG_FILE)
if path_exists(config_file):
config_files.append(config_file)
config_logs = os.path.join('data', CONFIG_LOGS)
if path_exists(config_logs):
config_files.append(config_logs)
return config_files
def test_create_dirs_already_exists_symlink_too(self):
"""test that creating a Main instance works as expected."""
link = os.path.join(self.root, 'Shared With Me')
make_link(self.shares, link)
self.assertTrue(is_link(link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
main = self.build_main()
# check that the shares link is actually a link
self.assertTrue(is_link(main.shares_dir_link))
def test_create_dirs_none_exists(self):
"""test that creating a Main instance works as expected."""
# remove the existing dirs
remove_dir(self.root)
remove_dir(self.shares)
main = self.build_main()
# check that the shares link is actually a link
self.assertTrue(is_link(main.shares_dir_link))
self.assertTrue(path_exists(self.shares))
self.assertTrue(path_exists(self.root))
def test_rename_file(self, target=None):
"""Rename a file."""
if target is None:
target = os.path.join(self.basedir, 'target')
assert path_exists(self.testfile)
rename(self.testfile, target)
self.assertFalse(path_exists(self.testfile),
'Path %r should not exist after rename.' % self.testfile)
self.assertTrue(path_exists(target),
'Path %r should exist after rename.' % target)
def test_delete_backups_too(self):
"""test that delitem also deletes the .old/.new files left around"""
self.shelf["foo"] = "bar"
# create the .old backup
self.shelf["foo"] = "bar1"
path = self.shelf.key_file('foo')
# create a .new file (a hard reboot during the rename dance)
open_file(path+'.new', 'w').close()
# write 0 bytes to both
del self.shelf['foo']
self.assertFalse(path_exists(path))
self.assertFalse(path_exists(path+'.old'), 'there is a .old file!')
self.assertFalse(path_exists(path+'.new'), 'there is a .new file!')
def test_rename_dir(self, source=None, target=None):
"""Rename a dir."""
if source is None:
source = os.path.join(self.basedir, 'source')
os.mkdir(source)
if target is None:
target = os.path.join(self.basedir, 'target')
rename(source, target)
self.assertFalse(path_exists(source),
'Path %r should not exist after rename.' % source)
self.assertTrue(path_exists(target),
'Path %r should exist after rename.' % target)
def is_ignored(self, path):
"""should we ignore this path?"""
# check first if the platform code knows hat to do with it
if not self.platform_is_ignored(path):
# check if we can read
if path_exists(path) and not access(path):
self.log.warning("Ignoring path as we don't have enough "
"permissions to track it: %r", path)
return True
is_conflict = self.conflict_RE.search
dirname, filename = os.path.split(path)
# ignore conflicts
if is_conflict(filename):
return True
# ignore partial downloads
if filename == '.u1partial' or filename.startswith('.u1partial.'):
return True
# and ignore paths that are inside conflicts (why are we even
# getting the event?)
if any(part.endswith('.u1partial') or is_conflict(part)
for part in dirname.split(os.path.sep)):
return True
if self.ignore_RE is not None and self.ignore_RE.match(filename):
return True
return False
return True
def test_allow_writes(self):
"""Test for allow_writes."""
set_dir_readonly(self.basedir)
with allow_writes(self.basedir):
foo_dir = os.path.join(self.basedir, 'foo')
os.mkdir(foo_dir)
self.assertTrue(path_exists(foo_dir))
def test_write_extra(self):
"""Writing the throttling back to the file, with extra sections."""
conf_file = os.path.join(
self.test_root, 'test_write_extra_config.conf')
# write some throttling values to the config file
with open_file(conf_file, 'w') as fp:
fp.write('[__main__]\n')
fp.write('log_level = INFO\n')
fp.write('disable_ssl_verify = True\n')
fp.write('\n')
fp.write('[bandwidth_throttling]\n')
fp.write('on = False\n')
fp.write('read_limit = 2000\n')
fp.write('write_limit = 200\n')
self.assertTrue(path_exists(conf_file))
conf = config._Config(conf_file)
conf.set_throttling(True)
conf.set_throttling_read_limit(3000)
conf.set_throttling_write_limit(300)
conf.save()
# load the config in a barebone ConfigParser and check
conf_1 = ConfigParser()
conf_1.read(conf_file)
self.assertThrottlingSection(conf_1, conf, True, 3000, 300)
self.assertEqual(conf_1.get('__main__', 'log_level'),
conf.get('__main__', 'log_level'))
self.assertEqual(conf_1.getboolean('__main__', 'disable_ssl_verify'),
conf.getboolean('__main__', 'disable_ssl_verify'))
def start(self):
"""Start the comparison."""
log_info("start scan all volumes")
to_scan = self._get_volumes(all_volumes=False)
for vol in to_scan:
# check that the path exists in disk
if not path_exists(vol.path):
log_warning('Volume dissapeared: %r - %r', vol.volume_id, vol.path)
if isinstance(vol, volume_manager.Share):
log_debug('Removing %r metadata', vol.volume_id)
self.vm.share_deleted(vol.volume_id)
elif isinstance(vol, volume_manager.UDF):
log_debug('Unsubscribing %r', vol.volume_id)
self.vm.unsubscribe_udf(vol.volume_id)
# if root is missing, we should crash and burn as it's
# created on each startup!
continue
try:
mdobj = self.fsm.get_by_path(vol.path)
except KeyError:
# this could happen in a strange corruption situation where FSM
# lost the share information, so we remove it, because VM will
# download it again
self.vm.share_deleted(vol.volume_id)
else:
self._queue.appendleft((vol, vol.path, mdobj.mdid, False))
# first of all, remove old partials and clean trash
self._remove_partials()
self._process_limbo()
self._process_ro_shares()
yield self._queue_scan()
self._show_broken_nodes()
def __init__(self, path, auto_merge=True, dead_bytes_threshold=0.5, max_immutable_files=20):
"""Initialize the instance.
@param auto_merge: disable auto merge/compaction.
@param dead_bytes_threshold: the limit factor of dead vs live bytes to
trigger a merge and/or live file rotation.
@param max_immutable_files: the max number of inactive files to use,
once this value is reached a merge is triggered.
"""
logger.info("Initializing Tritcask on: %s", path)
self._keydir = Keydir()
self.base_path = path
self.dead_bytes_threshold = dead_bytes_threshold
self.max_immutable_files = max_immutable_files
self.auto_merge = auto_merge
if not path_exists(self.base_path):
make_dir(self.base_path, recursive=True)
elif not is_dir(self.base_path):
raise ValueError("path must be a directory.")
self.live_file = None
self._immutable = {}
self._find_data_files()
self._build_keydir()
# now check if we should rotate the live file
# and merge immutable ones
self._rotate_and_merge()
# check if we found a live data file
# if not, define one (it will be created later)
if self.live_file is None:
# it's a clean start, let's create the first file
self.live_file = DataFile(self.base_path)
def test_broken_metadata_items(self):
"""Test that broken metadata is ignored during iteritems."""
self.shelf['ok_key'] = {'status': 'this is valid metadata'}
self.shelf['bad_file'] = {}
path = self.shelf.key_file('bad_file')
open_file(path, 'w').close()
self.assertRaises(KeyError, self.shelf.__getitem__, 'bad_file')
self.assertEqual(1, len(list(self.shelf.items())))
self.assertFalse(path_exists(path))
self.shelf['broken_pickle'] = {}
path = self.shelf.key_file('broken_pickle')
with open_file(path, 'w') as f:
f.write(BROKEN_PICKLE)
self.assertRaises(KeyError, self.shelf.__getitem__, 'broken_pickle')
self.assertEqual(1, len(list(self.shelf.items())))
self.assertFalse(path_exists(path))
def _check_move_file(self, src, dst, real_dst):
"""Check that a file was indeed moved."""
with open_file(src, "rb") as f:
contents = f.read()
recursive_move(src, dst)
with open_file(real_dst, "rb") as f:
self.assertEqual(contents, f.read())
self.assertFalse(path_exists(src))
def test_movetotrash_file_ok(self):
"""Move a file to trash ok.
Just check it was removed because can't monkeypatch the trash.
to see that that was actually called.
"""
move_to_trash(self.testfile)
self.assertFalse(path_exists(self.testfile))
def mktemp(self, name="temp"):
""" Customized mktemp that accepts an optional name argument. """
tempdir = os.path.join(self.tmpdir, name)
if path_exists(tempdir):
self.rmtree(tempdir)
self.makedirs(tempdir)
self.addCleanup(self.rmtree, tempdir)
assert isinstance(tempdir, str)
return tempdir
def add_share(self, share):
"""Add share to the shares dict."""
self.shares[share.id] = share
# if the share don't exists, create it
if not path_exists(share.path):
make_dir(share.path)
# if it's a ro share, change the perms
if not share.can_write():
set_dir_readonly(share.path)
def test_ignore_one(self):
"""Test ignore files config, one regex."""
conf_file = os.path.join(self.test_root, 'test_new_config.conf')
with open_file(conf_file, 'w') as fp:
fp.write('[__main__]\n')
fp.write('ignore = .*\\.pyc\n') # all .pyc files
self.assertTrue(path_exists(conf_file))
self.cp.read([conf_file])
self.cp.parse_all()
self.assertEqual(self.cp.get('__main__', 'ignore').value, [r'.*\.pyc'])
def test_movetotrash_dir_ok(self):
"""Move a dir to trash ok.
Just check it was removed because can't monkeypatch the trash
to see that that was actually called.
"""
path = os.path.join(self.basedir, 'foo')
make_dir(path)
move_to_trash(path)
self.assertFalse(path_exists(path))
def test_set_dir_readwrite(self):
"""Test for set_dir_readwrite."""
set_dir_readonly(self.basedir)
# do not queue up any cleanup function since we're restoring perms in
# the next call
set_dir_readwrite(self.basedir)
foo_dir = os.path.join(self.basedir, 'foo')
os.mkdir(foo_dir)
self.assertTrue(path_exists(foo_dir))
def save(self):
"""Save the config object to disk"""
# cleanup empty sections
for section in [MAIN, THROTTLING, NOTIFICATIONS]:
if self.has_section(section) and not self.options(section):
self.remove_section(section)
with open(self.config_file + '.new', 'w') as fp:
self.write(fp)
if path_exists(self.config_file):
rename(self.config_file, self.config_file + '.old')
rename(self.config_file + '.new', self.config_file)
def test_log_level_new_config(self):
"""Test log_level upgrade hook with new config."""
conf_file = os.path.join(self.test_root, 'test_new_config.conf')
# write some throttling values to the config file
with open_file(conf_file, 'w') as fp:
fp.write('[logging]\n')
fp.write('level = DEBUG\n')
self.assertTrue(path_exists(conf_file))
self.cp.read([conf_file])
self.cp.parse_all()
self.assertEqual(self.cp.get('logging', 'level').value, 10)
def test_different_depth_sizes(self):
"""Test the basic operations (delitem, getitem, setitem) with
depths between 0 and len(hashlib.sha1().hexdigest())
"""
base_path = os.path.join(self.path, 'shelf_depth-')
sha1 = hashlib.sha1()
for idx in xrange(0, len(sha1.hexdigest())):
path = base_path + str(idx)
shelf = self.fileshelf_class(path, depth=idx)
key = sha1.hexdigest()
# test __setitem__
shelf[key] = 'foo'
key_path = os.path.join(path, *[key[i] for i in xrange(0, idx)])
self.assertTrue(path_exists(os.path.join(key_path, key)))
# test __getitem__
self.assertEqual('foo', shelf[key])
# test __delitem__
del shelf[key]
self.assertRaises(KeyError, shelf.__getitem__, key)
self.assertFalse(path_exists(os.path.join(key_path, key)))
def __init__(self, path):
"""Create the instance."""
self.path = path
self.tempfile = None
if path_exists(self.path) and stat_path(self.path).st_size > 0:
# if it's there and size > 0, open only for read
self.fd = open_file(self.path, "rb")
else:
# this is a new hint file, lets create it as a tempfile.
self.tempfile = tempfile.mktemp(dir=os.path.dirname(self.path))
self.fd = open_file(self.tempfile, "w+b")
def __setitem__(self, key, value):
""" setitem backed by the file storage """
path = self.key_file(key)
new_path = path + ".new"
old_path = path + ".old"
self._check_and_create_dirs(os.path.dirname(path))
with open_file(new_path, "wb") as fh:
self._pickle(value, fh, protocol=2)
fh.flush()
if path_exists(path):
rename(path, old_path)
rename(new_path, path)
def test_fs_monitor_not_default(self):
"""Test get monitor."""
monitor_id = 'my_monitor'
conf_file = os.path.join(self.test_root, 'test_new_config.conf')
with open_file(conf_file, 'w') as fd:
fd.write('[__main__]\n')
fd.write('fs_monitor = %s\n' % monitor_id)
self.assertTrue(path_exists(conf_file))
self.cp.read([conf_file])
self.cp.parse_all()
self.assertEqual(
self.cp.get('__main__', 'fs_monitor').value, monitor_id)
def create_shares_link(source, dest):
"""Create the shares symlink."""
result = False
if not path_exists(dest):
# remove the symlink if it's broken
if is_link(dest) and read_link(dest) != source:
remove_link(dest)
if not is_link(dest):
# only create the link if it does not exist
make_link(source, dest)
result = True
return result
def test_write_new(self):
"""Test writing the throttling section to a new config file."""
conf_file = os.path.join(self.test_root, 'test_write_new_config.conf')
self.assertFalse(path_exists(conf_file))
conf = config._Config(conf_file)
conf.set_throttling(True)
conf.set_throttling_read_limit(1000)
conf.set_throttling_write_limit(100)
conf.save()
# load the config in a barebone ConfigParser and check
conf_1 = ConfigParser()
conf_1.read(conf_file)
self.assertThrottlingSection(conf_1, conf, True, 1000, 100)
