def close(self):
"""Close the fd."""
if self.fd:
self.fd.flush()
os.fsync(self.fd.fileno())
self.fd.close()
# if this is a new hint file, rename it to the real path.
if self.tempfile:
rename(self.tempfile, self.path)
def make_immutable(self):
"""Make this data file immutable."""
self.close()
new_name = self.filename.replace(self.temp_name, INACTIVE)
rename(self.filename, new_name)
if self.has_hint:
new_hint_name = self.hint_filename.replace(self.temp_name, INACTIVE)
rename(self.hint_filename, new_hint_name)
return ImmutableDataFile(*os.path.split(new_name))
def save(self):
"""Save the config object to disk"""
# cleanup empty sections
for section in [MAIN, THROTTLING, NOTIFICATIONS]:
if self.has_section(section) and not self.options(section):
self.remove_section(section)
with open(self.config_file + '.new', 'w') as fp:
self.write(fp)
if path_exists(self.config_file):
rename(self.config_file, self.config_file + '.old')
rename(self.config_file + '.new', self.config_file)
def test_rename_file(self, target=None):
"""Rename a file."""
if target is None:
target = os.path.join(self.basedir, 'target')
assert path_exists(self.testfile)
rename(self.testfile, target)
self.assertFalse(path_exists(self.testfile),
'Path %r should not exist after rename.' % self.testfile)
self.assertTrue(path_exists(target),
'Path %r should exist after rename.' % target)
def __setitem__(self, key, value):
""" setitem backed by the file storage """
path = self.key_file(key)
new_path = path + ".new"
old_path = path + ".old"
self._check_and_create_dirs(os.path.dirname(path))
with open_file(new_path, "wb") as fh:
self._pickle(value, fh, protocol=2)
fh.flush()
if path_exists(path):
rename(path, old_path)
rename(new_path, path)
def test_rename_dir(self, source=None, target=None):
"""Rename a dir."""
if source is None:
source = os.path.join(self.basedir, 'source')
os.mkdir(source)
if target is None:
target = os.path.join(self.basedir, 'target')
rename(source, target)
self.assertFalse(path_exists(source),
'Path %r should not exist after rename.' % source)
self.assertTrue(path_exists(target),
'Path %r should exist after rename.' % target)
def make_zombie(self):
"""Rename the file but leave it open.
Actually close, rename and open it again.
"""
new_name = self.filename.replace(INACTIVE, DEAD)
new_hint_name = self.hint_filename.replace(INACTIVE, DEAD)
self.close()
rename(self.filename, new_name)
self.filename = new_name
if self.has_hint:
rename(self.hint_filename, new_hint_name)
self.hint_filename = new_hint_name
self._open()
return self
def test_file_moved_from_partial(self):
"""Test the handling of the FILE_MOVE event when source is partial."""
fromfile = os.path.join(self.root_dir, "mdid.u1partial.foo")
root_dir = os.path.join(self.root_dir, "my_files")
tofile = os.path.join(root_dir, "foo")
os.mkdir(root_dir)
open(fromfile, "w").close()
self.monitor.add_to_mute_filter("FS_FILE_CREATE", path=tofile)
self.monitor.add_to_mute_filter("FS_FILE_CLOSE_WRITE", path=tofile)
self.assertEqual(self._how_many_muted(), 2)
yield self.monitor.add_watch(root_dir)
# generate the event
rename(fromfile, tofile)
reactor.callLater(self.timeout - 0.2, self.check_filter)
test_result = yield self._deferred
defer.returnValue(test_result)
def _find_data_files(self):
"""Collect the files we need to work with."""
logger.debug("lookingup data files")
dead_files = 0
broken_files = 0
# becuase listdir appends / at the end of the path if there is not
# os.path.sep at the end we are going to add it, otherwhise on windows
# we will have wrong paths formed
base_path = self.base_path
if not base_path.endswith(os.path.sep):
base_path += os.path.sep
files = listdir(self.base_path)
for filename in files:
# first check for hint and dead
if is_hint(filename):
continue
elif is_dead(filename):
# a dead file...let's remove it
dead_files += 1
DeadDataFile(self.base_path, filename).delete()
continue
# if it's a live or immutable file try to open it, but if it's
# "broken", just rename it and continue
try:
if is_live(filename):
self.live_file = DataFile(self.base_path, filename)
elif is_immutable(filename):
# it's an immutable file
data_file = ImmutableDataFile(self.base_path, filename)
self._immutable[data_file.file_id] = data_file
except IOError as e:
# oops, failed to open the file..discard it
broken_files += 1
orig = os.path.join(self.base_path, filename)
# get the kind of the file so we can rename it
kind = LIVE if is_live(filename) else INACTIVE
dest = orig.replace(kind, BROKEN)
logger.warning("Failed to open %s, renaming it to: %s - " "error: %s", orig, dest, e)
# rename it to "broken"
rename(orig, dest)
# immutable files + live
logger.info(
"found %s data files, %s dead and %s broken files", len(self._immutable) + 1, dead_files, broken_files
)
def test_file_moved_from_conflict(self):
"""Test the handling of the FILE_MOVE event when source is conflict."""
fromfile = os.path.join(self.root_dir, "foo.u1conflict")
self.fs.create(fromfile, "")
self.fs.set_node_id(fromfile, "from_node_id")
tofile = os.path.join(self.root_dir, "foo")
self.fs.create(tofile, "")
self.fs.set_node_id(tofile, "to_node_id")
open(fromfile, "w").close()
self.monitor.add_to_mute_filter("FS_FILE_MOVE",
path_from=fromfile, path_to=tofile)
self.assertEqual(self._how_many_muted(), 2)
yield self.monitor.add_watch(self.root_dir)
# generate the event
rename(fromfile, tofile)
reactor.callLater(self.timeout - 0.2, self.check_filter)
test_result = yield self._deferred
defer.returnValue(test_result)
def test_dir_moved_inside(self):
"""Test the synthesis of the DIR_MOVE event."""
fromdir = os.path.join(self.root_dir, "foo")
self.fs.create(fromdir, "")
self.fs.set_node_id(fromdir, "from_node_id")
todir = os.path.join(self.root_dir, "bar")
self.fs.create(todir, "")
self.fs.set_node_id(todir, "to_node_id")
os.mkdir(fromdir)
self.monitor.add_to_mute_filter("FS_DIR_MOVE",
path_from=fromdir, path_to=todir)
self.assertEqual(self._how_many_muted(), 1)
yield self.monitor.add_watch(self.root_dir)
# generate the event
rename(fromdir, todir)
reactor.callLater(self.timeout - 0.2, self.check_filter)
test_result = yield self._deferred
defer.returnValue(test_result)
def despair(message, fullname, also_children=False, also_remove=None):
"""Something went very bad with this node, converge!"""
# if asked, remove metadata por children
if also_children:
log_debug("Removing metadata for %r children", fullname)
# pylint: disable-msg=W0612
children = self.fsm.get_paths_starting_with(fullname, False)
for path, is_dir in children:
self.fsm.delete_metadata(path)
# remove fullname after removing its children,
# otherwise metadata removal may fail
log_info(message, fullname)
rename(fullname, fullname + ".u1conflict")
self.fsm.delete_metadata(fullname)
# if asked, remove also that file (if still exists)
if also_remove is not None:
try:
log_info("Also remove %r", also_remove)
remove_file(also_remove)
except OSError, e:
if e.errno != errno.ENOENT:
raise
def make_immutable(self):
"""Make this data file immutable."""
self.close()
new_name = self.filename.replace(LIVE, INACTIVE)
rename(self.filename, new_name)
return ImmutableDataFile(*os.path.split(new_name))
