def test_calculate_does_not_indicate_removal_of_files_involved_in_backup(
self, m_free):
# paths are part of backup, and should not be removed
walk_paths = {
'/dst': [('/dst', ['a'], ['0.txt']), ('/dst/a', [], ['2.txt'])]
}
reconciler = keepfilesreconciler.KeepFilesReconciler(
self.resolver, self.options)
with filesystemhelpers.mock_walk(walk_paths):
filepaths = reconciler.calculate(self.copyfiles, copied_indexes=[])
assert filepaths == set()
def test_calculate_indicates_removal_of_unrelated_files(self, m_free):
""" Indicate removal of files that are not a part of the backup.
"""
# files are unrelated to backup
walk_paths = {
'/dst': [('/dst', ['/a'], ['x0.txt']), ('/dst/a', [], ['x1.txt'])]
}
copied_indexes = []
reconciler = keepfilesreconciler.KeepFilesReconciler(
self.resolver, self.options)
with filesystemhelpers.mock_walk(walk_paths):
filepaths = reconciler.calculate(self.copyfiles, copied_indexes)
assert filepaths == {'/dst/a/x1.txt', '/dst/x0.txt'}
def test_calculate_indicates_removal_of_files_device_will_not_have_room_for(
self, m_free):
""" Indicate removal of files that won't fit on this device.
"""
# volume has enough room for 1x 1024b file.
isfile_results = {
'/dst/0.txt': True,
'/dst/a/1.txt': True,
'/dst/a/2.txt': True
}
copied_indexes = []
reconciler = keepfilesreconciler.KeepFilesReconciler(
self.resolver, self.options)
with filesystemhelpers.mock_isfile(isfile_results):
filepaths = reconciler.calculate(self.copyfiles, copied_indexes)
assert filepaths == {'/dst/a/1.txt', '/dst/a/2.txt'}
def __init__(self, resolver, options=None, reconciler=None):
"""
Args:
resolver (resolver.Resolver):
Resolver, determines files to be copied.
output (str): ``(ex: '/mnt/backup' )``
The directory you'd like to backup to.
options (CopyOptions, None):
Options to use while performing copy
"""
super(MultiProcessCopier, self).__init__(resolver, options)
# NOTE: queueing using multiprocessing.Manager.list() to approximate LIFO queue.
# (for backup-file reconciliation, order must remain consistent in queue)
# (LIFO allows us to re-enqeueue failed copies due to diskfull error)
# queues/locks
self._mpmanager = multiprocessing.Manager()
self.joblist = self._mpmanager.list()
self.completed_queue = multiprocessing.Queue()
self.error_queue = multiprocessing.Queue()
self.started_queue = multiprocessing.Queue()
self.device_full_lock = multiprocessing.Event()
# components
self.prompt = commandlineprompt.CommandlinePrompt()
self._interpreter = interpreter.Interpreter()
self.manager = _MultiProcessCopierWorkerManager(self.joblist,
self.started_queue,
self.completed_queue,
self.error_queue,
self.device_full_lock,
options)
self._progress_formatter = lineformatter.LineFormatter()
self.reconciler = reconciler or keepfilesreconciler.KeepFilesReconciler(resolver, options)
# internal data
self._copyfiles = tuple()
self._copied_indexes = []
self._error_indexes = []
self._started_indexes = []
def test_calculate_indicates_removal_of_already_copied_files(self, m_free):
""" Indicate removal of files that have been already copied to another device.
Notes:
The reconciler runs first wtih device-index, second with
the last successfully copied index once a device is full.
`copied_indexes` in both cases indicate files that do not belong on this device,
and they should be removed.
"""
# files belong to backup, but have been copied to another device
# (indicated by copied_indexes)
walk_paths = {
'/dst': [('/dst', ['a'], ['0.txt']), ('/dst/a', [], ['2.txt'])]
}
copied_indexes = [0, 2]
reconciler = keepfilesreconciler.KeepFilesReconciler(
self.resolver, self.options)
with filesystemhelpers.mock_walk(walk_paths):
filepaths = reconciler.calculate(self.copyfiles, copied_indexes)
assert filepaths == {'/dst/0.txt', '/dst/a/2.txt'}