def test_backup_restore(self):
""" Test if backup and restore works correctly """
backup_id = common.backup(self.backend, self.backup_dir)
old_file = os.path.join(self.storage_dir, backup_id)
# Create new backup, this should reuse the last metadata set and the
# checksum should be reused. Metadata set should be identical
with mock.patch('logging.info') as mock_log:
backup_id = common.backup(self.backend, self.backup_dir)
mock_log.assert_any_call('Skipped unchanged sub/o\xcc\x88')
new_file = os.path.join(self.storage_dir, backup_id)
self.assertEqual(utils.sha256_file(old_file), utils.sha256_file(new_file))
# Check if data deduplication works
chunks = utils.find_modified_files(self.storage_dir)
storage_size = 0
for filename, stat in chunks.items():
if filename.startswith('c-'):
storage_size += stat['s']
self.assertTrue(storage_size < self.original_size)
common.restore(self.backend, self.restore_dir, backup_id)
# Compare original file content to restored file content
for fn in ['x', 'sub/y']:
old_filename = os.path.join(self.backup_dir, fn)
old_hash = utils.sha256_file(old_filename)
new_filename = os.path.join(self.restore_dir, fn)
new_hash = utils.sha256_file(new_filename)
self.assertEqual(old_hash, new_hash)
def main(argv=sys.argv):
""" Main function as used in CLI mode. """
logging.basicConfig(format=LOG_FORMAT, level=LOG_LEVEL)
parser = argparse.ArgumentParser(description="Object-storage friendly deduplication backup")
parser.add_argument("--hmac-key", default="", help="HMAC Key")
parser.add_argument("--tag", default="default", help="Tag to use for this backup")
subparsers = parser.add_subparsers(dest="subparsers")
parser_backup = subparsers.add_parser("backup", help="Create new backup")
parser_backup.add_argument("src", action="store", type=str, help="Source path")
parser_backup.add_argument("path", action="store", type=str, help="Destination path")
parser_restore = subparsers.add_parser("restore", help="Restore from backup")
parser_restore.add_argument("path", action="store", type=str, help="Source path")
parser_restore.add_argument("dst", action="store", type=str, help="Destination path")
parser_restore.add_argument("backup_id", action="store", type=str, help="Backup ID")
parser_restore = subparsers.add_parser("gc", help="Garbage collect")
parser_restore.add_argument("path", action="store", type=str, help="Backup path")
parser_list = subparsers.add_parser("list", help="List backups")
parser_list.add_argument("path", action="store", type=str, help="Backup path")
parser_list.add_argument("--backup-id", action="store", type=str, help="Backup ID")
args = parser.parse_args(argv[1:])
path = os.path.expanduser(args.path)
backend = backends.LocalStorage(path)
if args.subparsers == "backup":
common.backup(backend, args.src, args.tag)
if args.subparsers == "restore":
common.restore(backend, args.dst, args.backup_id)
if args.subparsers == "gc":
common.gc(backend)
if args.subparsers == "list":
common.list_backups(backend, args.path, args.backup_id)
def test_gc(self):
""" Test if deletion of no longer required chunks works correctly
1. Create a backup
2. Delete a file
3. Delete old backup
4. Create another backup
5. Run garbage collector
6. Restore and ensure backup is identical to files from step 4 """
backup_id = common.backup(self.backend, self.backup_dir)
backup_path, backup_file = self.backend.fullname(backup_id)
os.remove(backup_file)
os.remove(os.path.join(self.backup_dir, "x"))
backup_id = common.backup(self.backend, self.backup_dir)
removed = common.gc(self.backend)
self.assertTrue(removed)
common.restore(self.backend, self.restore_dir, backup_id)
result = dircmp(self.restore_dir, self.backup_dir)
self.assertFalse(result.diff_files)
for _, entry in result.subdirs.items():
self.assertFalse(entry.diff_files)