def test_freezer_main(monkeypatch):
fake_sys = FakeSys()
monkeypatch.setattr(job, 'create_job', fake_create_job)
monkeypatch.setattr(sys, 'exit', fake_sys.fake_sys_exit)
with pytest.raises(SystemExit):
freezer_main()
monkeypatch.setattr(job, 'create_job', fake_create_job)
monkeypatch.setattr(sys, 'argv', FakeSys.fake_sys_argv())
assert freezer_main() is not None
def test_no_lvm_level0(self):
"""
Maximum level filesystem backup
freezerc --action backup
--path-to-backup /var/log
--backup-name rsync-var-log-test-XX
--container var-log-test-XX
"""
max_retry = 5
# Set arguments
backup_args = {
#'proxy' : '',
'action' : 'backup',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
# Namespace backup_args object
name_list = []
retry = 0
while backup_args['container'] not in name_list and retry < max_retry:
ns_backup_args = main.freezer_main(backup_args)
ns_backup_args = swift.get_container_content(ns_backup_args)
name_list = [item['name'] for item in ns_backup_args.containers_list]
retry += 1
self.assertTrue(ns_backup_args.container in name_list)
self.assertTrue(segments_name(ns_backup_args.container) in name_list)
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
self.damage_tmp_tree(self.tmp_files)
# Restore
restore_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container'])
}
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
self.assertEqual(len(self.tmp_files), len(fdict_after))
for key in self.tmp_files:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def test_no_lvm_level0(self):
"""
Maximum level filesystem backup
freezerc --action backup
--path-to-backup /var/log
--backup-name rsync-var-log-test-XX
--container var-log-test-XX
"""
# Set arguments
test_args = {
#'proxy' : '',
'action' : 'backup',
'src_file' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
(backup_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(backup_args.mode, 'fs')
self.assertEqual(backup_args.max_backup_level, 0)
main.freezer_main(backup_args)
backup_args = swift.get_containers_list(backup_args)
name_list = [item['name'] for item in backup_args.containers_list]
self.assertTrue(backup_args.container in name_list)
self.assertTrue(backup_args.container_segments in name_list)
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
self.damage_tmp_tree(self.tmp_files)
# Restore
test_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args.backup_name),
'container' : copy(backup_args.container)
}
(restore_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(backup_args.mode, 'fs')
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
self.assertEqual(len(self.tmp_files), len(fdict_after))
for key in self.tmp_files:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def test_lvm_incremental_level5(self):
"""
Incremental LVM snapshots filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
--max-level 5
"""
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
max_level = 5
backup_args = {
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4()),
'max_level' : max_level
}
fdict_before = []
# print ''
for i in range(0, max_level):
# print "TEST FILE CONTENT BEFORE BACKUP %s:" % i
# print open(self.tmp_path + os.path.sep + 'foo', 'r').read()
fdict_before.append(
self.snap_tmp_tree_sha1(self.tmp_files)
)
ns_backup_args = main.freezer_main(backup_args)
self.damage_tmp_tree(self.tmp_files)
# time.sleep(2)
# Filter only the container names from all other data
ns_backup_args = swift.get_container_content(
ns_backup_args.client_manager,
ns_backup_args.container)
name_list = [item['name'] for item in ns_backup_args]
for counter in range(0, max_level):
found_objects = [obj for obj in name_list if obj.endswith('_%s' % counter)]
objects_str = ' '.join(found_objects)
# print objects_str
self.assertEqual('%s(%s)' % (objects_str,
len(found_objects)), objects_str + '(2)')
found_objects = sorted(found_objects)
self.assertEqual(found_objects[1], found_objects[0][-len(found_objects[1]):])
# From max_level-1 downto 0
for i in range(max_level - 1, -1, -1):
restore_level = i
restore_epoch = re.findall('_(\d{10}?)_%s' % restore_level , ' '.join(name_list))[0]
restore_epoch = int(restore_epoch) + 0
restore_date = time.strftime('%Y-%m-%dT%H:%M:%S',
time.localtime(int(restore_epoch))
)
# print 'Restore level:', restore_level, restore_epoch, restore_date
# Remove all files in the whole testing directory
shutil.rmtree(self.tmp_path)
self.assertFalse(os.path.isdir(self.tmp_path))
os.makedirs(self.tmp_path)
# Build
restore_args = {
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container']),
'restore_from_date' : restore_date
}
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(fdict_before[restore_level]), len(fdict_after))
for key in fdict_before[restore_level]:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[restore_level][key], key + fdict_after[key])
def test_bandwith_limit(self):
"""
Freezer upload/download speed limit test. We set a fixed 512KB/s speed and
try to backup (upload) 1MB file na restore (download) the backup. Each of
those action on avarage should not take more than 2s or less than 3s
2s < EXEC_TIME < 3s. Without throttle it is normaly about 0.4s.
freezerc --action backup
--path-to-backup /tmp/freezer_test_XXXX
--backup-name UUID
--container UUID
--upload-limit 524288
freezerc --action restore
--path-to-backup /tmp/freezer_test_XXXX
--backup-name UUID
--container UUID
--download-limit 524288
"""
# print '\nWorking in:', self.tmp_path
# Set 512KB/s connection limit
speed_limit_bytes = 512 * 1024
time_low = 2
abs_file_name = self.tmp_path + os.path.sep + 'limitfoo'
# Create 1MB test text file with random data
self.create_big_file(abs_file_name, 2 * speed_limit_bytes)
# Freezer CLI for backup argument dictionary
backup_args = {
'action' : 'backup',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4()),
'upload_limit' : speed_limit_bytes
}
start_time = time.time()
# Call Freezer CLI backup
main.freezer_main(backup_args)
end_time = time.time()
# Calculate backup time in sec
upload_time = end_time - start_time
# print "\nUpload time: %g seconds" % upload_time
# Test that time is longer than the theoretical 2 sec
self.assertTrue(time_low < upload_time)
# Delete test file
os.unlink(abs_file_name)
# Build dictionary for Freezer CLI restore
restore_args = {
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container']),
'download_limit' : speed_limit_bytes
}
start_time = time.time()
# Call the actual Freezer CLI restore
main.freezer_main(restore_args)
end_time = time.time()
self.assertTrue(os.path.isfile(abs_file_name))
# Calculate restore time in sec
download_time = end_time - start_time
# print "Download time: %g seconds" % download_time
# sys.stdout.flush()
# Test that time is longer than the theoretical 2 sec
self.assertTrue(time_low < download_time)
def test_lvm_level0(self):
"""
LVM snapshot filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
"""
max_retry = 5
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
backup_args = {
#'proxy' : '',
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'exclude' : '*.lock',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
# Call the actual BACKUP
# Namespace backup_args object
name_list = []
retry = 0
while backup_args['container'] not in name_list and retry < max_retry:
ns_backup_args = main.freezer_main(backup_args)
ns_backup_args = swift.get_container_content(ns_backup_args)
name_list = [item['name'] for item in ns_backup_args.containers_list]
retry += 1
self.assertTrue(ns_backup_args.container in name_list)
self.assertTrue(segments_name(ns_backup_args.container) in name_list)
# Create a file => SAH1 hash dictionary that will recored file
# hashes before any files being modified or deleted
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
# Delete and modify random files in the test directory
# structure
self.damage_tmp_tree(self.tmp_files)
# RESTORE section
# Create RESTORE action dictionary to be passed to
# arguments.backup_arguments() they will emulate the
# command line arguments
restore_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container'])
}
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
# Check if cout of all original files match recovered files
# plus the number of deleted .LOCK files which were not restored
self.assertEqual(len(self.tmp_files), len(fdict_after) +
len([x for x in self.tmp_deleted if x.endswith('.lock')]))
for key in self.tmp_files:
if key.endswith('.lock') and key in self.tmp_deleted:
self.assertFalse(os.path.isfile(key))
elif key.endswith('.lock') and key in self.tmp_modified:
self.assertNotEqual(key + fdict_before[key], key + fdict_after[key])
else:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def test_freezer_main(monkeypatch):
monkeypatch.setattr(job, 'create_job', fake_create_job)
assert freezer_main() is None
def test_lvm_level0(self):
"""
LVM snapshot filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
"""
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
test_args = {
#'proxy' : '',
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'exclude' : '*.lock',
'src_file' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
(backup_args, _) = arguments.backup_arguments(test_args)
# Make sure default value for MODE is filesystem
self.assertEqual(backup_args.mode, 'fs')
# Check that if not explicitly defined the MAX-BACKUP is 0
self.assertEqual(backup_args.max_backup_level, 0)
# Call the actual BACKUP
main.freezer_main(backup_args)
# Retrieve a list of all container data on Swift
backup_args = swift.get_containers_list(backup_args)
# Filter only the container names from all other data
name_list = [item['name'] for item in backup_args.containers_list]
# Amke sure that we have created a container with the desired name
# in Swift
self.assertTrue(backup_args.container in name_list)
# Ensure that the SEGMENTS container is found on Swift as well
self.assertTrue(backup_args.container_segments in name_list)
# Create a file => SAH1 hash dictionary that will recored file
# hashes before any files being modified or deleted
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
# Delete and modify random files in the test directory
# structure
self.damage_tmp_tree(self.tmp_files)
# RESTORE section
# Create RESTORE action dictionary to be passed to
# arguments.backup_arguments() they will emulate the
# command line arguments
test_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args.backup_name),
'container' : copy(backup_args.container)
}
(restore_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(restore_args.mode, 'fs')
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
# Check if cout of all original files match recovered files
# plus the number of deleted .LOCK files which were not restored
self.assertEqual(len(self.tmp_files), len(fdict_after) +
len([x for x in self.tmp_deleted if x.endswith('.lock')]))
for key in self.tmp_files:
if key.endswith('.lock') and key in self.tmp_deleted:
self.assertFalse(os.path.isfile(key))
elif key.endswith('.lock') and key in self.tmp_modified:
self.assertNotEqual(key + fdict_before[key], key + fdict_after[key])
else:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])