def wrapper(self):
self.start_time = utils.DateTime.now()
self.conf.time_stamp = self.start_time.timestamp
logging.info('[*] Job execution Started at: {0}'.
format(self.start_time))
self.conf = swift.get_client(self.conf)
self.conf = swift.get_containers_list(self.conf)
retval = func(self)
end_time = utils.DateTime.now()
logging.info('[*] Job execution Finished, at: {0}'.
format(end_time))
logging.info('[*] Job time Elapsed: {0}'.
format(end_time - self.start_time))
return retval
def wrapper(self):
self.start_time = utils.DateTime.now()
self.conf.time_stamp = self.start_time.timestamp
logging.info('[*] Job execution Started at: {0}'.format(
self.start_time))
self.conf = swift.get_client(self.conf)
self.conf = swift.get_containers_list(self.conf)
retval = func(self)
end_time = utils.DateTime.now()
logging.info(
'[*] Job execution Finished, at: {0}'.format(end_time))
logging.info('[*] Job time Elapsed: {0}'.format(end_time -
self.start_time))
return retval
def test_no_lvm_level0(self):
"""
Maximum level filesystem backup
freezerc --action backup
--path-to-backup /var/log
--backup-name rsync-var-log-test-XX
--container var-log-test-XX
"""
# Set arguments
test_args = {
#'proxy' : '',
'action' : 'backup',
'src_file' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
(backup_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(backup_args.mode, 'fs')
self.assertEqual(backup_args.max_backup_level, 0)
main.freezer_main(backup_args)
backup_args = swift.get_containers_list(backup_args)
name_list = [item['name'] for item in backup_args.containers_list]
self.assertTrue(backup_args.container in name_list)
self.assertTrue(backup_args.container_segments in name_list)
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
self.damage_tmp_tree(self.tmp_files)
# Restore
test_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args.backup_name),
'container' : copy(backup_args.container)
}
(restore_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(backup_args.mode, 'fs')
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
self.assertEqual(len(self.tmp_files), len(fdict_after))
for key in self.tmp_files:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def test_get_containers_list(self, monkeypatch):
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
assert isinstance(get_containers_list(backup_opt), BackupOpt1) is True
fakeclient = FakeSwiftClient1()
fakeconnector = fakeclient.client()
fakeswclient = fakeconnector.Connection()
backup_opt = BackupOpt1()
backup_opt.sw_connector = fakeswclient
pytest.raises(Exception, get_containers_list, backup_opt)
def test_get_containers_list(self, monkeypatch):
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
assert isinstance(get_containers_list(backup_opt), BackupOpt1) is True
fakeclient = FakeSwiftClient1()
fakeconnector = fakeclient.client()
fakeswclient = fakeconnector.Connection()
backup_opt = BackupOpt1()
backup_opt.sw_connector = fakeswclient
pytest.raises(Exception, get_containers_list, backup_opt)
def freezer_main(backup_args):
'''
Program Main Execution. This main function is a wrapper for most
of the other functions. By calling main() the program execution start
and the respective actions are taken. If you want only use the single
function is probably better to not import main()
'''
# Computing execution start datetime and Timestamp
(time_stamp, today_start) = start_time()
# Add timestamp to the arguments namespace
backup_args.__dict__['time_stamp'] = time_stamp
# Initialize the swift connector and store it in the same dict passed
# as argument under the dict.sw_connector namespace. This is helpful
# so the swift client object doesn't need to be initialized every time
backup_args = get_client(backup_args)
# Get the list of the containers
backup_args = get_containers_list(backup_args)
if show_containers(backup_args):
elapsed_time(today_start)
return True
# Check if the provided container already exists in swift.
# If it doesn't exist a new one will be created along with the segments
# container as container_segments
backup_args = check_container_existance(backup_args)
# Get the object list of the remote containers and store id in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
if show_objects(backup_args):
elapsed_time(today_start)
return True
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args,
manifest_meta_dict) = set_backup_level(backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
# If any of the restore options was specified, then a data restore
# will be executed
if validate_any_args([
backup_args.restore_from_date, backup_args.restore_from_host,
backup_args.restore_abs_path
]):
logging.info('[*] Executing FS restore...')
restore_fs(backup_args)
else:
backup_mode_fs(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
remove_obj_older_than(backup_args)
# Elapsed time:
elapsed_time(today_start)
def test_lvm_level0(self):
"""
LVM snapshot filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
"""
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
test_args = {
#'proxy' : '',
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'exclude' : '*.lock',
'src_file' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
(backup_args, _) = arguments.backup_arguments(test_args)
# Make sure default value for MODE is filesystem
self.assertEqual(backup_args.mode, 'fs')
# Check that if not explicitly defined the MAX-BACKUP is 0
self.assertEqual(backup_args.max_backup_level, 0)
# Call the actual BACKUP
main.freezer_main(backup_args)
# Retrieve a list of all container data on Swift
backup_args = swift.get_containers_list(backup_args)
# Filter only the container names from all other data
name_list = [item['name'] for item in backup_args.containers_list]
# Amke sure that we have created a container with the desired name
# in Swift
self.assertTrue(backup_args.container in name_list)
# Ensure that the SEGMENTS container is found on Swift as well
self.assertTrue(backup_args.container_segments in name_list)
# Create a file => SAH1 hash dictionary that will recored file
# hashes before any files being modified or deleted
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
# Delete and modify random files in the test directory
# structure
self.damage_tmp_tree(self.tmp_files)
# RESTORE section
# Create RESTORE action dictionary to be passed to
# arguments.backup_arguments() they will emulate the
# command line arguments
test_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args.backup_name),
'container' : copy(backup_args.container)
}
(restore_args, _) = arguments.backup_arguments(test_args)
self.assertEqual(restore_args.mode, 'fs')
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
# Check if cout of all original files match recovered files
# plus the number of deleted .LOCK files which were not restored
self.assertEqual(len(self.tmp_files), len(fdict_after) +
len([x for x in self.tmp_deleted if x.endswith('.lock')]))
for key in self.tmp_files:
if key.endswith('.lock') and key in self.tmp_deleted:
self.assertFalse(os.path.isfile(key))
elif key.endswith('.lock') and key in self.tmp_modified:
self.assertNotEqual(key + fdict_before[key], key + fdict_after[key])
else:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def freezer_main(backup_args):
'''
Program Main Execution. This main function is a wrapper for most
of the other functions. By calling main() the program execution start
and the respective actions are taken. If you want only use the single
function is probably better to not import main()
'''
# Computing execution start datetime and Timestamp
(time_stamp, today_start) = start_time()
# Add timestamp to the arguments namespace
backup_args.__dict__['time_stamp'] = time_stamp
# Initialize the swift connector and store it in the same dict passed
# as argument under the dict.sw_connector namespace. This is helpful
# so the swift client object doesn't need to be initialized every time
backup_args = get_client(backup_args)
# Get the list of the containers
backup_args = get_containers_list(backup_args)
if show_containers(backup_args):
elapsed_time(today_start)
return True
# Check if the provided container already exists in swift.
# If it doesn't exist a new one will be created along with the segments
# container as container_segments
backup_args = check_container_existance(backup_args)
# Get the object list of the remote containers and store id in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
if show_objects(backup_args):
elapsed_time(today_start)
return True
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args, manifest_meta_dict) = set_backup_level(
backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
# If any of the restore options was specified, then a data restore
# will be executed
if validate_any_args([
backup_args.restore_from_date, backup_args.restore_from_host,
backup_args.restore_abs_path]):
logging.info('[*] Executing FS restore...')
restore_fs(backup_args)
else:
backup_mode_fs(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
remove_obj_older_than(backup_args)
# Elapsed time:
elapsed_time(today_start)