def execute(self):
logging.info('[*] Executing FS restore...')
# Check if the provided container already exists in swift.
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
raise ValueError('Container: {0} not found. Please provide an '
'existing container.'.format(self.conf.container))
# Get the object list of the remote containers and store it in the
# same dict passes as argument under the dict.remote_obj_list namespace
self.conf = swift.get_container_content(self.conf)
restore.restore_fs(self.conf)
def execute(self):
logging.info('[*] Executing FS restore...')
# Check if the provided container already exists in swift.
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
raise ValueError('Container: {0} not found. Please provide an '
'existing container.'
.format(self.conf.container))
# Get the object list of the remote containers and store it in the
# same dict passes as argument under the dict.remote_obj_list namespace
self.conf = swift.get_container_content(self.conf)
restore.restore_fs(self.conf)
def test_get_container_content(self, monkeypatch):
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
assert get_container_content(backup_opt) is not False
assert get_container_content(backup_opt) is not None
backup_opt = BackupOpt1()
backup_opt.container = False
pytest.raises(Exception, get_container_content, backup_opt)
fakeclient = FakeSwiftClient1()
fakeconnector = fakeclient.client()
fakeswclient = fakeconnector.Connection()
backup_opt = BackupOpt1()
backup_opt.sw_connector = fakeswclient
pytest.raises(Exception, get_container_content, backup_opt)
def test_get_container_content(self, monkeypatch):
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
assert get_container_content(backup_opt) is not False
assert get_container_content(backup_opt) is not None
backup_opt = BackupOpt1()
backup_opt.container = False
pytest.raises(Exception, get_container_content, backup_opt)
fakeclient = FakeSwiftClient1()
fakeconnector = fakeclient.client()
fakeswclient = fakeconnector.Connection()
backup_opt = BackupOpt1()
backup_opt.sw_connector = fakeswclient
pytest.raises(Exception, get_container_content, backup_opt)
def execute(self):
if self.conf.list_container:
swift.show_containers(self.conf)
elif self.conf.list_objects:
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
logging.critical('[*] Container {0} not available'.format(
self.conf.container))
return False
self.conf = swift.get_container_content(self.conf)
swift.show_objects(self.conf)
else:
logging.warning(
'[*] No retrieving info options were set. Exiting.')
return False
return True
def execute(self):
if self.conf.list_container:
swift.show_containers(self.conf)
elif self.conf.list_objects:
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
logging.critical(
'[*] Container {0} not available'.format(
self.conf.container))
return False
self.conf = swift.get_container_content(self.conf)
swift.show_objects(self.conf)
else:
logging.warning(
'[*] No retrieving info options were set. Exiting.')
return False
return True
def test_no_lvm_level0(self):
"""
Maximum level filesystem backup
freezerc --action backup
--path-to-backup /var/log
--backup-name rsync-var-log-test-XX
--container var-log-test-XX
"""
max_retry = 5
# Set arguments
backup_args = {
#'proxy' : '',
'action' : 'backup',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
# Namespace backup_args object
name_list = []
retry = 0
while backup_args['container'] not in name_list and retry < max_retry:
ns_backup_args = main.freezer_main(backup_args)
ns_backup_args = swift.get_container_content(ns_backup_args)
name_list = [item['name'] for item in ns_backup_args.containers_list]
retry += 1
self.assertTrue(ns_backup_args.container in name_list)
self.assertTrue(segments_name(ns_backup_args.container) in name_list)
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
self.damage_tmp_tree(self.tmp_files)
# Restore
restore_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container'])
}
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
self.assertEqual(len(self.tmp_files), len(fdict_after))
for key in self.tmp_files:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def execute(self):
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
swift.create_containers(self.conf)
if self.conf.no_incremental:
if self.conf.max_backup_level or \
self.conf.always_backup_level:
raise Exception(
'no-incremental option is not compatible '
'with backup level options')
manifest_meta_dict = {}
else:
# Get the object list of the remote containers
# and store it in self.conf.remote_obj_list
self.conf = swift.get_container_content(self.conf)
# Check if a backup exist in swift with same name.
# If not, set backup level to 0
manifest_meta_dict =\
utils.check_backup_and_tar_meta_existence(self.conf)
(self.conf, manifest_meta_dict) = utils.set_backup_level(
self.conf, manifest_meta_dict)
self.conf.manifest_meta_dict = manifest_meta_dict
if self.conf.mode == 'fs':
backup.backup_mode_fs(
self.conf, self.start_time.timestamp, manifest_meta_dict)
elif self.conf.mode == 'mongo':
backup.backup_mode_mongo(
self.conf, self.start_time.timestamp, manifest_meta_dict)
elif self.conf.mode == 'mysql':
backup.backup_mode_mysql(
self.conf, self.start_time.timestamp, manifest_meta_dict)
elif self.conf.mode == 'sqlserver':
backup.backup_mode_sql_server(
self.conf, self.time_stamp, manifest_meta_dict)
else:
raise ValueError('Please provide a valid backup mode')
def execute(self):
containers = swift.check_container_existance(self.conf)
if containers['main_container'] is not True:
swift.create_containers(self.conf)
if self.conf.no_incremental:
if self.conf.max_backup_level or \
self.conf.always_backup_level:
raise Exception('no-incremental option is not compatible '
'with backup level options')
manifest_meta_dict = {}
else:
# Get the object list of the remote containers
# and store it in self.conf.remote_obj_list
self.conf = swift.get_container_content(self.conf)
# Check if a backup exist in swift with same name.
# If not, set backup level to 0
manifest_meta_dict =\
utils.check_backup_and_tar_meta_existence(self.conf)
(self.conf,
manifest_meta_dict) = utils.set_backup_level(self.conf,
manifest_meta_dict)
self.conf.manifest_meta_dict = manifest_meta_dict
if self.conf.mode == 'fs':
backup.backup_mode_fs(self.conf, self.start_time.timestamp,
manifest_meta_dict)
elif self.conf.mode == 'mongo':
backup.backup_mode_mongo(self.conf, self.start_time.timestamp,
manifest_meta_dict)
elif self.conf.mode == 'mysql':
backup.backup_mode_mysql(self.conf, self.start_time.timestamp,
manifest_meta_dict)
elif self.conf.mode == 'sqlserver':
backup.backup_mode_sql_server(self.conf, self.time_stamp,
manifest_meta_dict)
else:
raise ValueError('Please provide a valid backup mode')
def execute(self):
self.conf = swift.get_container_content(self.conf)
swift.remove_obj_older_than(self.conf)
def test_lvm_incremental_level5(self):
"""
Incremental LVM snapshots filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
--max-level 5
"""
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
max_level = 5
backup_args = {
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4()),
'max_level' : max_level
}
fdict_before = []
# print ''
for i in range(0, max_level):
# print "TEST FILE CONTENT BEFORE BACKUP %s:" % i
# print open(self.tmp_path + os.path.sep + 'foo', 'r').read()
fdict_before.append(
self.snap_tmp_tree_sha1(self.tmp_files)
)
ns_backup_args = main.freezer_main(backup_args)
self.damage_tmp_tree(self.tmp_files)
# time.sleep(2)
# Filter only the container names from all other data
ns_backup_args = swift.get_container_content(
ns_backup_args.client_manager,
ns_backup_args.container)
name_list = [item['name'] for item in ns_backup_args]
for counter in range(0, max_level):
found_objects = [obj for obj in name_list if obj.endswith('_%s' % counter)]
objects_str = ' '.join(found_objects)
# print objects_str
self.assertEqual('%s(%s)' % (objects_str,
len(found_objects)), objects_str + '(2)')
found_objects = sorted(found_objects)
self.assertEqual(found_objects[1], found_objects[0][-len(found_objects[1]):])
# From max_level-1 downto 0
for i in range(max_level - 1, -1, -1):
restore_level = i
restore_epoch = re.findall('_(\d{10}?)_%s' % restore_level , ' '.join(name_list))[0]
restore_epoch = int(restore_epoch) + 0
restore_date = time.strftime('%Y-%m-%dT%H:%M:%S',
time.localtime(int(restore_epoch))
)
# print 'Restore level:', restore_level, restore_epoch, restore_date
# Remove all files in the whole testing directory
shutil.rmtree(self.tmp_path)
self.assertFalse(os.path.isdir(self.tmp_path))
os.makedirs(self.tmp_path)
# Build
restore_args = {
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container']),
'restore_from_date' : restore_date
}
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(fdict_before[restore_level]), len(fdict_after))
for key in fdict_before[restore_level]:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[restore_level][key], key + fdict_after[key])
def test_lvm_level0(self):
"""
LVM snapshot filesystem backup
freezerc --action backup
--lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol
--lvm-dirmount /tmp/freezer-test-lvm-snapshot
--lvm-volgroup freezer-test1-volgroup
--lvm-snapsize 1M
--file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/
--container UUID
--exclude "\*.lock"
--backup-name UUID
"""
max_retry = 5
# Set arguments
lvm_path = '/mnt/freezer-test-lvm'
self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path)
self.create_tmp_tree(self.tmp_path)
backup_args = {
#'proxy' : '',
'action' : 'backup',
'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol',
'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot',
'lvm_volgroup' : 'freezer-test1-volgroup',
'lvm_snapsize' : '1M',
'exclude' : '*.lock',
'path_to_backup' : copy(self.tmp_path),
'backup_name' : str(uuid.uuid4()),
'container' : str(uuid.uuid4())
}
# Call the actual BACKUP
# Namespace backup_args object
name_list = []
retry = 0
while backup_args['container'] not in name_list and retry < max_retry:
ns_backup_args = main.freezer_main(backup_args)
ns_backup_args = swift.get_container_content(ns_backup_args)
name_list = [item['name'] for item in ns_backup_args.containers_list]
retry += 1
self.assertTrue(ns_backup_args.container in name_list)
self.assertTrue(segments_name(ns_backup_args.container) in name_list)
# Create a file => SAH1 hash dictionary that will recored file
# hashes before any files being modified or deleted
fdict_before = self.snap_tmp_tree_sha1(self.tmp_files)
# Delete and modify random files in the test directory
# structure
self.damage_tmp_tree(self.tmp_files)
# RESTORE section
# Create RESTORE action dictionary to be passed to
# arguments.backup_arguments() they will emulate the
# command line arguments
restore_args = {
#'proxy' : '',
'action' : 'restore',
'restore_abs_path' : copy(self.tmp_path),
'backup_name' : copy(backup_args['backup_name']),
'container' : copy(backup_args['container'])
}
# Call RESTORE on Freezer code base
main.freezer_main(restore_args)
fdict_after = self.snap_tmp_tree_sha1(self.tmp_files)
self.assertEqual(len(self.tmp_files), len(fdict_before))
# Check if cout of all original files match recovered files
# plus the number of deleted .LOCK files which were not restored
self.assertEqual(len(self.tmp_files), len(fdict_after) +
len([x for x in self.tmp_deleted if x.endswith('.lock')]))
for key in self.tmp_files:
if key.endswith('.lock') and key in self.tmp_deleted:
self.assertFalse(os.path.isfile(key))
elif key.endswith('.lock') and key in self.tmp_modified:
self.assertNotEqual(key + fdict_before[key], key + fdict_after[key])
else:
self.assertTrue(os.path.isfile(key))
self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def execute(self):
self.conf = swift.get_container_content(self.conf)
swift.remove_obj_older_than(self.conf)
def freezer_main(backup_args):
'''
Program Main Execution. This main function is a wrapper for most
of the other functions. By calling main() the program execution start
and the respective actions are taken. If you want only use the single
function is probably better to not import main()
'''
# Computing execution start datetime and Timestamp
(time_stamp, today_start) = start_time()
# Add timestamp to the arguments namespace
backup_args.__dict__['time_stamp'] = time_stamp
# Initialize the swift connector and store it in the same dict passed
# as argument under the dict.sw_connector namespace. This is helpful
# so the swift client object doesn't need to be initialized every time
backup_args = get_client(backup_args)
# Get the list of the containers
backup_args = get_containers_list(backup_args)
if show_containers(backup_args):
elapsed_time(today_start)
return True
# Check if the provided container already exists in swift.
# If it doesn't exist a new one will be created along with the segments
# container as container_segments
backup_args = check_container_existance(backup_args)
# Get the object list of the remote containers and store id in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
if show_objects(backup_args):
elapsed_time(today_start)
return True
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args,
manifest_meta_dict) = set_backup_level(backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
# If any of the restore options was specified, then a data restore
# will be executed
if validate_any_args([
backup_args.restore_from_date, backup_args.restore_from_host,
backup_args.restore_abs_path
]):
logging.info('[*] Executing FS restore...')
restore_fs(backup_args)
else:
backup_mode_fs(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
remove_obj_older_than(backup_args)
# Elapsed time:
elapsed_time(today_start)
def freezer_main(backup_args):
'''
Program Main Execution. This main function is a wrapper for most
of the other functions. By calling main() the program execution start
and the respective actions are taken. If you want only use the single
function is probably better to not import main()
'''
# Computing execution start datetime and Timestamp
(time_stamp, today_start) = start_time()
# Add timestamp to the arguments namespace
backup_args.__dict__['time_stamp'] = time_stamp
# Initialize the swift connector and store it in the same dict passed
# as argument under the dict.sw_connector namespace. This is helpful
# so the swift client object doesn't need to be initialized every time
backup_args = get_client(backup_args)
# Get the list of the containers
backup_args = get_containers_list(backup_args)
if show_containers(backup_args):
elapsed_time(today_start)
return True
# Check if the provided container already exists in swift.
# If it doesn't exist a new one will be created along with the segments
# container as container_segments
backup_args = check_container_existance(backup_args)
# Get the object list of the remote containers and store id in the
# same dict passes as argument under the dict.remote_obj_list namespace
backup_args = get_container_content(backup_args)
if show_objects(backup_args):
elapsed_time(today_start)
return True
# Check if a backup exist in swift with same name. If not, set
# backup level to 0
manifest_meta_dict = check_backup_existance(backup_args)
# Set the right backup level for incremental backup
(backup_args, manifest_meta_dict) = set_backup_level(
backup_args, manifest_meta_dict)
backup_args.manifest_meta_dict = manifest_meta_dict
# File system backup mode selected
if backup_args.mode == 'fs':
# If any of the restore options was specified, then a data restore
# will be executed
if validate_any_args([
backup_args.restore_from_date, backup_args.restore_from_host,
backup_args.restore_abs_path]):
logging.info('[*] Executing FS restore...')
restore_fs(backup_args)
else:
backup_mode_fs(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mongo':
backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict)
elif backup_args.mode == 'mysql':
backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict)
else:
logging.critical('[*] Error: Please provide a valid backup mode')
raise ValueError
remove_obj_older_than(backup_args)
# Elapsed time:
elapsed_time(today_start)
