def execute(self): logging.info('[*] Executing FS restore...') # Check if the provided container already exists in swift. containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: raise ValueError('Container: {0} not found. Please provide an ' 'existing container.'.format(self.conf.container)) # Get the object list of the remote containers and store it in the # same dict passes as argument under the dict.remote_obj_list namespace self.conf = swift.get_container_content(self.conf) restore.restore_fs(self.conf)
def execute(self): logging.info('[*] Executing FS restore...') # Check if the provided container already exists in swift. containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: raise ValueError('Container: {0} not found. Please provide an ' 'existing container.' .format(self.conf.container)) # Get the object list of the remote containers and store it in the # same dict passes as argument under the dict.remote_obj_list namespace self.conf = swift.get_container_content(self.conf) restore.restore_fs(self.conf)
def test_get_container_content(self, monkeypatch): backup_opt = BackupOpt1() fakelogging = FakeLogging() monkeypatch.setattr(logging, 'critical', fakelogging.critical) monkeypatch.setattr(logging, 'warning', fakelogging.warning) monkeypatch.setattr(logging, 'exception', fakelogging.exception) monkeypatch.setattr(logging, 'error', fakelogging.error) assert get_container_content(backup_opt) is not False assert get_container_content(backup_opt) is not None backup_opt = BackupOpt1() backup_opt.container = False pytest.raises(Exception, get_container_content, backup_opt) fakeclient = FakeSwiftClient1() fakeconnector = fakeclient.client() fakeswclient = fakeconnector.Connection() backup_opt = BackupOpt1() backup_opt.sw_connector = fakeswclient pytest.raises(Exception, get_container_content, backup_opt)
def execute(self): if self.conf.list_container: swift.show_containers(self.conf) elif self.conf.list_objects: containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: logging.critical('[*] Container {0} not available'.format( self.conf.container)) return False self.conf = swift.get_container_content(self.conf) swift.show_objects(self.conf) else: logging.warning( '[*] No retrieving info options were set. Exiting.') return False return True
def execute(self): if self.conf.list_container: swift.show_containers(self.conf) elif self.conf.list_objects: containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: logging.critical( '[*] Container {0} not available'.format( self.conf.container)) return False self.conf = swift.get_container_content(self.conf) swift.show_objects(self.conf) else: logging.warning( '[*] No retrieving info options were set. Exiting.') return False return True
def test_no_lvm_level0(self): """ Maximum level filesystem backup freezerc --action backup --path-to-backup /var/log --backup-name rsync-var-log-test-XX --container var-log-test-XX """ max_retry = 5 # Set arguments backup_args = { #'proxy' : '', 'action' : 'backup', 'path_to_backup' : copy(self.tmp_path), 'backup_name' : str(uuid.uuid4()), 'container' : str(uuid.uuid4()) } # Namespace backup_args object name_list = [] retry = 0 while backup_args['container'] not in name_list and retry < max_retry: ns_backup_args = main.freezer_main(backup_args) ns_backup_args = swift.get_container_content(ns_backup_args) name_list = [item['name'] for item in ns_backup_args.containers_list] retry += 1 self.assertTrue(ns_backup_args.container in name_list) self.assertTrue(segments_name(ns_backup_args.container) in name_list) fdict_before = self.snap_tmp_tree_sha1(self.tmp_files) self.damage_tmp_tree(self.tmp_files) # Restore restore_args = { #'proxy' : '', 'action' : 'restore', 'restore_abs_path' : copy(self.tmp_path), 'backup_name' : copy(backup_args['backup_name']), 'container' : copy(backup_args['container']) } main.freezer_main(restore_args) fdict_after = self.snap_tmp_tree_sha1(self.tmp_files) self.assertEqual(len(self.tmp_files), len(fdict_before)) self.assertEqual(len(self.tmp_files), len(fdict_after)) for key in self.tmp_files: self.assertTrue(os.path.isfile(key)) self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def execute(self): containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: swift.create_containers(self.conf) if self.conf.no_incremental: if self.conf.max_backup_level or \ self.conf.always_backup_level: raise Exception( 'no-incremental option is not compatible ' 'with backup level options') manifest_meta_dict = {} else: # Get the object list of the remote containers # and store it in self.conf.remote_obj_list self.conf = swift.get_container_content(self.conf) # Check if a backup exist in swift with same name. # If not, set backup level to 0 manifest_meta_dict =\ utils.check_backup_and_tar_meta_existence(self.conf) (self.conf, manifest_meta_dict) = utils.set_backup_level( self.conf, manifest_meta_dict) self.conf.manifest_meta_dict = manifest_meta_dict if self.conf.mode == 'fs': backup.backup_mode_fs( self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'mongo': backup.backup_mode_mongo( self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'mysql': backup.backup_mode_mysql( self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'sqlserver': backup.backup_mode_sql_server( self.conf, self.time_stamp, manifest_meta_dict) else: raise ValueError('Please provide a valid backup mode')
def execute(self): containers = swift.check_container_existance(self.conf) if containers['main_container'] is not True: swift.create_containers(self.conf) if self.conf.no_incremental: if self.conf.max_backup_level or \ self.conf.always_backup_level: raise Exception('no-incremental option is not compatible ' 'with backup level options') manifest_meta_dict = {} else: # Get the object list of the remote containers # and store it in self.conf.remote_obj_list self.conf = swift.get_container_content(self.conf) # Check if a backup exist in swift with same name. # If not, set backup level to 0 manifest_meta_dict =\ utils.check_backup_and_tar_meta_existence(self.conf) (self.conf, manifest_meta_dict) = utils.set_backup_level(self.conf, manifest_meta_dict) self.conf.manifest_meta_dict = manifest_meta_dict if self.conf.mode == 'fs': backup.backup_mode_fs(self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'mongo': backup.backup_mode_mongo(self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'mysql': backup.backup_mode_mysql(self.conf, self.start_time.timestamp, manifest_meta_dict) elif self.conf.mode == 'sqlserver': backup.backup_mode_sql_server(self.conf, self.time_stamp, manifest_meta_dict) else: raise ValueError('Please provide a valid backup mode')
def execute(self): self.conf = swift.get_container_content(self.conf) swift.remove_obj_older_than(self.conf)
def test_lvm_incremental_level5(self): """ Incremental LVM snapshots filesystem backup freezerc --action backup --lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol --lvm-dirmount /tmp/freezer-test-lvm-snapshot --lvm-volgroup freezer-test1-volgroup --lvm-snapsize 1M --file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/ --container UUID --exclude "\*.lock" --backup-name UUID --max-level 5 """ # Set arguments lvm_path = '/mnt/freezer-test-lvm' self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path) self.create_tmp_tree(self.tmp_path) max_level = 5 backup_args = { 'action' : 'backup', 'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol', 'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot', 'lvm_volgroup' : 'freezer-test1-volgroup', 'lvm_snapsize' : '1M', 'path_to_backup' : copy(self.tmp_path), 'backup_name' : str(uuid.uuid4()), 'container' : str(uuid.uuid4()), 'max_level' : max_level } fdict_before = [] # print '' for i in range(0, max_level): # print "TEST FILE CONTENT BEFORE BACKUP %s:" % i # print open(self.tmp_path + os.path.sep + 'foo', 'r').read() fdict_before.append( self.snap_tmp_tree_sha1(self.tmp_files) ) ns_backup_args = main.freezer_main(backup_args) self.damage_tmp_tree(self.tmp_files) # time.sleep(2) # Filter only the container names from all other data ns_backup_args = swift.get_container_content( ns_backup_args.client_manager, ns_backup_args.container) name_list = [item['name'] for item in ns_backup_args] for counter in range(0, max_level): found_objects = [obj for obj in name_list if obj.endswith('_%s' % counter)] objects_str = ' '.join(found_objects) # print objects_str self.assertEqual('%s(%s)' % (objects_str, len(found_objects)), objects_str + '(2)') found_objects = sorted(found_objects) self.assertEqual(found_objects[1], found_objects[0][-len(found_objects[1]):]) # From max_level-1 downto 0 for i in range(max_level - 1, -1, -1): restore_level = i restore_epoch = re.findall('_(\d{10}?)_%s' % restore_level , ' '.join(name_list))[0] restore_epoch = int(restore_epoch) + 0 restore_date = time.strftime('%Y-%m-%dT%H:%M:%S', time.localtime(int(restore_epoch)) ) # print 'Restore level:', restore_level, restore_epoch, restore_date # Remove all files in the whole testing directory shutil.rmtree(self.tmp_path) self.assertFalse(os.path.isdir(self.tmp_path)) os.makedirs(self.tmp_path) # Build restore_args = { 'action' : 'restore', 'restore_abs_path' : copy(self.tmp_path), 'backup_name' : copy(backup_args['backup_name']), 'container' : copy(backup_args['container']), 'restore_from_date' : restore_date } # Call RESTORE on Freezer code base main.freezer_main(restore_args) fdict_after = self.snap_tmp_tree_sha1(self.tmp_files) self.assertEqual(len(fdict_before[restore_level]), len(fdict_after)) for key in fdict_before[restore_level]: self.assertTrue(os.path.isfile(key)) self.assertEqual(key + fdict_before[restore_level][key], key + fdict_after[key])
def test_lvm_level0(self): """ LVM snapshot filesystem backup freezerc --action backup --lvm-srcvol /dev/freezer-test1-volgroup/freezer-test1-vol --lvm-dirmount /tmp/freezer-test-lvm-snapshot --lvm-volgroup freezer-test1-volgroup --lvm-snapsize 1M --file-to-backup /mnt/freezer-test-lvm/lvm_test_XXXX/ --container UUID --exclude "\*.lock" --backup-name UUID """ max_retry = 5 # Set arguments lvm_path = '/mnt/freezer-test-lvm' self.tmp_path = tempfile.mkdtemp(prefix='lvm_test_', dir=lvm_path) self.create_tmp_tree(self.tmp_path) backup_args = { #'proxy' : '', 'action' : 'backup', 'lvm_srcvol' : '/dev/freezer-test1-volgroup/freezer-test1-vol', 'lvm_dirmount' : '/tmp/freezer-test-lvm-snapshot', 'lvm_volgroup' : 'freezer-test1-volgroup', 'lvm_snapsize' : '1M', 'exclude' : '*.lock', 'path_to_backup' : copy(self.tmp_path), 'backup_name' : str(uuid.uuid4()), 'container' : str(uuid.uuid4()) } # Call the actual BACKUP # Namespace backup_args object name_list = [] retry = 0 while backup_args['container'] not in name_list and retry < max_retry: ns_backup_args = main.freezer_main(backup_args) ns_backup_args = swift.get_container_content(ns_backup_args) name_list = [item['name'] for item in ns_backup_args.containers_list] retry += 1 self.assertTrue(ns_backup_args.container in name_list) self.assertTrue(segments_name(ns_backup_args.container) in name_list) # Create a file => SAH1 hash dictionary that will recored file # hashes before any files being modified or deleted fdict_before = self.snap_tmp_tree_sha1(self.tmp_files) # Delete and modify random files in the test directory # structure self.damage_tmp_tree(self.tmp_files) # RESTORE section # Create RESTORE action dictionary to be passed to # arguments.backup_arguments() they will emulate the # command line arguments restore_args = { #'proxy' : '', 'action' : 'restore', 'restore_abs_path' : copy(self.tmp_path), 'backup_name' : copy(backup_args['backup_name']), 'container' : copy(backup_args['container']) } # Call RESTORE on Freezer code base main.freezer_main(restore_args) fdict_after = self.snap_tmp_tree_sha1(self.tmp_files) self.assertEqual(len(self.tmp_files), len(fdict_before)) # Check if cout of all original files match recovered files # plus the number of deleted .LOCK files which were not restored self.assertEqual(len(self.tmp_files), len(fdict_after) + len([x for x in self.tmp_deleted if x.endswith('.lock')])) for key in self.tmp_files: if key.endswith('.lock') and key in self.tmp_deleted: self.assertFalse(os.path.isfile(key)) elif key.endswith('.lock') and key in self.tmp_modified: self.assertNotEqual(key + fdict_before[key], key + fdict_after[key]) else: self.assertTrue(os.path.isfile(key)) self.assertEqual(key + fdict_before[key], key + fdict_after[key])
def freezer_main(backup_args): ''' Program Main Execution. This main function is a wrapper for most of the other functions. By calling main() the program execution start and the respective actions are taken. If you want only use the single function is probably better to not import main() ''' # Computing execution start datetime and Timestamp (time_stamp, today_start) = start_time() # Add timestamp to the arguments namespace backup_args.__dict__['time_stamp'] = time_stamp # Initialize the swift connector and store it in the same dict passed # as argument under the dict.sw_connector namespace. This is helpful # so the swift client object doesn't need to be initialized every time backup_args = get_client(backup_args) # Get the list of the containers backup_args = get_containers_list(backup_args) if show_containers(backup_args): elapsed_time(today_start) return True # Check if the provided container already exists in swift. # If it doesn't exist a new one will be created along with the segments # container as container_segments backup_args = check_container_existance(backup_args) # Get the object list of the remote containers and store id in the # same dict passes as argument under the dict.remote_obj_list namespace backup_args = get_container_content(backup_args) if show_objects(backup_args): elapsed_time(today_start) return True # Check if a backup exist in swift with same name. If not, set # backup level to 0 manifest_meta_dict = check_backup_existance(backup_args) # Set the right backup level for incremental backup (backup_args, manifest_meta_dict) = set_backup_level(backup_args, manifest_meta_dict) backup_args.manifest_meta_dict = manifest_meta_dict # File system backup mode selected if backup_args.mode == 'fs': # If any of the restore options was specified, then a data restore # will be executed if validate_any_args([ backup_args.restore_from_date, backup_args.restore_from_host, backup_args.restore_abs_path ]): logging.info('[*] Executing FS restore...') restore_fs(backup_args) else: backup_mode_fs(backup_args, time_stamp, manifest_meta_dict) elif backup_args.mode == 'mongo': backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict) elif backup_args.mode == 'mysql': backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict) else: logging.critical('[*] Error: Please provide a valid backup mode') raise ValueError remove_obj_older_than(backup_args) # Elapsed time: elapsed_time(today_start)
def freezer_main(backup_args): ''' Program Main Execution. This main function is a wrapper for most of the other functions. By calling main() the program execution start and the respective actions are taken. If you want only use the single function is probably better to not import main() ''' # Computing execution start datetime and Timestamp (time_stamp, today_start) = start_time() # Add timestamp to the arguments namespace backup_args.__dict__['time_stamp'] = time_stamp # Initialize the swift connector and store it in the same dict passed # as argument under the dict.sw_connector namespace. This is helpful # so the swift client object doesn't need to be initialized every time backup_args = get_client(backup_args) # Get the list of the containers backup_args = get_containers_list(backup_args) if show_containers(backup_args): elapsed_time(today_start) return True # Check if the provided container already exists in swift. # If it doesn't exist a new one will be created along with the segments # container as container_segments backup_args = check_container_existance(backup_args) # Get the object list of the remote containers and store id in the # same dict passes as argument under the dict.remote_obj_list namespace backup_args = get_container_content(backup_args) if show_objects(backup_args): elapsed_time(today_start) return True # Check if a backup exist in swift with same name. If not, set # backup level to 0 manifest_meta_dict = check_backup_existance(backup_args) # Set the right backup level for incremental backup (backup_args, manifest_meta_dict) = set_backup_level( backup_args, manifest_meta_dict) backup_args.manifest_meta_dict = manifest_meta_dict # File system backup mode selected if backup_args.mode == 'fs': # If any of the restore options was specified, then a data restore # will be executed if validate_any_args([ backup_args.restore_from_date, backup_args.restore_from_host, backup_args.restore_abs_path]): logging.info('[*] Executing FS restore...') restore_fs(backup_args) else: backup_mode_fs(backup_args, time_stamp, manifest_meta_dict) elif backup_args.mode == 'mongo': backup_mode_mongo(backup_args, time_stamp, manifest_meta_dict) elif backup_args.mode == 'mysql': backup_mode_mysql(backup_args, time_stamp, manifest_meta_dict) else: logging.critical('[*] Error: Please provide a valid backup mode') raise ValueError remove_obj_older_than(backup_args) # Elapsed time: elapsed_time(today_start)