def snapshot_create(backup_opt_dict): if is_windows(): if backup_opt_dict.vssadmin: # Create a shadow copy. backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ vss_create_shadow_copy(backup_opt_dict.windows_volume) # execute this after the snapshot creation if backup_opt_dict.mode == 'sqlserver': start_sql_server(backup_opt_dict) else: # If lvm_auto_snap is true, the volume group and volume name will # be extracted automatically if backup_opt_dict.lvm_auto_snap: lvm_list = get_lvm_info( backup_opt_dict.lvm_auto_snap) backup_opt_dict.lvm_volgroup = lvm_list[0] backup_opt_dict.lvm_srcvol = lvm_list[2] # Generate the lvm_snap if lvm arguments are available lvm_snap(backup_opt_dict) if is_windows() and backup_opt_dict.vssadmin: backup_opt_dict.path_to_backup = use_shadow( backup_opt_dict.path_to_backup, backup_opt_dict.windows_volume) return backup_opt_dict
def test_snapshot_mount_error_raises_Exception(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen, mock_lvm_snap_remove): mock_get_vol_fs_type.return_value = 'xfs' mock_get_lvm_info.return_value = { 'volgroup': 'lvm_volgroup', 'srcvol': 'lvm_device', 'snap_path': 'snap_path'} mock_lvcreate_process, mock_mount_process = Mock(), Mock() mock_lvcreate_process.communicate.return_value = '', '' mock_lvcreate_process.returncode = 0 mock_mount_process.communicate.return_value = '', 'mount error' mock_mount_process.returncode = 1 mock_popen.side_effect = [mock_lvcreate_process, mock_mount_process] backup_opt = Mock() backup_opt.snapshot = True backup_opt.lvm_auto_snap = '' backup_opt.path_to_backup = '/just/a/path' backup_opt.lvm_dirmount = '/var/mountpoint' backup_opt.lvm_snapperm = 'ro' with self.assertRaises(Exception) as cm: lvm.lvm_snap(backup_opt) the_exception = cm.exception self.assertIn('lvm snapshot mounting error', the_exception.message) mock_lvm_snap_remove.assert_called_once_with(backup_opt)
def test_mysql_mode_locks_unlocks_tables(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen): mock_get_vol_fs_type.return_value = 'xfs' mock_get_lvm_info.return_value = { 'volgroup': 'lvm_volgroup', 'srcvol': 'lvm_device', 'snap_path': 'snap_path'} mock_process = Mock() mock_process.communicate.return_value = '', '' mock_process.returncode = 0 mock_popen.return_value = mock_process backup_opt = Mock() backup_opt.snapshot = True backup_opt.lvm_auto_snap = '' backup_opt.path_to_backup = '/just/a/path' backup_opt.lvm_dirmount = '/var/mountpoint' backup_opt.lvm_snapperm = 'ro' backup_opt.mode = 'mysql' backup_opt.mysql_db_inst = Mock() mock_cursor = Mock() backup_opt.mysql_db_inst.cursor.return_value = mock_cursor self.assertTrue(lvm.lvm_snap(backup_opt)) first_call = call('FLUSH TABLES WITH READ LOCK') second_call = call('UNLOCK TABLES') self.assertEquals(first_call, mock_cursor.execute.call_args_list[0]) self.assertEquals(second_call, mock_cursor.execute.call_args_list[1])
def snapshot_create(backup_opt_dict): """ Calls the code to take fs snapshots, depending on the platform :param backup_opt_dict: :return: boolean value, True if snapshot has been taken, false otherwise """ if is_windows(): # vssadmin is to be deprecated in favor of the --snapshot flag if backup_opt_dict.snapshot: backup_opt_dict.vssadmin = True if backup_opt_dict.vssadmin: # Create a shadow copy. backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ vss_create_shadow_copy(backup_opt_dict.windows_volume) backup_opt_dict.path_to_backup = use_shadow( backup_opt_dict.path_to_backup, backup_opt_dict.windows_volume) # execute this after the snapshot creation if backup_opt_dict.mode == 'sqlserver': start_sql_server(backup_opt_dict.sql_server_instance) return True return False else: return lvm.lvm_snap(backup_opt_dict)
def test_with_snapshot_opt_simple_sets_correct_path_and_raises_on_perm(self, mock_create_dir, mock_get_lvm_info): mock_get_lvm_info.return_value = { 'volgroup': 'lvm_volgroup', 'srcvol': 'lvm_device', 'snap_path': 'snap_path'} backup_opt = Mock() backup_opt.snapshot = True backup_opt.lvm_auto_snap = '' backup_opt.path_to_backup = '/just/a/path' backup_opt.lvm_dirmount = '/var/mountpoint' backup_opt.lvm_snapperm = 'invalid_value' with self.assertRaises(Exception) as cm: lvm.lvm_snap(backup_opt) the_exception = cm.exception self.assertIn('Invalid value for option lvm-snap-perm', the_exception.message)
def test_snapshot_fails(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen, mock_validate_lvm_params): mock_get_lvm_info.return_value = { 'volgroup': 'lvm_volgroup', 'srcvol': 'lvm_device', 'snap_path': 'snap_path'} mock_process = Mock() mock_process.communicate.return_value = '', '' mock_process.returncode = 1 mock_popen.return_value = mock_process backup_opt = Mock() backup_opt.snapshot = True backup_opt.lvm_auto_snap = '' backup_opt.path_to_backup = '/just/a/path' backup_opt.lvm_dirmount = '/var/mountpoint' backup_opt.lvm_snapperm = 'ro' with self.assertRaises(Exception) as cm: lvm.lvm_snap(backup_opt) the_exception = cm.exception self.assertIn('lvm snapshot creation error', the_exception.message)
def test_ok(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen, mock_validate_lvm_params): mock_get_lvm_info.return_value = { 'volgroup': 'lvm_volgroup', 'srcvol': 'lvm_device', 'snap_path': 'snap_path'} mock_process = Mock() mock_process.communicate.return_value = '', '' mock_process.returncode = 0 mock_popen.return_value = mock_process backup_opt = Mock() backup_opt.snapshot = True backup_opt.lvm_auto_snap = '' backup_opt.path_to_backup = '/just/a/path' backup_opt.lvm_dirmount = '/var/mountpoint' backup_opt.lvm_snapperm = 'ro' backup_opt.lvm_volgroup = '' backup_opt.lvm_srcvol = '' self.assertTrue(lvm.lvm_snap(backup_opt))
def snapshot_create(backup_opt_dict): """ Calls the code to take fs snapshots, depending on the platform :param backup_opt_dict: :return: boolean value, True if snapshot has been taken, false otherwise """ if winutils.is_windows(): if backup_opt_dict.snapshot: # Create a shadow copy. backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ vss.vss_create_shadow_copy(backup_opt_dict.windows_volume) backup_opt_dict.path_to_backup = winutils.use_shadow( backup_opt_dict.path_to_backup, backup_opt_dict.windows_volume) return True return False else: return lvm.lvm_snap(backup_opt_dict)
def test_no_lvm_configured_returns_false(self, mock_validate_lvm_params): backup_opt = Mock() backup_opt.lvm_auto_snap = '' mock_validate_lvm_params.return_value = False backup_opt.snapshot = False self.assertFalse(lvm.lvm_snap(backup_opt))
def test_lvm_snap(self, monkeypatch): backup_opt = BackupOpt1() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False pytest.raises(Exception, lvm_snap, backup_opt) backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess2() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess3() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess1() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess4() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() backup_opt.lvm_dirmount = None fakeos = Os() fakesubprocess = FakeSubProcess4() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True backup_opt = BackupOpt1() backup_opt.lvm_snapperm = False pytest.raises(Exception, lvm_snap, backup_opt)
def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict): ''' Execute the necessary tasks for file system backup mode ''' logging.info('[*] File System backup is being executed...') lvm_snap(backup_opt_dict) # Extract some values from arguments that will be used later on # Initialize swift client object, generate container segments name # and extract backup name sw_connector = backup_opt_dict.sw_connector # Execute a tar gzip of the specified directory and return # small chunks (default 128MB), timestamp, backup, filename, # file chunk index and the tar meta-data file # Generate a string hostname, backup name, timestamp and backup level file_name = add_host_name_ts_level(backup_opt_dict, time_stamp) meta_data_backup_file = u'tar_metadata_{0}'.format(file_name) (backup_opt_dict, tar_command, manifest_meta_dict) = gen_tar_command( opt_dict=backup_opt_dict, time_stamp=time_stamp, remote_manifest_meta=manifest_meta_dict) # Initialize a Queue for a maximum of 2 items tar_backup_queue = Queue(maxsize=2) tar_backup_stream = Process(target=tar_backup, args=( backup_opt_dict, tar_command, tar_backup_queue, )) tar_backup_stream.daemon = True tar_backup_stream.start() add_object_stream = Process(target=add_object, args=(backup_opt_dict, tar_backup_queue, file_name, time_stamp)) add_object_stream.daemon = True add_object_stream.start() tar_backup_stream.join() tar_backup_queue.put(({False: False})) tar_backup_queue.close() add_object_stream.join() (backup_opt_dict, manifest_meta_dict, tar_meta_to_upload, tar_meta_prev) = gen_manifest_meta(backup_opt_dict, manifest_meta_dict, meta_data_backup_file) manifest_file = u'' meta_data_abs_path = '{0}/{1}'.format(backup_opt_dict.workdir, tar_meta_prev) # Upload swift manifest for segments if backup_opt_dict.upload: if not backup_opt_dict.no_incremental: # Upload tar incremental meta data file and remove it logging.info('[*] Uploading tar meta data file: {0}'.format( tar_meta_to_upload)) with open(meta_data_abs_path, 'r') as meta_fd: sw_connector.put_object(backup_opt_dict.container, tar_meta_to_upload, meta_fd) # Removing tar meta data file, so we have only one authoritative # version on swift logging.info('[*] Removing tar meta data file: {0}'.format( meta_data_abs_path)) os.remove(meta_data_abs_path) # Upload manifest to swift manifest_upload(manifest_file, backup_opt_dict, file_name, manifest_meta_dict) # Unmount and remove lvm snapshot volume lvm_snap_remove(backup_opt_dict)
def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict): """ Execute the necessary tasks for file system backup mode """ logging.info('[*] File System backup is being executed...') try: if is_windows(): # Create a shadow copy. # Create a shadow copy. backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ vss_create_shadow_copy(backup_opt_dict.volume) else: # If lvm_auto_snap is true, the volume group and volume name will # be extracted automatically if backup_opt_dict.lvm_auto_snap: backup_opt_dict = get_lvm_info(backup_opt_dict) # Generate the lvm_snap if lvm arguments are available lvm_snap(backup_opt_dict) # Generate a string hostname, backup name, timestamp and backup level file_name = add_host_name_ts_level(backup_opt_dict, time_stamp) meta_data_backup_file = u'tar_metadata_{0}'.format(file_name) backup_opt_dict.meta_data_file = meta_data_backup_file # Initialize a Queue for a maximum of 2 items tar_backup_queue = multiprocessing.Queue(maxsize=2) if is_windows(): backup_opt_dict.absolute_path = backup_opt_dict.src_file backup_opt_dict.src_file = use_shadow(backup_opt_dict.src_file, backup_opt_dict.volume) # Execute a tar gzip of the specified directory and return # small chunks (default 128MB), timestamp, backup, filename, # file chunk index and the tar meta-data file (backup_opt_dict, tar_command, manifest_meta_dict) = \ gen_tar_command(opt_dict=backup_opt_dict, time_stamp=time_stamp, remote_manifest_meta=manifest_meta_dict) tar_backup_stream = multiprocessing.Process( target=tar_backup, args=( backup_opt_dict, tar_command, tar_backup_queue,)) tar_backup_stream.daemon = True tar_backup_stream.start() add_object_stream = multiprocessing.Process( target=add_object, args=( backup_opt_dict, tar_backup_queue, file_name, time_stamp)) add_object_stream.daemon = True add_object_stream.start() tar_backup_stream.join() tar_backup_queue.put(({False: False})) tar_backup_queue.close() add_object_stream.join() if add_object_stream.exitcode: raise Exception('failed to upload object to swift server') (backup_opt_dict, manifest_meta_dict, tar_meta_to_upload, tar_meta_prev) = gen_manifest_meta( backup_opt_dict, manifest_meta_dict, meta_data_backup_file) manifest_file = u'' meta_data_abs_path = os.path.join(backup_opt_dict.workdir, tar_meta_prev) # Upload swift manifest for segments if backup_opt_dict.upload: # Request a new auth client in case the current token # is expired before uploading tar meta data or the swift manifest backup_opt_dict = get_client(backup_opt_dict) if not backup_opt_dict.no_incremental: # Upload tar incremental meta data file and remove it logging.info('[*] Uploading tar meta data file: {0}'.format( tar_meta_to_upload)) with open(meta_data_abs_path, 'r') as meta_fd: backup_opt_dict.sw_connector.put_object( backup_opt_dict.container, tar_meta_to_upload, meta_fd) # Removing tar meta data file, so we have only one # authoritative version on swift logging.info('[*] Removing tar meta data file: {0}'.format( meta_data_abs_path)) os.remove(meta_data_abs_path) # Upload manifest to swift manifest_upload( manifest_file, backup_opt_dict, file_name, manifest_meta_dict) finally: if is_windows(): # Delete the shadow copy after the backup vss_delete_shadow_copy(backup_opt_dict.shadow, backup_opt_dict.volume) else: # Unmount and remove lvm snapshot volume lvm_snap_remove(backup_opt_dict)
def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict): """ Execute the necessary tasks for file system backup mode """ logging.info('[*] File System backup is being executed...') try: if is_windows(): # Create a shadow copy. # Create a shadow copy. backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ vss_create_shadow_copy(backup_opt_dict.volume) else: # If lvm_auto_snap is true, the volume group and volume name will # be extracted automatically if backup_opt_dict.lvm_auto_snap: backup_opt_dict = get_lvm_info(backup_opt_dict) # Generate the lvm_snap if lvm arguments are available lvm_snap(backup_opt_dict) # Generate a string hostname, backup name, timestamp and backup level file_name = add_host_name_ts_level(backup_opt_dict, time_stamp) meta_data_backup_file = u'tar_metadata_{0}'.format(file_name) backup_opt_dict.meta_data_file = meta_data_backup_file # Initialize a Queue for a maximum of 2 items tar_backup_queue = multiprocessing.Queue(maxsize=2) if is_windows(): backup_opt_dict.absolute_path = backup_opt_dict.src_file backup_opt_dict.src_file = use_shadow(backup_opt_dict.src_file, backup_opt_dict.volume) # Execute a tar gzip of the specified directory and return # small chunks (default 128MB), timestamp, backup, filename, # file chunk index and the tar meta-data file (backup_opt_dict, tar_command, manifest_meta_dict) = \ gen_tar_command(opt_dict=backup_opt_dict, time_stamp=time_stamp, remote_manifest_meta=manifest_meta_dict) tar_backup_stream = multiprocessing.Process(target=tar_backup, args=( backup_opt_dict, tar_command, tar_backup_queue, )) tar_backup_stream.daemon = True tar_backup_stream.start() add_object_stream = multiprocessing.Process( target=add_object, args=(backup_opt_dict, tar_backup_queue, file_name, time_stamp)) add_object_stream.daemon = True add_object_stream.start() tar_backup_stream.join() tar_backup_queue.put(({False: False})) tar_backup_queue.close() add_object_stream.join() if add_object_stream.exitcode: raise Exception('failed to upload object to swift server') (backup_opt_dict, manifest_meta_dict, tar_meta_to_upload, tar_meta_prev) = gen_manifest_meta(backup_opt_dict, manifest_meta_dict, meta_data_backup_file) manifest_file = u'' meta_data_abs_path = os.path.join(backup_opt_dict.workdir, tar_meta_prev) # Upload swift manifest for segments if backup_opt_dict.upload: # Request a new auth client in case the current token # is expired before uploading tar meta data or the swift manifest backup_opt_dict = get_client(backup_opt_dict) if not backup_opt_dict.no_incremental: # Upload tar incremental meta data file and remove it logging.info('[*] Uploading tar meta data file: {0}'.format( tar_meta_to_upload)) with open(meta_data_abs_path, 'r') as meta_fd: backup_opt_dict.sw_connector.put_object( backup_opt_dict.container, tar_meta_to_upload, meta_fd) # Removing tar meta data file, so we have only one # authoritative version on swift logging.info('[*] Removing tar meta data file: {0}'.format( meta_data_abs_path)) os.remove(meta_data_abs_path) # Upload manifest to swift manifest_upload(manifest_file, backup_opt_dict, file_name, manifest_meta_dict) finally: if is_windows(): # Delete the shadow copy after the backup vss_delete_shadow_copy(backup_opt_dict.shadow, backup_opt_dict.volume) else: # Unmount and remove lvm snapshot volume lvm_snap_remove(backup_opt_dict)
def test_lvm_snap(self, monkeypatch): backup_opt = BackupOpt1() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False pytest.raises(Exception, lvm_snap, backup_opt) backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess2() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess3() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess1() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) pytest.raises(Exception, lvm_snap, backup_opt) get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() fakeos = Os() fakesubprocess = FakeSubProcess4() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True get_vol_fs_type = Fake_get_vol_fs_type() backup_opt = BackupOpt1() backup_opt.lvm_dirmount = None fakeos = Os() fakesubprocess = FakeSubProcess4() backup_opt.lvm_snapsize = False backup_opt.lvm_snapname = False monkeypatch.setattr(os, 'path', fakeos) monkeypatch.setattr(subprocess, 'Popen', fakesubprocess.Popen) monkeypatch.setattr(utils, 'get_vol_fs_type', get_vol_fs_type.get_vol_fs_type1) fakere = FakeRe() monkeypatch.setattr(re, 'search', fakere.search) assert lvm_snap(backup_opt) is True