class EnterpriseBackupRestoreBase(BaseTestCase): def setUp(self): super(EnterpriseBackupRestoreBase, self).setUp() self.backupset = Backupset() shell = RemoteMachineShellConnection(self.servers[0]) info = shell.extract_remote_info().type.lower() if info == 'linux': self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param("dir", "/tmp/entbackup") self.backup_validation_files_location = "/tmp/backuprestore" elif info == 'windows': self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param("dir", testconstants.WIN_TMP_PATH + "entbackup") self.backup_validation_files_location = testconstants.WIN_TMP_PATH + "backuprestore" elif info == 'mac': self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param("dir", "/tmp/entbackup") self.backup_validation_files_location = "/tmp/backuprestore" else: raise Exception("OS not supported.") self.backupset.backup_host = self.input.clusters[1][0] self.backupset.name = self.input.param("name", "backup") self.non_master_host = self.input.param("non-master", False) if self.non_master_host: self.backupset.cluster_host = self.servers[1] self.backupset.cluster_host_username = self.servers[1].rest_username self.backupset.cluster_host_password = self.servers[1].rest_password else: self.backupset.cluster_host = self.servers[0] self.backupset.cluster_host_username = self.servers[0].rest_username self.backupset.cluster_host_password = self.servers[0].rest_password self.same_cluster = self.input.param("same-cluster", False) self.reset_restore_cluster = self.input.param("reset-restore-cluster", True) self.no_progress_bar = self.input.param("no-progress-bar", True) if self.same_cluster: self.backupset.restore_cluster_host = self.servers[0] self.backupset.restore_cluster_host_username = self.servers[0].rest_username self.backupset.restore_cluster_host_password = self.servers[0].rest_password else: self.backupset.restore_cluster_host = self.input.clusters[0][0] self.backupset.restore_cluster_host_username = self.input.clusters[0][0].rest_username self.backupset.restore_cluster_host_password = self.input.clusters[0][0].rest_password self.backupset.exclude_buckets = self.input.param("exclude-buckets", "") self.backupset.include_buckets = self.input.param("include-buckets", "") self.backupset.disable_bucket_config = self.input.param("disable-bucket-config", False) self.backupset.disable_views = self.input.param("disable-views", False) self.backupset.disable_gsi_indexes = self.input.param("disable-gsi-indexes", False) self.backupset.disable_ft_indexes = self.input.param("disable-ft-indexes", False) self.backupset.disable_data = self.input.param("disable-data", False) self.backupset.force_updates = self.input.param("force-updates", False) self.backupset.resume = self.input.param("resume", False) self.backupset.purge = self.input.param("purge", False) self.backupset.threads = self.input.param("threads", self.number_of_processors()) self.backupset.start = self.input.param("start", 1) self.backupset.end = self.input.param("stop", 1) self.backupset.number_of_backups = self.input.param("number_of_backups", 1) self.backupset.filter_keys = self.input.param("filter-keys", "") self.backupset.filter_values = self.input.param("filter-values", "") self.backupset.backup_list_name = self.input.param("list-names", None) self.backupset.backup_incr_backup = self.input.param("incr-backup", None) self.backupset.bucket_backup = self.input.param("bucket-backup", None) self.backupset.backup_to_compact = self.input.param("backup-to-compact", 0) self.backups = [] self.validation_helper = BackupRestoreValidations(self.backupset, self.cluster_to_backup, self.cluster_to_restore, self.buckets, self.backup_validation_files_location, self.backups, self.num_items) self.number_of_backups_taken = 0 self.vbucket_seqno = [] self.expires = self.input.param("expires", 0) self.auto_failover = self.input.param("enable-autofailover", False) self.auto_failover_timeout = self.input.param("autofailover-timeout", 30) self.graceful = self.input.param("graceful",False) self.recoveryType = self.input.param("recoveryType", "full") if not os.path.exists(self.backup_validation_files_location): os.mkdir(self.backup_validation_files_location) def tearDown(self): super(EnterpriseBackupRestoreBase, self).tearDown() if not self.input.param("skip_cleanup", False): remote_client = RemoteMachineShellConnection(self.input.clusters[1][0]) info = remote_client.extract_remote_info().type.lower() if info == 'linux' or info == 'mac': backup_directory = "/tmp/entbackup" validation_files_location = "/tmp/backuprestore" elif info == 'windows': backup_directory = testconstants.WIN_TMP_PATH + "entbackup" validation_files_location = testconstants.WIN_TMP_PATH + "backuprestore" else: raise Exception("OS not supported.") command = "rm -rf {0}".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if info == 'linux': command = "rm -rf /cbqe3043/entbackup".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if self.input.clusters: for key in self.input.clusters.keys(): servers = self.input.clusters[key] self.backup_reset_clusters(servers) if os.path.exists(validation_files_location): shutil.rmtree(validation_files_location) @property def cluster_to_backup(self): return self.get_nodes_in_cluster(self.backupset.cluster_host) @property def cluster_to_restore(self): return self.get_nodes_in_cluster(self.backupset.restore_cluster_host) def number_of_processors(self): remote_client = RemoteMachineShellConnection(self.input.clusters[1][0]) command = "nproc" output, error = remote_client.execute_command(command) if output: return output[0] else: return error[0] def backup_reset_clusters(self, servers): BucketOperationHelper.delete_all_buckets_or_assert(servers, self) ClusterOperationHelper.cleanup_cluster(servers, master=servers[0]) ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self) def store_vbucket_seqno(self): vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets) self.vbucket_seqno.append(vseqno) def backup_create(self): args = "create --dir {0} --name {1}".format(self.backupset.directory, self.backupset.name) if self.backupset.exclude_buckets: args += " --exclude-buckets \"{0}\"".format(",".join(self.backupset.exclude_buckets)) if self.backupset.include_buckets: args += " --include-buckets=\"{0}\"".format(",".join(self.backupset.include_buckets)) if self.backupset.disable_bucket_config: args += " --disable-bucket-config" if self.backupset.disable_views: args += " --disable-views" if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes" if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes" if self.backupset.disable_data: args += " --disable-data" remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) return output, error def backup_create_validate(self): output, error = self.backup_create() if error or "Backup `{0}` created successfully".format(self.backupset.name) not in output[0]: self.fail("Creating backupset failed.") status, msg = self.validation_helper.validate_backup_create() if not status: self.fail(msg) self.log.info(msg) def backup_cluster(self): args = "cluster --dir {0} --name {1} --host http://{2}:{3} --username {4} --password {5}". \ format(self.backupset.directory, self.backupset.name, self.backupset.cluster_host.ip, self.backupset.cluster_host.port, self.backupset.cluster_host_username, self.backupset.cluster_host_password) if self.backupset.resume: args += " --resume" if self.backupset.purge: args += " --purge" if self.no_progress_bar: args += " --no-progress-bar" remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Backup successfully completed" not in output[0]: return output, error command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.append(o[0]) self.number_of_backups_taken += 1 self.log.info("Finished taking backup with args: {0}".format(args)) return output, error def backup_cluster_validate(self): output, error = self.backup_cluster() if error or "Backup successfully completed" not in output[0]: self.fail("Taking cluster backup failed.") status, msg = self.validation_helper.validate_backup() if not status: self.fail(msg) self.log.info(msg) self.store_vbucket_seqno() self.validation_helper.store_keys(self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location) self.validation_helper.store_latest(self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location) def backup_restore(self): try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = "restore --dir {0} --name {1} --host http://{2}:{3} --username {4} --password {5} --start {6} " \ "--end {7}".format(self.backupset.directory, self.backupset.name, self.backupset.restore_cluster_host.ip, self.backupset.restore_cluster_host.port, self.backupset.restore_cluster_host_username, self.backupset.restore_cluster_host_password, backup_start, backup_end) if self.backupset.exclude_buckets: args += " --exclude-buckets {0}".format(self.backupset.exclude_buckets) if self.backupset.include_buckets: args += " --include-buckets {0}".format(self.backupset.include_buckets) if self.backupset.disable_bucket_config: args += " --disable-bucket-config {0}".format(self.backupset.disable_bucket_config) if self.backupset.disable_views: args += " --disable-views {0}".format(self.backupset.disable_views) if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes {0}".format(self.backupset.disable_gsi_indexes) if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes {0}".format(self.backupset.disable_ft_indexes) if self.backupset.disable_data: args += " --disable-data {0}".format(self.backupset.disable_data) if self.backupset.filter_keys: args += " --filter_keys {0}".format(self.backupset.filter_keys) if self.backupset.filter_values: args += " --filter_values {0}".format(self.backupset.filter_values) if self.backupset.force_updates: args += " --force-updates" if self.no_progress_bar: args += " --no-progress-bar" remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) return output, error def backup_restore_validate(self, compare_uuid=False, seqno_compare_function="==", replicas=False, mode="memory"): self.backup_restore() remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if not output: self.fail("Restoring backup failed.") self.log.info("Finished restoring backup") current_vseqno = self.get_vbucket_seqnos(self.cluster_to_restore, self.buckets) status, msg = self.validation_helper.validate_restore(self.backupset.end, self.vbucket_seqno, current_vseqno, compare_uuid=compare_uuid, compare=seqno_compare_function, get_replica=replicas, mode=mode) if not status: self.fail(msg) self.log.info(msg) def backup_list(self): args = "list --dir {0}".format(self.backupset.directory) if self.backupset.backup_list_name: args += " --name {0}".format(self.backupset.backup_list_name) if self.backupset.backup_incr_backup: args += " --incr-backup {0}".format(self.backupset.backup_incr_backup) if self.backupset.bucket_backup: args += " --bucket-backup {0}".format(self.backupset.bucket_backup) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error: return False, error, "Getting backup list failed." else: return True, output, "Backup list obtained" def backup_compact(self): args = "compact --dir {0} --name {1} --backup {2}".format(self.backupset.directory, self.backupset.name, self.backups[self.backupset.backup_to_compact]) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if "Compaction succeeded," not in output[0]: return False, output, "Compacting backup failed." else: return True, output, "Compaction of backup success" def backup_remove(self): args = "remove --dir {0} --name {1}".format(self.backupset.directory, self.backupset.name) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) self.verify_cluster_stats() if error: return False, error, "Removing backup failed." else: return True, output, "Removing of backup success" def backup_list_validate(self): status, output, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_backup_list(output) if not status: self.fail(message) self.log.info(message) def backup_compact_validate(self): self.log.info("Listing backup details before compact") status, output_before_compact, message = self.backup_list() if not status: self.fail(message) status, output, message = self.backup_compact() if not status: self.fail(message) self.log.info("Listing backup details after compact") status, output_after_compact, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_compact_lists(output_before_compact, output_after_compact) if not status: self.fail(message) self.log.info(message) def backup_merge(self): self.log.info("backups before merge: " + str(self.backups)) self.log.info("number_of_backups_taken before merge: " + str(self.number_of_backups_taken)) try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = "merge --dir {0} --name {1} --start {2} --end {3}".format(self.backupset.directory, self.backupset.name, backup_start, backup_end) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/backup {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Merge completed successfully" not in output[0]: return False, error, "Merging backup failed" del self.backups[self.backupset.start - 1:self.backupset.end] command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.insert(self.backupset.start - 1, o[0]) self.number_of_backups_taken -= (self.backupset.end - self.backupset.start + 1) self.number_of_backups_taken += 1 self.log.info("backups after merge: " + str(self.backups)) self.log.info("number_of_backups_taken after merge: " + str(self.number_of_backups_taken)) return True, output, "Merging backup succeeded"
class EnterpriseBackupRestoreBase(BaseTestCase): def setUp(self): super(EnterpriseBackupRestoreBase, self).setUp() """ from version 4.6.0 and later, --host flag is deprecated """ self.cluster_flag = "--host" if self.cb_version[:5] in COUCHBASE_FROM_4DOT6: self.cluster_flag = "--cluster" self.backupset = Backupset() self.cmd_ext = "" shell = RemoteMachineShellConnection(self.servers[0]) info = shell.extract_remote_info().type.lower() if info == 'linux': if self.nonroot: self.cli_command_location = "/home/%s%s" % ( self.master.ssh_username, LINUX_COUCHBASE_BIN_PATH) else: self.cli_command_location = LINUX_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param( "dir", "/tmp/entbackup") elif info == 'windows': self.cmd_ext = ".exe" self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW self.backupset.directory = self.input.param( "dir", testconstants.WIN_TMP_PATH_RAW + "entbackup") elif info == 'mac': self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param( "dir", "/tmp/entbackup") else: raise Exception("OS not supported.") self.backup_validation_files_location = "/tmp/backuprestore" self.backupset.backup_host = self.input.clusters[1][0] self.backupset.name = self.input.param("name", "backup") self.non_master_host = self.input.param("non-master", False) self.compact_backup = self.input.param("compact-backup", False) if self.non_master_host: self.backupset.cluster_host = self.servers[1] self.backupset.cluster_host_username = self.servers[ 1].rest_username self.backupset.cluster_host_password = self.servers[ 1].rest_password else: self.backupset.cluster_host = self.servers[0] self.backupset.cluster_host_username = self.servers[ 0].rest_username self.backupset.cluster_host_password = self.servers[ 0].rest_password self.same_cluster = self.input.param("same-cluster", False) self.reset_restore_cluster = self.input.param("reset-restore-cluster", True) self.no_progress_bar = self.input.param("no-progress-bar", True) if self.same_cluster: self.backupset.restore_cluster_host = self.servers[0] self.backupset.restore_cluster_host_username = self.servers[ 0].rest_username self.backupset.restore_cluster_host_password = self.servers[ 0].rest_password else: self.backupset.restore_cluster_host = self.input.clusters[0][0] self.backupset.restore_cluster_host_username = self.input.clusters[ 0][0].rest_username self.backupset.restore_cluster_host_password = self.input.clusters[ 0][0].rest_password self.backupset.exclude_buckets = self.input.param( "exclude-buckets", "") self.backupset.include_buckets = self.input.param( "include-buckets", "") self.backupset.disable_bucket_config = self.input.param( "disable-bucket-config", False) self.backupset.disable_views = self.input.param("disable-views", False) self.backupset.disable_gsi_indexes = self.input.param( "disable-gsi-indexes", False) self.backupset.disable_ft_indexes = self.input.param( "disable-ft-indexes", False) self.backupset.disable_data = self.input.param("disable-data", False) self.backupset.disable_conf_res_restriction = self.input.param( "disable-conf-res-restriction", None) self.backupset.force_updates = self.input.param("force-updates", False) self.backupset.resume = self.input.param("resume", False) self.backupset.purge = self.input.param("purge", False) self.backupset.threads = self.input.param("threads", self.number_of_processors()) self.backupset.start = self.input.param("start", 1) self.backupset.end = self.input.param("stop", 1) self.backupset.number_of_backups = self.input.param( "number_of_backups", 1) self.backupset.filter_keys = self.input.param("filter-keys", "") self.backupset.filter_values = self.input.param("filter-values", "") self.backupset.backup_list_name = self.input.param("list-names", None) self.backupset.backup_incr_backup = self.input.param( "incr-backup", None) self.backupset.bucket_backup = self.input.param("bucket-backup", None) self.backupset.backup_to_compact = self.input.param( "backup-to-compact", 0) self.backups = [] self.validation_helper = BackupRestoreValidations( self.backupset, self.cluster_to_backup, self.cluster_to_restore, self.buckets, self.backup_validation_files_location, self.backups, self.num_items) self.number_of_backups_taken = 0 self.vbucket_seqno = [] self.expires = self.input.param("expires", 0) self.auto_failover = self.input.param("enable-autofailover", False) self.auto_failover_timeout = self.input.param("autofailover-timeout", 30) self.graceful = self.input.param("graceful", False) self.recoveryType = self.input.param("recoveryType", "full") self.skip_buckets = self.input.param("skip_buckets", False) self.lww_new = self.input.param("lww_new", False) self.skip_consistency = self.input.param("skip_consistency", False) self.per_node = self.input.param("per_node", True) if not os.path.exists(self.backup_validation_files_location): os.mkdir(self.backup_validation_files_location) def tearDown(self): super(EnterpriseBackupRestoreBase, self).tearDown() if not self.input.param("skip_cleanup", False): remote_client = RemoteMachineShellConnection( self.input.clusters[1][0]) info = remote_client.extract_remote_info().type.lower() if info == 'linux' or info == 'mac': backup_directory = "/tmp/entbackup" elif info == 'windows': backup_directory = testconstants.WIN_TMP_PATH_RAW + "entbackup" else: raise Exception("OS not supported.") validation_files_location = "/tmp/backuprestore" if info == 'linux': command = "rm -rf {0}".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) elif info == 'windows': remote_client.remove_directory_recursive(backup_directory) if info == 'linux': command = "rm -rf /cbqe3043/entbackup".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if self.input.clusters: for key in self.input.clusters.keys(): servers = self.input.clusters[key] self.backup_reset_clusters(servers) if os.path.exists(validation_files_location): shutil.rmtree(validation_files_location) @property def cluster_to_backup(self): return self.get_nodes_in_cluster(self.backupset.cluster_host) @property def cluster_to_restore(self): return self.get_nodes_in_cluster(self.backupset.restore_cluster_host) def number_of_processors(self): remote_client = RemoteMachineShellConnection(self.input.clusters[1][0]) info = remote_client.extract_remote_info().type.lower() if info == 'linux' or info == 'mac': command = "nproc" output, error = remote_client.execute_command(command) if output: return output[0] else: return error[0] elif info == 'windows': sysinfo = remote_client.get_windows_system_info() numprocs = sysinfo['Processor(s)'].split(' ') return numprocs[0] def backup_reset_clusters(self, servers): BucketOperationHelper.delete_all_buckets_or_assert(servers, self) ClusterOperationHelper.cleanup_cluster(servers, master=servers[0]) ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self) def store_vbucket_seqno(self): vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets, self.skip_consistency, self.per_node) self.vbucket_seqno.append(vseqno) def backup_create(self, del_old_backup=True): args = "config --archive {0} --repo {1}".format( self.backupset.directory, self.backupset.name) if self.backupset.exclude_buckets: args += " --exclude-buckets \"{0}\"".format(",".join( self.backupset.exclude_buckets)) if self.backupset.include_buckets: args += " --include-buckets \"{0}\"".format(",".join( self.backupset.include_buckets)) if self.backupset.disable_bucket_config: args += " --disable-bucket-config" if self.backupset.disable_views: args += " --disable-views" if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes" if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes" if self.backupset.disable_data: args += " --disable-data" remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) if del_old_backup: self.log.info("Remove any old dir before create new one") remote_client.execute_command("rm -rf %s" % self.backupset.directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) return output, error def backup_create_validate(self): output, error = self.backup_create() if error or "Backup repository `{0}` created successfully".format( self.backupset.name) not in output[0]: self.fail("Creating backupset failed.") status, msg = self.validation_helper.validate_backup_create() if not status: self.fail(msg) self.log.info(msg) def backup_cluster(self): args = "backup --archive {0} --repo {1} {6} http://{2}:{3} --username {4} --password {5}". \ format(self.backupset.directory, self.backupset.name, self.backupset.cluster_host.ip, self.backupset.cluster_host.port, self.backupset.cluster_host_username, self.backupset.cluster_host_password, self.cluster_flag) if self.backupset.resume: args += " --resume" if self.backupset.purge: args += " --purge" if self.no_progress_bar: args += " --no-progress-bar" remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Backup successfully completed" not in output[0]: return output, error command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.append(o[0]) self.number_of_backups_taken += 1 self.log.info("Finished taking backup with args: {0}".format(args)) return output, error def backup_cluster_validate(self): output, error = self.backup_cluster() if error or "Backup successfully completed" not in output[-1]: self.fail("Taking cluster backup failed.") status, msg = self.validation_helper.validate_backup() if not status: self.fail(msg) self.log.info(msg) self.store_vbucket_seqno() self.validation_helper.store_keys( self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location) self.validation_helper.store_latest( self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location) def backup_restore(self): try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = "restore --archive {0} --repo {1} {2} http://{3}:{4} --username {5} "\ "--password {6} --start {7} --end {8}" \ .format(self.backupset.directory, self.backupset.name, self.cluster_flag, self.backupset.restore_cluster_host.ip, self.backupset.restore_cluster_host.port, self.backupset.restore_cluster_host_username, self.backupset.restore_cluster_host_password, backup_start, backup_end) if self.backupset.exclude_buckets: args += " --exclude-buckets {0}".format( self.backupset.exclude_buckets) if self.backupset.include_buckets: args += " --include-buckets {0}".format( self.backupset.include_buckets) if self.backupset.disable_bucket_config: args += " --disable-bucket-config {0}".format( self.backupset.disable_bucket_config) if self.backupset.disable_views: args += " --disable-views {0}".format(self.backupset.disable_views) if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes {0}".format( self.backupset.disable_gsi_indexes) if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes {0}".format( self.backupset.disable_ft_indexes) if self.backupset.disable_data: args += " --disable-data {0}".format(self.backupset.disable_data) if self.backupset.disable_conf_res_restriction is not None: args += " --disable-conf-res-restriction {0}".format( self.backupset.disable_conf_res_restriction) if self.backupset.filter_keys: args += " --filter_keys {0}".format(self.backupset.filter_keys) if self.backupset.filter_values: args += " --filter_values {0}".format(self.backupset.filter_values) if self.backupset.force_updates: args += " --force-updates" if self.no_progress_bar: args += " --no-progress-bar" if not self.skip_buckets: rest_conn = RestConnection(self.backupset.restore_cluster_host) rest_helper = RestHelper(rest_conn) for bucket in self.buckets: if not rest_helper.bucket_exists(bucket.name): self.log.info( "Creating bucket {0} in restore host {1}".format( bucket.name, self.backupset.restore_cluster_host.ip)) rest_conn.create_bucket(bucket=bucket.name, ramQuotaMB=512, authType=bucket.authType if bucket.authType else 'none', proxyPort=bucket.port, saslPassword=bucket.saslPassword, lww=self.lww_new) bucket_ready = rest_helper.vbucket_map_ready(bucket.name) if not bucket_ready: self.fail("Bucket %s not created after 120 seconds." % bucket.name) remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) res = output res.extend(error) error_str = "Error restoring cluster: Transfer failed. Check the logs for more information." if error_str in res: command = "cat " + self.backupset.directory + "/logs/backup.log | grep '" + error_str + "' -A 10 -B 100" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if 'Required Flags:' in res: self.fail("Command line failed. Please check test params.") return output, error def backup_restore_validate(self, compare_uuid=False, seqno_compare_function="==", replicas=False, mode="memory", expected_error=None): output, error = self.backup_restore() if expected_error: output.extend(error) error_found = False if expected_error: for line in output: if line.find(expected_error) != -1: error_found = True break self.assertTrue(error_found, "Expected error not found: %s" % expected_error) return remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if not output: self.fail("Restoring backup failed.") command = "grep 'Transfer failed' " + self.backupset.directory + "/logs/backup.log" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if output: self.fail("Restoring backup failed.") self.log.info("Finished restoring backup") self.log.info("Get current vseqno on node %s " % self.cluster_to_restore[0].ip) current_vseqno = self.get_vbucket_seqnos(self.cluster_to_restore, self.buckets, self.skip_consistency, self.per_node) self.log.info("*** Start to validate the restore ") status, msg = self.validation_helper.validate_restore( self.backupset.end, self.vbucket_seqno, current_vseqno, compare_uuid=compare_uuid, compare=seqno_compare_function, get_replica=replicas, mode=mode) if not status: self.fail(msg) self.log.info(msg) def backup_list(self): args = "list --archive {0}".format(self.backupset.directory) if self.backupset.backup_list_name: args += " --repo {0}".format(self.backupset.backup_list_name) if self.backupset.backup_incr_backup: args += " --backup {0}".format(self.backupset.backup_incr_backup) if self.backupset.bucket_backup: args += " --bucket {0}".format(self.backupset.bucket_backup) remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error: return False, error, "Getting backup list failed." else: return True, output, "Backup list obtained" def backup_compact(self): args = "compact --archive {0} --repo {1} --backup {2}".format( self.backupset.directory, self.backupset.name, self.backups[self.backupset.backup_to_compact]) remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if "Compaction succeeded," not in output[0]: return False, output, "Compacting backup failed." else: return True, output, "Compaction of backup success" def backup_remove(self): args = "remove --archive {0} --repo {1}".format( self.backupset.directory, self.backupset.name) remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) self.verify_cluster_stats() if error: return False, error, "Removing backup failed." else: return True, output, "Removing of backup success" def backup_list_validate(self): status, output, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_backup_list(output) if not status: self.fail(message) self.log.info(message) def backup_compact_validate(self): self.log.info("Listing backup details before compact") status, output_before_compact, message = self.backup_list() if not status: self.fail(message) status, output, message = self.backup_compact() if not status: self.fail(message) self.log.info("Listing backup details after compact") status, output_after_compact, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_compact_lists( output_before_compact, output_after_compact) if not status: self.fail(message) self.log.info(message) def backup_compact_deleted_keys_validation(self, delete_keys): self.log.info("Check deleted keys status in file after compact") conn = RemoteMachineShellConnection(self.backupset.backup_host) output, error = conn.execute_command("ls %s/backup/201*/default*/data "\ % self.backupset.directory) deleted_key_status = {} if "shard_0.fdb" in output: cmd = "%sforestdb_dump%s --plain-meta --no-body "\ "%s/backup/201*/default*/data/shard_0.fdb | grep -A 6 ent-backup "\ % (self.cli_command_location, self.cmd_ext,\ self.backupset.directory) dump_output, error = conn.execute_command(cmd) if dump_output: key_ids = [ x.split(":")[1].strip(' ') for x in dump_output[0::8] ] miss_keys = [x for x in delete_keys if x not in key_ids] if miss_keys: raise Exception("Lost some keys %s ", miss_keys) partition_ids = [ x.split(":")[1].strip(' ') for x in dump_output[1::8] ] status_ids = [ x.split(" ")[-3].strip(' ') for x in dump_output[6::8] ] for idx, key in enumerate(key_ids): deleted_key_status[key] = \ {"KV store name":partition_ids[idx], "Status":status_ids[idx]} if status_ids[idx] != "deleted": raise Exception("key %s status was not deleted. " % key) else: raise Exception( "backup compaction failed to keep delete docs in file") else: raise Exception("file shard_0.fdb did not created ") return deleted_key_status def backup_merge(self): self.log.info("backups before merge: " + str(self.backups)) self.log.info("number_of_backups_taken before merge: " + str(self.number_of_backups_taken)) try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = "merge --archive {0} --repo {1} --start {2} --end {3}".format( self.backupset.directory, self.backupset.name, backup_start, backup_end) remote_client = RemoteMachineShellConnection( self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Merge completed successfully" not in output[0]: return False, error, "Merging backup failed" del self.backups[self.backupset.start - 1:self.backupset.end] command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.insert(self.backupset.start - 1, o[0]) self.number_of_backups_taken -= (self.backupset.end - self.backupset.start + 1) self.number_of_backups_taken += 1 self.log.info("backups after merge: " + str(self.backups)) self.log.info("number_of_backups_taken after merge: " + str(self.number_of_backups_taken)) return True, output, "Merging backup succeeded"
class EnterpriseBackupRestoreBase(BaseTestCase): def setUp(self): super(EnterpriseBackupRestoreBase, self).setUp() """ from version 4.6.0 and later, --host flag is deprecated """ self.cluster_flag = "--host" if self.cb_version[:5] in COUCHBASE_FROM_4DOT6: self.cluster_flag = "--cluster" self.backupset = Backupset() shell = RemoteMachineShellConnection(self.servers[0]) info = shell.extract_remote_info().type.lower() if info == "linux": self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param("dir", "/tmp/entbackup") elif info == "windows": self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW self.backupset.directory = self.input.param("dir", testconstants.WIN_TMP_PATH_RAW + "entbackup") elif info == "mac": self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH self.backupset.directory = self.input.param("dir", "/tmp/entbackup") else: raise Exception("OS not supported.") self.backup_validation_files_location = "/tmp/backuprestore" self.backupset.backup_host = self.input.clusters[1][0] self.backupset.name = self.input.param("name", "backup") self.non_master_host = self.input.param("non-master", False) if self.non_master_host: self.backupset.cluster_host = self.servers[1] self.backupset.cluster_host_username = self.servers[1].rest_username self.backupset.cluster_host_password = self.servers[1].rest_password else: self.backupset.cluster_host = self.servers[0] self.backupset.cluster_host_username = self.servers[0].rest_username self.backupset.cluster_host_password = self.servers[0].rest_password self.same_cluster = self.input.param("same-cluster", False) self.reset_restore_cluster = self.input.param("reset-restore-cluster", True) self.no_progress_bar = self.input.param("no-progress-bar", True) if self.same_cluster: self.backupset.restore_cluster_host = self.servers[0] self.backupset.restore_cluster_host_username = self.servers[0].rest_username self.backupset.restore_cluster_host_password = self.servers[0].rest_password else: self.backupset.restore_cluster_host = self.input.clusters[0][0] self.backupset.restore_cluster_host_username = self.input.clusters[0][0].rest_username self.backupset.restore_cluster_host_password = self.input.clusters[0][0].rest_password self.backupset.exclude_buckets = self.input.param("exclude-buckets", "") self.backupset.include_buckets = self.input.param("include-buckets", "") self.backupset.disable_bucket_config = self.input.param("disable-bucket-config", False) self.backupset.disable_views = self.input.param("disable-views", False) self.backupset.disable_gsi_indexes = self.input.param("disable-gsi-indexes", False) self.backupset.disable_ft_indexes = self.input.param("disable-ft-indexes", False) self.backupset.disable_data = self.input.param("disable-data", False) self.backupset.disable_conf_res_restriction = self.input.param("disable-conf-res-restriction", None) self.backupset.force_updates = self.input.param("force-updates", False) self.backupset.resume = self.input.param("resume", False) self.backupset.purge = self.input.param("purge", False) self.backupset.threads = self.input.param("threads", self.number_of_processors()) self.backupset.start = self.input.param("start", 1) self.backupset.end = self.input.param("stop", 1) self.backupset.number_of_backups = self.input.param("number_of_backups", 1) self.backupset.filter_keys = self.input.param("filter-keys", "") self.backupset.filter_values = self.input.param("filter-values", "") self.backupset.backup_list_name = self.input.param("list-names", None) self.backupset.backup_incr_backup = self.input.param("incr-backup", None) self.backupset.bucket_backup = self.input.param("bucket-backup", None) self.backupset.backup_to_compact = self.input.param("backup-to-compact", 0) self.backups = [] self.validation_helper = BackupRestoreValidations( self.backupset, self.cluster_to_backup, self.cluster_to_restore, self.buckets, self.backup_validation_files_location, self.backups, self.num_items, ) self.number_of_backups_taken = 0 self.vbucket_seqno = [] self.expires = self.input.param("expires", 0) self.auto_failover = self.input.param("enable-autofailover", False) self.auto_failover_timeout = self.input.param("autofailover-timeout", 30) self.graceful = self.input.param("graceful", False) self.recoveryType = self.input.param("recoveryType", "full") self.skip_buckets = self.input.param("skip_buckets", False) self.lww_new = self.input.param("lww_new", False) self.skip_consistency = self.input.param("skip_consistency", False) self.per_node = self.input.param("per_node", True) if not os.path.exists(self.backup_validation_files_location): os.mkdir(self.backup_validation_files_location) def tearDown(self): super(EnterpriseBackupRestoreBase, self).tearDown() if not self.input.param("skip_cleanup", False): remote_client = RemoteMachineShellConnection(self.input.clusters[1][0]) info = remote_client.extract_remote_info().type.lower() if info == "linux" or info == "mac": backup_directory = "/tmp/entbackup" elif info == "windows": backup_directory = testconstants.WIN_TMP_PATH_RAW + "entbackup" else: raise Exception("OS not supported.") validation_files_location = "/tmp/backuprestore" if info == "linux": command = "rm -rf {0}".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) elif info == "windows": remote_client.remove_directory_recursive(backup_directory) if info == "linux": command = "rm -rf /cbqe3043/entbackup".format(backup_directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if self.input.clusters: for key in self.input.clusters.keys(): servers = self.input.clusters[key] self.backup_reset_clusters(servers) if os.path.exists(validation_files_location): shutil.rmtree(validation_files_location) @property def cluster_to_backup(self): return self.get_nodes_in_cluster(self.backupset.cluster_host) @property def cluster_to_restore(self): return self.get_nodes_in_cluster(self.backupset.restore_cluster_host) def number_of_processors(self): remote_client = RemoteMachineShellConnection(self.input.clusters[1][0]) info = remote_client.extract_remote_info().type.lower() if info == "linux" or info == "mac": command = "nproc" output, error = remote_client.execute_command(command) if output: return output[0] else: return error[0] elif info == "windows": sysinfo = remote_client.get_windows_system_info() numprocs = sysinfo["Processor(s)"].split(" ") return numprocs[0] def backup_reset_clusters(self, servers): BucketOperationHelper.delete_all_buckets_or_assert(servers, self) ClusterOperationHelper.cleanup_cluster(servers, master=servers[0]) ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self) def store_vbucket_seqno(self): vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets, self.skip_consistency, self.per_node) self.vbucket_seqno.append(vseqno) def backup_create(self, del_old_backup=True): args = "config --archive {0} --repo {1}".format(self.backupset.directory, self.backupset.name) if self.backupset.exclude_buckets: args += ' --exclude-buckets "{0}"'.format(",".join(self.backupset.exclude_buckets)) if self.backupset.include_buckets: args += ' --include-buckets "{0}"'.format(",".join(self.backupset.include_buckets)) if self.backupset.disable_bucket_config: args += " --disable-bucket-config" if self.backupset.disable_views: args += " --disable-views" if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes" if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes" if self.backupset.disable_data: args += " --disable-data" remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) if del_old_backup: self.log.info("Remove any old dir before create new one") remote_client.execute_command("rm -rf %s" % self.backupset.directory) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) return output, error def backup_create_validate(self): output, error = self.backup_create() if error or "Backup repository `{0}` created successfully".format(self.backupset.name) not in output[0]: self.fail("Creating backupset failed.") status, msg = self.validation_helper.validate_backup_create() if not status: self.fail(msg) self.log.info(msg) def backup_cluster(self): args = "backup --archive {0} --repo {1} {6} http://{2}:{3} --username {4} --password {5}".format( self.backupset.directory, self.backupset.name, self.backupset.cluster_host.ip, self.backupset.cluster_host.port, self.backupset.cluster_host_username, self.backupset.cluster_host_password, self.cluster_flag, ) if self.backupset.resume: args += " --resume" if self.backupset.purge: args += " --purge" if self.no_progress_bar: args += " --no-progress-bar" remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Backup successfully completed" not in output[0]: return output, error command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.append(o[0]) self.number_of_backups_taken += 1 self.log.info("Finished taking backup with args: {0}".format(args)) return output, error def backup_cluster_validate(self): output, error = self.backup_cluster() if error or "Backup successfully completed" not in output[-1]: self.fail("Taking cluster backup failed.") status, msg = self.validation_helper.validate_backup() if not status: self.fail(msg) self.log.info(msg) self.store_vbucket_seqno() self.validation_helper.store_keys( self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location ) self.validation_helper.store_latest( self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location ) def backup_restore(self): try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = ( "restore --archive {0} --repo {1} {2} http://{3}:{4} --username {5} " "--password {6} --start {7} --end {8}".format( self.backupset.directory, self.backupset.name, self.cluster_flag, self.backupset.restore_cluster_host.ip, self.backupset.restore_cluster_host.port, self.backupset.restore_cluster_host_username, self.backupset.restore_cluster_host_password, backup_start, backup_end, ) ) if self.backupset.exclude_buckets: args += " --exclude-buckets {0}".format(self.backupset.exclude_buckets) if self.backupset.include_buckets: args += " --include-buckets {0}".format(self.backupset.include_buckets) if self.backupset.disable_bucket_config: args += " --disable-bucket-config {0}".format(self.backupset.disable_bucket_config) if self.backupset.disable_views: args += " --disable-views {0}".format(self.backupset.disable_views) if self.backupset.disable_gsi_indexes: args += " --disable-gsi-indexes {0}".format(self.backupset.disable_gsi_indexes) if self.backupset.disable_ft_indexes: args += " --disable-ft-indexes {0}".format(self.backupset.disable_ft_indexes) if self.backupset.disable_data: args += " --disable-data {0}".format(self.backupset.disable_data) if self.backupset.disable_conf_res_restriction is not None: args += " --disable-conf-res-restriction {0}".format(self.backupset.disable_conf_res_restriction) if self.backupset.filter_keys: args += " --filter_keys {0}".format(self.backupset.filter_keys) if self.backupset.filter_values: args += " --filter_values {0}".format(self.backupset.filter_values) if self.backupset.force_updates: args += " --force-updates" if self.no_progress_bar: args += " --no-progress-bar" if not self.skip_buckets: rest_conn = RestConnection(self.backupset.restore_cluster_host) rest_helper = RestHelper(rest_conn) for bucket in self.buckets: if not rest_helper.bucket_exists(bucket.name): self.log.info( "Creating bucket {0} in restore host {1}".format( bucket.name, self.backupset.restore_cluster_host.ip ) ) rest_conn.create_bucket( bucket=bucket.name, ramQuotaMB=512, authType=bucket.authType if bucket.authType else "none", proxyPort=bucket.port, saslPassword=bucket.saslPassword, lww=self.lww_new, ) bucket_ready = rest_helper.vbucket_map_ready(bucket.name) if not bucket_ready: self.fail("Bucket %s not created after 120 seconds." % bucket.name) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) res = output res.extend(error) error_str = "Error restoring cluster: Transfer failed. Check the logs for more information." if error_str in res: command = "cat " + self.backupset.directory + "/logs/backup.log | grep '" + error_str + "' -A 10 -B 100" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if "Required Flags:" in res: self.fail("Command line failed. Please check test params.") return output, error def backup_restore_validate( self, compare_uuid=False, seqno_compare_function="==", replicas=False, mode="memory", expected_error=None ): output, error = self.backup_restore() if expected_error: output.extend(error) error_found = False if expected_error: for line in output: if line.find(expected_error) != -1: error_found = True break self.assertTrue(error_found, "Expected error not found: %s" % expected_error) return remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if not output: self.fail("Restoring backup failed.") command = "grep 'Transfer failed' " + self.backupset.directory + "/logs/backup.log" output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if output: self.fail("Restoring backup failed.") self.log.info("Finished restoring backup") current_vseqno = self.get_vbucket_seqnos( self.cluster_to_restore, self.buckets, self.skip_consistency, self.per_node ) self.log.info("*** Start to validate the restore ") status, msg = self.validation_helper.validate_restore( self.backupset.end, self.vbucket_seqno, current_vseqno, compare_uuid=compare_uuid, compare=seqno_compare_function, get_replica=replicas, mode=mode, ) if not status: self.fail(msg) self.log.info(msg) def backup_list(self): args = "list --archive {0}".format(self.backupset.directory) if self.backupset.backup_list_name: args += " --repo {0}".format(self.backupset.backup_list_name) if self.backupset.backup_incr_backup: args += " --backup {0}".format(self.backupset.backup_incr_backup) if self.backupset.bucket_backup: args += " --bucket {0}".format(self.backupset.bucket_backup) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error: return False, error, "Getting backup list failed." else: return True, output, "Backup list obtained" def backup_compact(self): args = "compact --archive {0} --repo {1} --backup {2}".format( self.backupset.directory, self.backupset.name, self.backups[self.backupset.backup_to_compact] ) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if "Compaction succeeded," not in output[0]: return False, output, "Compacting backup failed." else: return True, output, "Compaction of backup success" def backup_remove(self): args = "remove --archive {0} --repo {1}".format(self.backupset.directory, self.backupset.name) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) self.verify_cluster_stats() if error: return False, error, "Removing backup failed." else: return True, output, "Removing of backup success" def backup_list_validate(self): status, output, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_backup_list(output) if not status: self.fail(message) self.log.info(message) def backup_compact_validate(self): self.log.info("Listing backup details before compact") status, output_before_compact, message = self.backup_list() if not status: self.fail(message) status, output, message = self.backup_compact() if not status: self.fail(message) self.log.info("Listing backup details after compact") status, output_after_compact, message = self.backup_list() if not status: self.fail(message) status, message = self.validation_helper.validate_compact_lists(output_before_compact, output_after_compact) if not status: self.fail(message) self.log.info(message) def backup_merge(self): self.log.info("backups before merge: " + str(self.backups)) self.log.info("number_of_backups_taken before merge: " + str(self.number_of_backups_taken)) try: backup_start = self.backups[int(self.backupset.start) - 1] except IndexError: backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start) try: backup_end = self.backups[int(self.backupset.end) - 1] except IndexError: backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end) args = "merge --archive {0} --repo {1} --start {2} --end {3}".format( self.backupset.directory, self.backupset.name, backup_start, backup_end ) remote_client = RemoteMachineShellConnection(self.backupset.backup_host) command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args) output, error = remote_client.execute_command(command) remote_client.log_command_output(output, error) if error or "Merge completed successfully" not in output[0]: return False, error, "Merging backup failed" del self.backups[self.backupset.start - 1 : self.backupset.end] command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name) o, e = remote_client.execute_command(command) if o: self.backups.insert(self.backupset.start - 1, o[0]) self.number_of_backups_taken -= self.backupset.end - self.backupset.start + 1 self.number_of_backups_taken += 1 self.log.info("backups after merge: " + str(self.backups)) self.log.info("number_of_backups_taken after merge: " + str(self.number_of_backups_taken)) return True, output, "Merging backup succeeded"