def setUp(self):
super(EventingTools, self).setUp()
self.rest.set_service_memoryQuota(service='memoryQuota',
memoryQuota=500)
if self.create_functions_buckets:
self.bucket_size = 100
log.info(self.bucket_size)
bucket_params = self._create_bucket_params(
server=self.server,
size=self.bucket_size,
replicas=self.num_replicas)
self.cluster.create_standard_bucket(name=self.src_bucket_name,
port=STANDARD_BUCKET_PORT + 1,
bucket_params=bucket_params)
self.src_bucket = RestConnection(self.master).get_buckets()
self.cluster.create_standard_bucket(name=self.dst_bucket_name,
port=STANDARD_BUCKET_PORT + 1,
bucket_params=bucket_params)
self.cluster.create_standard_bucket(name=self.metadata_bucket_name,
port=STANDARD_BUCKET_PORT + 1,
bucket_params=bucket_params)
self.buckets = RestConnection(self.master).get_buckets()
self.gens_load = self.generate_docs(self.docs_per_day)
self.expiry = 3
handler_code = self.input.param('handler_code', 'bucket_op')
if handler_code == 'bucket_op':
self.handler_code = HANDLER_CODE.DELETE_BUCKET_OP_ON_DELETE
elif handler_code == 'bucket_op_with_timers':
self.handler_code = HANDLER_CODE.BUCKET_OPS_WITH_TIMERS
elif handler_code == 'bucket_op_with_cron_timers':
self.handler_code = HANDLER_CODE.BUCKET_OPS_WITH_CRON_TIMERS
elif handler_code == 'n1ql_op_with_timers':
# index is required for delete operation through n1ql
self.n1ql_node = self.get_nodes_from_services_map(
service_type="n1ql")
self.n1ql_helper = N1QLHelper(shell=self.shell,
max_verify=self.max_verify,
buckets=self.buckets,
item_flag=self.item_flag,
n1ql_port=self.n1ql_port,
full_docs_list=self.full_docs_list,
log=self.log,
input=self.input,
master=self.master,
use_rest=True)
self.n1ql_helper.create_primary_index(using_gsi=True,
server=self.n1ql_node)
self.handler_code = HANDLER_CODE.N1QL_OPS_WITH_TIMERS
else:
self.handler_code = HANDLER_CODE.DELETE_BUCKET_OP_ON_DELETE
self.backupset = Backupset()
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
self.backupset.name = self.input.param("name", "backup")
self.backupset.backup_host = self.servers[0]
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[0].rest_username
self.backupset.cluster_host_password = self.servers[0].rest_password
self.backupset.restore_cluster_host = self.servers[1]
self.backupset.restore_cluster_host_username = self.servers[
1].rest_username
self.backupset.restore_cluster_host_password = self.servers[
1].rest_password
self.num_shards = self.input.param("num_shards", None)
self.debug_logs = self.input.param("debug-logs", False)
cmd = 'curl -g %s:8091/diag/eval -u Administrator:password ' % self.master.ip
cmd += '-d "path_config:component_path(bin)."'
bin_path = subprocess.check_output(cmd, shell=True)
try:
bin_path = bin_path.decode()
except AttributeError:
pass
if "bin" not in bin_path:
self.fail("Check if cb server install on %s" % self.master.ip)
else:
self.cli_command_location = bin_path.replace('"', '') + "/"
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
self.root_path = LINUX_ROOT_PATH
self.wget = "wget"
self.os_name = "linux"
self.tmp_path = "/tmp/"
self.long_help_flag = "--help"
self.short_help_flag = "-h"
if info == 'linux':
if self.nonroot:
base_path = "/home/%s" % self.master.ssh_username
self.database_path = "%s%s" % (base_path, COUCHBASE_DATA_PATH)
self.root_path = "/home/%s/" % self.master.ssh_username
elif info == 'windows':
self.os_name = "windows"
self.cmd_ext = ".exe"
self.wget = "/cygdrive/c/automation/wget.exe"
self.database_path = WIN_COUCHBASE_DATA_PATH_RAW
self.root_path = WIN_ROOT_PATH
self.tmp_path = WIN_TMP_PATH
self.long_help_flag = "help"
self.short_help_flag = "h"
win_format = "C:/Program Files"
cygwin_format = "/cygdrive/c/Program\ Files"
if win_format in self.cli_command_location:
self.cli_command_location = self.cli_command_location.replace(
win_format, cygwin_format)
self.backupset.directory = self.input.param(
"dir", WIN_TMP_PATH_RAW + "entbackup")
elif info == 'mac':
self.backupset.directory = self.input.param(
"dir", "/tmp/entbackup")
else:
raise Exception("OS not supported.")
self.backup_validation_files_location = "/tmp/backuprestore" + self.master.ip
self.backups = []
self.validation_helper = BackupRestoreValidations(
self.backupset, self.cluster_to_backup, self.cluster_to_restore,
self.buckets, self.backup_validation_files_location, self.backups,
self.num_items, self.vbuckets)
self.restore_only = self.input.param("restore-only", False)
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster",
True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
self.multi_threads = self.input.param("multi_threads", False)
self.threads_count = self.input.param("threads_count", 1)
self.bucket_delete = self.input.param("bucket_delete", False)
self.bucket_flush = self.input.param("bucket_flush", False)
include_buckets = self.input.param("include-buckets", "")
include_buckets = include_buckets.split(",") if include_buckets else []
exclude_buckets = self.input.param("exclude-buckets", "")
exclude_buckets = exclude_buckets.split(",") if exclude_buckets else []
self.backupset.exclude_buckets = exclude_buckets
self.backupset.include_buckets = include_buckets
self.backupset.disable_bucket_config = self.input.param(
"disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param(
"disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param(
"disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.disable_conf_res_restriction = self.input.param(
"disable-conf-res-restriction", None)
self.backupset.force_updates = self.input.param("force-updates", True)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads",
self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param(
"number_of_backups", 1)
self.backupset.number_of_backups_after_upgrade = \
self.input.param("number_of_backups_after_upgrade", 0)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.random_keys = self.input.param("random_keys", False)
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.no_ssl_verify = self.input.param("no-ssl-verify", False)
self.backupset.secure_conn = self.input.param("secure-conn", False)
self.backupset.bk_no_cert = self.input.param("bk-no-cert", False)
self.backupset.rt_no_cert = self.input.param("rt-no-cert", False)
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param(
"incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param(
"backup-to-compact", 0)
self.backupset.map_buckets = self.input.param("map-buckets", None)
self.add_node_services = self.input.param("add-node-services", "kv")
self.backupset.backup_compressed = \
self.input.param("backup-conpressed", False)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout",
30)
self.graceful = self.input.param("graceful", False)
self.recoveryType = self.input.param("recoveryType", "full")
self.skip_buckets = self.input.param("skip_buckets", False)
self.lww_new = self.input.param("lww_new", False)
self.skip_consistency = self.input.param("skip_consistency", False)
self.master_services = self.get_services([self.backupset.cluster_host],
self.services_init,
start_node=0)
if not self.master_services:
self.master_services = ["kv"]
self.per_node = self.input.param("per_node", True)
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
self.total_buckets = len(self.buckets)
self.replace_ttl = self.input.param("replace-ttl", None)
self.replace_ttl_with = self.input.param("replace-ttl-with", None)
self.verify_before_expired = self.input.param("verify-before-expired",
False)
self.vbucket_filter = self.input.param("vbucket-filter", None)
self.new_replicas = self.input.param("new-replicas", None)
self.should_fail = self.input.param("should-fail", False)
self.restore_compression_mode = self.input.param(
"restore-compression-mode", None)
self.enable_firewall = False
self.vbuckets_filter_no_data = False
self.test_fts = self.input.param("test_fts", False)
self.restore_should_fail = self.input.param("restore_should_fail",
False)
def setUp(self):
super(EnterpriseBackupRestoreBase, self).setUp()
""" from version 4.6.0 and later, --host flag is deprecated """
self.cluster_flag = "--host"
if self.cb_version[:5] in COUCHBASE_FROM_4DOT6:
self.cluster_flag = "--cluster"
self.backupset = Backupset()
self.cmd_ext = ""
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
if info == 'linux':
if self.nonroot:
self.cli_command_location = "/home/%s%s" % (
self.master.ssh_username, LINUX_COUCHBASE_BIN_PATH)
else:
self.cli_command_location = LINUX_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param(
"dir", "/tmp/entbackup")
elif info == 'windows':
self.cmd_ext = ".exe"
self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW
self.backupset.directory = self.input.param(
"dir", testconstants.WIN_TMP_PATH_RAW + "entbackup")
elif info == 'mac':
self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param(
"dir", "/tmp/entbackup")
else:
raise Exception("OS not supported.")
self.backup_validation_files_location = "/tmp/backuprestore"
self.backupset.backup_host = self.input.clusters[1][0]
self.backupset.name = self.input.param("name", "backup")
self.non_master_host = self.input.param("non-master", False)
self.compact_backup = self.input.param("compact-backup", False)
if self.non_master_host:
self.backupset.cluster_host = self.servers[1]
self.backupset.cluster_host_username = self.servers[
1].rest_username
self.backupset.cluster_host_password = self.servers[
1].rest_password
else:
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[
0].rest_username
self.backupset.cluster_host_password = self.servers[
0].rest_password
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster",
True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
if self.same_cluster:
self.backupset.restore_cluster_host = self.servers[0]
self.backupset.restore_cluster_host_username = self.servers[
0].rest_username
self.backupset.restore_cluster_host_password = self.servers[
0].rest_password
else:
self.backupset.restore_cluster_host = self.input.clusters[0][0]
self.backupset.restore_cluster_host_username = self.input.clusters[
0][0].rest_username
self.backupset.restore_cluster_host_password = self.input.clusters[
0][0].rest_password
self.backupset.exclude_buckets = self.input.param(
"exclude-buckets", "")
self.backupset.include_buckets = self.input.param(
"include-buckets", "")
self.backupset.disable_bucket_config = self.input.param(
"disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param(
"disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param(
"disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.disable_conf_res_restriction = self.input.param(
"disable-conf-res-restriction", None)
self.backupset.force_updates = self.input.param("force-updates", False)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads",
self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param(
"number_of_backups", 1)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param(
"incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param(
"backup-to-compact", 0)
self.backups = []
self.validation_helper = BackupRestoreValidations(
self.backupset, self.cluster_to_backup, self.cluster_to_restore,
self.buckets, self.backup_validation_files_location, self.backups,
self.num_items)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout",
30)
self.graceful = self.input.param("graceful", False)
self.recoveryType = self.input.param("recoveryType", "full")
self.skip_buckets = self.input.param("skip_buckets", False)
self.lww_new = self.input.param("lww_new", False)
self.skip_consistency = self.input.param("skip_consistency", False)
self.per_node = self.input.param("per_node", True)
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
class EnterpriseBackupRestoreBase(BaseTestCase):
def setUp(self):
super(EnterpriseBackupRestoreBase, self).setUp()
""" from version 4.6.0 and later, --host flag is deprecated """
self.cluster_flag = "--host"
if self.cb_version[:5] in COUCHBASE_FROM_4DOT6:
self.cluster_flag = "--cluster"
self.backupset = Backupset()
self.cmd_ext = ""
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
if info == 'linux':
if self.nonroot:
self.cli_command_location = "/home/%s%s" % (
self.master.ssh_username, LINUX_COUCHBASE_BIN_PATH)
else:
self.cli_command_location = LINUX_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param(
"dir", "/tmp/entbackup")
elif info == 'windows':
self.cmd_ext = ".exe"
self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW
self.backupset.directory = self.input.param(
"dir", testconstants.WIN_TMP_PATH_RAW + "entbackup")
elif info == 'mac':
self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param(
"dir", "/tmp/entbackup")
else:
raise Exception("OS not supported.")
self.backup_validation_files_location = "/tmp/backuprestore"
self.backupset.backup_host = self.input.clusters[1][0]
self.backupset.name = self.input.param("name", "backup")
self.non_master_host = self.input.param("non-master", False)
self.compact_backup = self.input.param("compact-backup", False)
if self.non_master_host:
self.backupset.cluster_host = self.servers[1]
self.backupset.cluster_host_username = self.servers[
1].rest_username
self.backupset.cluster_host_password = self.servers[
1].rest_password
else:
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[
0].rest_username
self.backupset.cluster_host_password = self.servers[
0].rest_password
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster",
True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
if self.same_cluster:
self.backupset.restore_cluster_host = self.servers[0]
self.backupset.restore_cluster_host_username = self.servers[
0].rest_username
self.backupset.restore_cluster_host_password = self.servers[
0].rest_password
else:
self.backupset.restore_cluster_host = self.input.clusters[0][0]
self.backupset.restore_cluster_host_username = self.input.clusters[
0][0].rest_username
self.backupset.restore_cluster_host_password = self.input.clusters[
0][0].rest_password
self.backupset.exclude_buckets = self.input.param(
"exclude-buckets", "")
self.backupset.include_buckets = self.input.param(
"include-buckets", "")
self.backupset.disable_bucket_config = self.input.param(
"disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param(
"disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param(
"disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.disable_conf_res_restriction = self.input.param(
"disable-conf-res-restriction", None)
self.backupset.force_updates = self.input.param("force-updates", False)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads",
self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param(
"number_of_backups", 1)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param(
"incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param(
"backup-to-compact", 0)
self.backups = []
self.validation_helper = BackupRestoreValidations(
self.backupset, self.cluster_to_backup, self.cluster_to_restore,
self.buckets, self.backup_validation_files_location, self.backups,
self.num_items)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout",
30)
self.graceful = self.input.param("graceful", False)
self.recoveryType = self.input.param("recoveryType", "full")
self.skip_buckets = self.input.param("skip_buckets", False)
self.lww_new = self.input.param("lww_new", False)
self.skip_consistency = self.input.param("skip_consistency", False)
self.per_node = self.input.param("per_node", True)
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
def tearDown(self):
super(EnterpriseBackupRestoreBase, self).tearDown()
if not self.input.param("skip_cleanup", False):
remote_client = RemoteMachineShellConnection(
self.input.clusters[1][0])
info = remote_client.extract_remote_info().type.lower()
if info == 'linux' or info == 'mac':
backup_directory = "/tmp/entbackup"
elif info == 'windows':
backup_directory = testconstants.WIN_TMP_PATH_RAW + "entbackup"
else:
raise Exception("OS not supported.")
validation_files_location = "/tmp/backuprestore"
if info == 'linux':
command = "rm -rf {0}".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
elif info == 'windows':
remote_client.remove_directory_recursive(backup_directory)
if info == 'linux':
command = "rm -rf /cbqe3043/entbackup".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if self.input.clusters:
for key in self.input.clusters.keys():
servers = self.input.clusters[key]
self.backup_reset_clusters(servers)
if os.path.exists(validation_files_location):
shutil.rmtree(validation_files_location)
@property
def cluster_to_backup(self):
return self.get_nodes_in_cluster(self.backupset.cluster_host)
@property
def cluster_to_restore(self):
return self.get_nodes_in_cluster(self.backupset.restore_cluster_host)
def number_of_processors(self):
remote_client = RemoteMachineShellConnection(self.input.clusters[1][0])
info = remote_client.extract_remote_info().type.lower()
if info == 'linux' or info == 'mac':
command = "nproc"
output, error = remote_client.execute_command(command)
if output:
return output[0]
else:
return error[0]
elif info == 'windows':
sysinfo = remote_client.get_windows_system_info()
numprocs = sysinfo['Processor(s)'].split(' ')
return numprocs[0]
def backup_reset_clusters(self, servers):
BucketOperationHelper.delete_all_buckets_or_assert(servers, self)
ClusterOperationHelper.cleanup_cluster(servers, master=servers[0])
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self)
def store_vbucket_seqno(self):
vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets,
self.skip_consistency, self.per_node)
self.vbucket_seqno.append(vseqno)
def backup_create(self, del_old_backup=True):
args = "config --archive {0} --repo {1}".format(
self.backupset.directory, self.backupset.name)
if self.backupset.exclude_buckets:
args += " --exclude-buckets \"{0}\"".format(",".join(
self.backupset.exclude_buckets))
if self.backupset.include_buckets:
args += " --include-buckets \"{0}\"".format(",".join(
self.backupset.include_buckets))
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config"
if self.backupset.disable_views:
args += " --disable-views"
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes"
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes"
if self.backupset.disable_data:
args += " --disable-data"
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
if del_old_backup:
self.log.info("Remove any old dir before create new one")
remote_client.execute_command("rm -rf %s" %
self.backupset.directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
return output, error
def backup_create_validate(self):
output, error = self.backup_create()
if error or "Backup repository `{0}` created successfully".format(
self.backupset.name) not in output[0]:
self.fail("Creating backupset failed.")
status, msg = self.validation_helper.validate_backup_create()
if not status:
self.fail(msg)
self.log.info(msg)
def backup_cluster(self):
args = "backup --archive {0} --repo {1} {6} http://{2}:{3} --username {4} --password {5}". \
format(self.backupset.directory, self.backupset.name, self.backupset.cluster_host.ip,
self.backupset.cluster_host.port, self.backupset.cluster_host_username,
self.backupset.cluster_host_password, self.cluster_flag)
if self.backupset.resume:
args += " --resume"
if self.backupset.purge:
args += " --purge"
if self.no_progress_bar:
args += " --no-progress-bar"
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Backup successfully completed" not in output[0]:
return output, error
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory,
self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.append(o[0])
self.number_of_backups_taken += 1
self.log.info("Finished taking backup with args: {0}".format(args))
return output, error
def backup_cluster_validate(self):
output, error = self.backup_cluster()
if error or "Backup successfully completed" not in output[-1]:
self.fail("Taking cluster backup failed.")
status, msg = self.validation_helper.validate_backup()
if not status:
self.fail(msg)
self.log.info(msg)
self.store_vbucket_seqno()
self.validation_helper.store_keys(
self.cluster_to_backup, self.buckets, self.number_of_backups_taken,
self.backup_validation_files_location)
self.validation_helper.store_latest(
self.cluster_to_backup, self.buckets, self.number_of_backups_taken,
self.backup_validation_files_location)
def backup_restore(self):
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1],
self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = "restore --archive {0} --repo {1} {2} http://{3}:{4} --username {5} "\
"--password {6} --start {7} --end {8}" \
.format(self.backupset.directory, self.backupset.name,
self.cluster_flag, self.backupset.restore_cluster_host.ip,
self.backupset.restore_cluster_host.port,
self.backupset.restore_cluster_host_username,
self.backupset.restore_cluster_host_password,
backup_start, backup_end)
if self.backupset.exclude_buckets:
args += " --exclude-buckets {0}".format(
self.backupset.exclude_buckets)
if self.backupset.include_buckets:
args += " --include-buckets {0}".format(
self.backupset.include_buckets)
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config {0}".format(
self.backupset.disable_bucket_config)
if self.backupset.disable_views:
args += " --disable-views {0}".format(self.backupset.disable_views)
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes {0}".format(
self.backupset.disable_gsi_indexes)
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes {0}".format(
self.backupset.disable_ft_indexes)
if self.backupset.disable_data:
args += " --disable-data {0}".format(self.backupset.disable_data)
if self.backupset.disable_conf_res_restriction is not None:
args += " --disable-conf-res-restriction {0}".format(
self.backupset.disable_conf_res_restriction)
if self.backupset.filter_keys:
args += " --filter_keys {0}".format(self.backupset.filter_keys)
if self.backupset.filter_values:
args += " --filter_values {0}".format(self.backupset.filter_values)
if self.backupset.force_updates:
args += " --force-updates"
if self.no_progress_bar:
args += " --no-progress-bar"
if not self.skip_buckets:
rest_conn = RestConnection(self.backupset.restore_cluster_host)
rest_helper = RestHelper(rest_conn)
for bucket in self.buckets:
if not rest_helper.bucket_exists(bucket.name):
self.log.info(
"Creating bucket {0} in restore host {1}".format(
bucket.name,
self.backupset.restore_cluster_host.ip))
rest_conn.create_bucket(bucket=bucket.name,
ramQuotaMB=512,
authType=bucket.authType
if bucket.authType else 'none',
proxyPort=bucket.port,
saslPassword=bucket.saslPassword,
lww=self.lww_new)
bucket_ready = rest_helper.vbucket_map_ready(bucket.name)
if not bucket_ready:
self.fail("Bucket %s not created after 120 seconds." %
bucket.name)
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
res = output
res.extend(error)
error_str = "Error restoring cluster: Transfer failed. Check the logs for more information."
if error_str in res:
command = "cat " + self.backupset.directory + "/logs/backup.log | grep '" + error_str + "' -A 10 -B 100"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if 'Required Flags:' in res:
self.fail("Command line failed. Please check test params.")
return output, error
def backup_restore_validate(self,
compare_uuid=False,
seqno_compare_function="==",
replicas=False,
mode="memory",
expected_error=None):
output, error = self.backup_restore()
if expected_error:
output.extend(error)
error_found = False
if expected_error:
for line in output:
if line.find(expected_error) != -1:
error_found = True
break
self.assertTrue(error_found,
"Expected error not found: %s" % expected_error)
return
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if not output:
self.fail("Restoring backup failed.")
command = "grep 'Transfer failed' " + self.backupset.directory + "/logs/backup.log"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if output:
self.fail("Restoring backup failed.")
self.log.info("Finished restoring backup")
self.log.info("Get current vseqno on node %s " %
self.cluster_to_restore[0].ip)
current_vseqno = self.get_vbucket_seqnos(self.cluster_to_restore,
self.buckets,
self.skip_consistency,
self.per_node)
self.log.info("*** Start to validate the restore ")
status, msg = self.validation_helper.validate_restore(
self.backupset.end,
self.vbucket_seqno,
current_vseqno,
compare_uuid=compare_uuid,
compare=seqno_compare_function,
get_replica=replicas,
mode=mode)
if not status:
self.fail(msg)
self.log.info(msg)
def backup_list(self):
args = "list --archive {0}".format(self.backupset.directory)
if self.backupset.backup_list_name:
args += " --repo {0}".format(self.backupset.backup_list_name)
if self.backupset.backup_incr_backup:
args += " --backup {0}".format(self.backupset.backup_incr_backup)
if self.backupset.bucket_backup:
args += " --bucket {0}".format(self.backupset.bucket_backup)
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error:
return False, error, "Getting backup list failed."
else:
return True, output, "Backup list obtained"
def backup_compact(self):
args = "compact --archive {0} --repo {1} --backup {2}".format(
self.backupset.directory, self.backupset.name,
self.backups[self.backupset.backup_to_compact])
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if "Compaction succeeded," not in output[0]:
return False, output, "Compacting backup failed."
else:
return True, output, "Compaction of backup success"
def backup_remove(self):
args = "remove --archive {0} --repo {1}".format(
self.backupset.directory, self.backupset.name)
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
self.verify_cluster_stats()
if error:
return False, error, "Removing backup failed."
else:
return True, output, "Removing of backup success"
def backup_list_validate(self):
status, output, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_backup_list(output)
if not status:
self.fail(message)
self.log.info(message)
def backup_compact_validate(self):
self.log.info("Listing backup details before compact")
status, output_before_compact, message = self.backup_list()
if not status:
self.fail(message)
status, output, message = self.backup_compact()
if not status:
self.fail(message)
self.log.info("Listing backup details after compact")
status, output_after_compact, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_compact_lists(
output_before_compact, output_after_compact)
if not status:
self.fail(message)
self.log.info(message)
def backup_compact_deleted_keys_validation(self, delete_keys):
self.log.info("Check deleted keys status in file after compact")
conn = RemoteMachineShellConnection(self.backupset.backup_host)
output, error = conn.execute_command("ls %s/backup/201*/default*/data "\
% self.backupset.directory)
deleted_key_status = {}
if "shard_0.fdb" in output:
cmd = "%sforestdb_dump%s --plain-meta --no-body "\
"%s/backup/201*/default*/data/shard_0.fdb | grep -A 6 ent-backup "\
% (self.cli_command_location, self.cmd_ext,\
self.backupset.directory)
dump_output, error = conn.execute_command(cmd)
if dump_output:
key_ids = [
x.split(":")[1].strip(' ') for x in dump_output[0::8]
]
miss_keys = [x for x in delete_keys if x not in key_ids]
if miss_keys:
raise Exception("Lost some keys %s ", miss_keys)
partition_ids = [
x.split(":")[1].strip(' ') for x in dump_output[1::8]
]
status_ids = [
x.split(" ")[-3].strip(' ') for x in dump_output[6::8]
]
for idx, key in enumerate(key_ids):
deleted_key_status[key] = \
{"KV store name":partition_ids[idx], "Status":status_ids[idx]}
if status_ids[idx] != "deleted":
raise Exception("key %s status was not deleted. " %
key)
else:
raise Exception(
"backup compaction failed to keep delete docs in file")
else:
raise Exception("file shard_0.fdb did not created ")
return deleted_key_status
def backup_merge(self):
self.log.info("backups before merge: " + str(self.backups))
self.log.info("number_of_backups_taken before merge: " +
str(self.number_of_backups_taken))
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1],
self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = "merge --archive {0} --repo {1} --start {2} --end {3}".format(
self.backupset.directory, self.backupset.name, backup_start,
backup_end)
remote_client = RemoteMachineShellConnection(
self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Merge completed successfully" not in output[0]:
return False, error, "Merging backup failed"
del self.backups[self.backupset.start - 1:self.backupset.end]
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory,
self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.insert(self.backupset.start - 1, o[0])
self.number_of_backups_taken -= (self.backupset.end -
self.backupset.start + 1)
self.number_of_backups_taken += 1
self.log.info("backups after merge: " + str(self.backups))
self.log.info("number_of_backups_taken after merge: " +
str(self.number_of_backups_taken))
return True, output, "Merging backup succeeded"
def setUp(self):
super(EnterpriseBackupRestoreBase, self).setUp()
self.backupset = Backupset()
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
if info == 'linux':
self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
self.backup_validation_files_location = "/tmp/backuprestore"
elif info == 'windows':
self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", testconstants.WIN_TMP_PATH + "entbackup")
self.backup_validation_files_location = testconstants.WIN_TMP_PATH + "backuprestore"
elif info == 'mac':
self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
self.backup_validation_files_location = "/tmp/backuprestore"
else:
raise Exception("OS not supported.")
self.backupset.backup_host = self.input.clusters[1][0]
self.backupset.name = self.input.param("name", "backup")
self.non_master_host = self.input.param("non-master", False)
if self.non_master_host:
self.backupset.cluster_host = self.servers[1]
self.backupset.cluster_host_username = self.servers[1].rest_username
self.backupset.cluster_host_password = self.servers[1].rest_password
else:
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[0].rest_username
self.backupset.cluster_host_password = self.servers[0].rest_password
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster", True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
if self.same_cluster:
self.backupset.restore_cluster_host = self.servers[0]
self.backupset.restore_cluster_host_username = self.servers[0].rest_username
self.backupset.restore_cluster_host_password = self.servers[0].rest_password
else:
self.backupset.restore_cluster_host = self.input.clusters[0][0]
self.backupset.restore_cluster_host_username = self.input.clusters[0][0].rest_username
self.backupset.restore_cluster_host_password = self.input.clusters[0][0].rest_password
self.backupset.exclude_buckets = self.input.param("exclude-buckets", "")
self.backupset.include_buckets = self.input.param("include-buckets", "")
self.backupset.disable_bucket_config = self.input.param("disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param("disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param("disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.force_updates = self.input.param("force-updates", False)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads", self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param("number_of_backups", 1)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param("incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param("backup-to-compact", 0)
self.backups = []
self.validation_helper = BackupRestoreValidations(self.backupset, self.cluster_to_backup, self.cluster_to_restore,
self.buckets, self.backup_validation_files_location, self.backups,
self.num_items)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout", 30)
self.graceful = self.input.param("graceful",False)
self.recoveryType = self.input.param("recoveryType", "full")
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
class EnterpriseBackupRestoreBase(BaseTestCase):
def setUp(self):
super(EnterpriseBackupRestoreBase, self).setUp()
self.backupset = Backupset()
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
if info == 'linux':
self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
self.backup_validation_files_location = "/tmp/backuprestore"
elif info == 'windows':
self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", testconstants.WIN_TMP_PATH + "entbackup")
self.backup_validation_files_location = testconstants.WIN_TMP_PATH + "backuprestore"
elif info == 'mac':
self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
self.backup_validation_files_location = "/tmp/backuprestore"
else:
raise Exception("OS not supported.")
self.backupset.backup_host = self.input.clusters[1][0]
self.backupset.name = self.input.param("name", "backup")
self.non_master_host = self.input.param("non-master", False)
if self.non_master_host:
self.backupset.cluster_host = self.servers[1]
self.backupset.cluster_host_username = self.servers[1].rest_username
self.backupset.cluster_host_password = self.servers[1].rest_password
else:
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[0].rest_username
self.backupset.cluster_host_password = self.servers[0].rest_password
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster", True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
if self.same_cluster:
self.backupset.restore_cluster_host = self.servers[0]
self.backupset.restore_cluster_host_username = self.servers[0].rest_username
self.backupset.restore_cluster_host_password = self.servers[0].rest_password
else:
self.backupset.restore_cluster_host = self.input.clusters[0][0]
self.backupset.restore_cluster_host_username = self.input.clusters[0][0].rest_username
self.backupset.restore_cluster_host_password = self.input.clusters[0][0].rest_password
self.backupset.exclude_buckets = self.input.param("exclude-buckets", "")
self.backupset.include_buckets = self.input.param("include-buckets", "")
self.backupset.disable_bucket_config = self.input.param("disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param("disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param("disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.force_updates = self.input.param("force-updates", False)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads", self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param("number_of_backups", 1)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param("incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param("backup-to-compact", 0)
self.backups = []
self.validation_helper = BackupRestoreValidations(self.backupset, self.cluster_to_backup, self.cluster_to_restore,
self.buckets, self.backup_validation_files_location, self.backups,
self.num_items)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout", 30)
self.graceful = self.input.param("graceful",False)
self.recoveryType = self.input.param("recoveryType", "full")
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
def tearDown(self):
super(EnterpriseBackupRestoreBase, self).tearDown()
if not self.input.param("skip_cleanup", False):
remote_client = RemoteMachineShellConnection(self.input.clusters[1][0])
info = remote_client.extract_remote_info().type.lower()
if info == 'linux' or info == 'mac':
backup_directory = "/tmp/entbackup"
validation_files_location = "/tmp/backuprestore"
elif info == 'windows':
backup_directory = testconstants.WIN_TMP_PATH + "entbackup"
validation_files_location = testconstants.WIN_TMP_PATH + "backuprestore"
else:
raise Exception("OS not supported.")
command = "rm -rf {0}".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if info == 'linux':
command = "rm -rf /cbqe3043/entbackup".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if self.input.clusters:
for key in self.input.clusters.keys():
servers = self.input.clusters[key]
self.backup_reset_clusters(servers)
if os.path.exists(validation_files_location):
shutil.rmtree(validation_files_location)
@property
def cluster_to_backup(self):
return self.get_nodes_in_cluster(self.backupset.cluster_host)
@property
def cluster_to_restore(self):
return self.get_nodes_in_cluster(self.backupset.restore_cluster_host)
def number_of_processors(self):
remote_client = RemoteMachineShellConnection(self.input.clusters[1][0])
command = "nproc"
output, error = remote_client.execute_command(command)
if output:
return output[0]
else:
return error[0]
def backup_reset_clusters(self, servers):
BucketOperationHelper.delete_all_buckets_or_assert(servers, self)
ClusterOperationHelper.cleanup_cluster(servers, master=servers[0])
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self)
def store_vbucket_seqno(self):
vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets)
self.vbucket_seqno.append(vseqno)
def backup_create(self):
args = "create --dir {0} --name {1}".format(self.backupset.directory, self.backupset.name)
if self.backupset.exclude_buckets:
args += " --exclude-buckets \"{0}\"".format(",".join(self.backupset.exclude_buckets))
if self.backupset.include_buckets:
args += " --include-buckets=\"{0}\"".format(",".join(self.backupset.include_buckets))
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config"
if self.backupset.disable_views:
args += " --disable-views"
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes"
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes"
if self.backupset.disable_data:
args += " --disable-data"
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
return output, error
def backup_create_validate(self):
output, error = self.backup_create()
if error or "Backup `{0}` created successfully".format(self.backupset.name) not in output[0]:
self.fail("Creating backupset failed.")
status, msg = self.validation_helper.validate_backup_create()
if not status:
self.fail(msg)
self.log.info(msg)
def backup_cluster(self):
args = "cluster --dir {0} --name {1} --host http://{2}:{3} --username {4} --password {5}". \
format(self.backupset.directory, self.backupset.name, self.backupset.cluster_host.ip,
self.backupset.cluster_host.port, self.backupset.cluster_host_username,
self.backupset.cluster_host_password)
if self.backupset.resume:
args += " --resume"
if self.backupset.purge:
args += " --purge"
if self.no_progress_bar:
args += " --no-progress-bar"
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Backup successfully completed" not in output[0]:
return output, error
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.append(o[0])
self.number_of_backups_taken += 1
self.log.info("Finished taking backup with args: {0}".format(args))
return output, error
def backup_cluster_validate(self):
output, error = self.backup_cluster()
if error or "Backup successfully completed" not in output[0]:
self.fail("Taking cluster backup failed.")
status, msg = self.validation_helper.validate_backup()
if not status:
self.fail(msg)
self.log.info(msg)
self.store_vbucket_seqno()
self.validation_helper.store_keys(self.cluster_to_backup, self.buckets, self.number_of_backups_taken,
self.backup_validation_files_location)
self.validation_helper.store_latest(self.cluster_to_backup, self.buckets, self.number_of_backups_taken,
self.backup_validation_files_location)
def backup_restore(self):
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = "restore --dir {0} --name {1} --host http://{2}:{3} --username {4} --password {5} --start {6} " \
"--end {7}".format(self.backupset.directory, self.backupset.name,
self.backupset.restore_cluster_host.ip,
self.backupset.restore_cluster_host.port,
self.backupset.restore_cluster_host_username,
self.backupset.restore_cluster_host_password, backup_start,
backup_end)
if self.backupset.exclude_buckets:
args += " --exclude-buckets {0}".format(self.backupset.exclude_buckets)
if self.backupset.include_buckets:
args += " --include-buckets {0}".format(self.backupset.include_buckets)
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config {0}".format(self.backupset.disable_bucket_config)
if self.backupset.disable_views:
args += " --disable-views {0}".format(self.backupset.disable_views)
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes {0}".format(self.backupset.disable_gsi_indexes)
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes {0}".format(self.backupset.disable_ft_indexes)
if self.backupset.disable_data:
args += " --disable-data {0}".format(self.backupset.disable_data)
if self.backupset.filter_keys:
args += " --filter_keys {0}".format(self.backupset.filter_keys)
if self.backupset.filter_values:
args += " --filter_values {0}".format(self.backupset.filter_values)
if self.backupset.force_updates:
args += " --force-updates"
if self.no_progress_bar:
args += " --no-progress-bar"
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
return output, error
def backup_restore_validate(self, compare_uuid=False, seqno_compare_function="==", replicas=False, mode="memory"):
self.backup_restore()
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if not output:
self.fail("Restoring backup failed.")
self.log.info("Finished restoring backup")
current_vseqno = self.get_vbucket_seqnos(self.cluster_to_restore, self.buckets)
status, msg = self.validation_helper.validate_restore(self.backupset.end, self.vbucket_seqno, current_vseqno,
compare_uuid=compare_uuid, compare=seqno_compare_function,
get_replica=replicas, mode=mode)
if not status:
self.fail(msg)
self.log.info(msg)
def backup_list(self):
args = "list --dir {0}".format(self.backupset.directory)
if self.backupset.backup_list_name:
args += " --name {0}".format(self.backupset.backup_list_name)
if self.backupset.backup_incr_backup:
args += " --incr-backup {0}".format(self.backupset.backup_incr_backup)
if self.backupset.bucket_backup:
args += " --bucket-backup {0}".format(self.backupset.bucket_backup)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error:
return False, error, "Getting backup list failed."
else:
return True, output, "Backup list obtained"
def backup_compact(self):
args = "compact --dir {0} --name {1} --backup {2}".format(self.backupset.directory, self.backupset.name,
self.backups[self.backupset.backup_to_compact])
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if "Compaction succeeded," not in output[0]:
return False, output, "Compacting backup failed."
else:
return True, output, "Compaction of backup success"
def backup_remove(self):
args = "remove --dir {0} --name {1}".format(self.backupset.directory, self.backupset.name)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
self.verify_cluster_stats()
if error:
return False, error, "Removing backup failed."
else:
return True, output, "Removing of backup success"
def backup_list_validate(self):
status, output, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_backup_list(output)
if not status:
self.fail(message)
self.log.info(message)
def backup_compact_validate(self):
self.log.info("Listing backup details before compact")
status, output_before_compact, message = self.backup_list()
if not status:
self.fail(message)
status, output, message = self.backup_compact()
if not status:
self.fail(message)
self.log.info("Listing backup details after compact")
status, output_after_compact, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_compact_lists(output_before_compact, output_after_compact)
if not status:
self.fail(message)
self.log.info(message)
def backup_merge(self):
self.log.info("backups before merge: " + str(self.backups))
self.log.info("number_of_backups_taken before merge: " + str(self.number_of_backups_taken))
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = "merge --dir {0} --name {1} --start {2} --end {3}".format(self.backupset.directory, self.backupset.name,
backup_start, backup_end)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/backup {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Merge completed successfully" not in output[0]:
return False, error, "Merging backup failed"
del self.backups[self.backupset.start - 1:self.backupset.end]
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.insert(self.backupset.start - 1, o[0])
self.number_of_backups_taken -= (self.backupset.end - self.backupset.start + 1)
self.number_of_backups_taken += 1
self.log.info("backups after merge: " + str(self.backups))
self.log.info("number_of_backups_taken after merge: " + str(self.number_of_backups_taken))
return True, output, "Merging backup succeeded"
class EnterpriseBackupRestoreBase(BaseTestCase):
def setUp(self):
super(EnterpriseBackupRestoreBase, self).setUp()
""" from version 4.6.0 and later, --host flag is deprecated """
self.cluster_flag = "--host"
if self.cb_version[:5] in COUCHBASE_FROM_4DOT6:
self.cluster_flag = "--cluster"
self.backupset = Backupset()
shell = RemoteMachineShellConnection(self.servers[0])
info = shell.extract_remote_info().type.lower()
if info == "linux":
self.cli_command_location = testconstants.LINUX_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
elif info == "windows":
self.cli_command_location = testconstants.WIN_COUCHBASE_BIN_PATH_RAW
self.backupset.directory = self.input.param("dir", testconstants.WIN_TMP_PATH_RAW + "entbackup")
elif info == "mac":
self.cli_command_location = testconstants.MAC_COUCHBASE_BIN_PATH
self.backupset.directory = self.input.param("dir", "/tmp/entbackup")
else:
raise Exception("OS not supported.")
self.backup_validation_files_location = "/tmp/backuprestore"
self.backupset.backup_host = self.input.clusters[1][0]
self.backupset.name = self.input.param("name", "backup")
self.non_master_host = self.input.param("non-master", False)
if self.non_master_host:
self.backupset.cluster_host = self.servers[1]
self.backupset.cluster_host_username = self.servers[1].rest_username
self.backupset.cluster_host_password = self.servers[1].rest_password
else:
self.backupset.cluster_host = self.servers[0]
self.backupset.cluster_host_username = self.servers[0].rest_username
self.backupset.cluster_host_password = self.servers[0].rest_password
self.same_cluster = self.input.param("same-cluster", False)
self.reset_restore_cluster = self.input.param("reset-restore-cluster", True)
self.no_progress_bar = self.input.param("no-progress-bar", True)
if self.same_cluster:
self.backupset.restore_cluster_host = self.servers[0]
self.backupset.restore_cluster_host_username = self.servers[0].rest_username
self.backupset.restore_cluster_host_password = self.servers[0].rest_password
else:
self.backupset.restore_cluster_host = self.input.clusters[0][0]
self.backupset.restore_cluster_host_username = self.input.clusters[0][0].rest_username
self.backupset.restore_cluster_host_password = self.input.clusters[0][0].rest_password
self.backupset.exclude_buckets = self.input.param("exclude-buckets", "")
self.backupset.include_buckets = self.input.param("include-buckets", "")
self.backupset.disable_bucket_config = self.input.param("disable-bucket-config", False)
self.backupset.disable_views = self.input.param("disable-views", False)
self.backupset.disable_gsi_indexes = self.input.param("disable-gsi-indexes", False)
self.backupset.disable_ft_indexes = self.input.param("disable-ft-indexes", False)
self.backupset.disable_data = self.input.param("disable-data", False)
self.backupset.disable_conf_res_restriction = self.input.param("disable-conf-res-restriction", None)
self.backupset.force_updates = self.input.param("force-updates", False)
self.backupset.resume = self.input.param("resume", False)
self.backupset.purge = self.input.param("purge", False)
self.backupset.threads = self.input.param("threads", self.number_of_processors())
self.backupset.start = self.input.param("start", 1)
self.backupset.end = self.input.param("stop", 1)
self.backupset.number_of_backups = self.input.param("number_of_backups", 1)
self.backupset.filter_keys = self.input.param("filter-keys", "")
self.backupset.filter_values = self.input.param("filter-values", "")
self.backupset.backup_list_name = self.input.param("list-names", None)
self.backupset.backup_incr_backup = self.input.param("incr-backup", None)
self.backupset.bucket_backup = self.input.param("bucket-backup", None)
self.backupset.backup_to_compact = self.input.param("backup-to-compact", 0)
self.backups = []
self.validation_helper = BackupRestoreValidations(
self.backupset,
self.cluster_to_backup,
self.cluster_to_restore,
self.buckets,
self.backup_validation_files_location,
self.backups,
self.num_items,
)
self.number_of_backups_taken = 0
self.vbucket_seqno = []
self.expires = self.input.param("expires", 0)
self.auto_failover = self.input.param("enable-autofailover", False)
self.auto_failover_timeout = self.input.param("autofailover-timeout", 30)
self.graceful = self.input.param("graceful", False)
self.recoveryType = self.input.param("recoveryType", "full")
self.skip_buckets = self.input.param("skip_buckets", False)
self.lww_new = self.input.param("lww_new", False)
self.skip_consistency = self.input.param("skip_consistency", False)
self.per_node = self.input.param("per_node", True)
if not os.path.exists(self.backup_validation_files_location):
os.mkdir(self.backup_validation_files_location)
def tearDown(self):
super(EnterpriseBackupRestoreBase, self).tearDown()
if not self.input.param("skip_cleanup", False):
remote_client = RemoteMachineShellConnection(self.input.clusters[1][0])
info = remote_client.extract_remote_info().type.lower()
if info == "linux" or info == "mac":
backup_directory = "/tmp/entbackup"
elif info == "windows":
backup_directory = testconstants.WIN_TMP_PATH_RAW + "entbackup"
else:
raise Exception("OS not supported.")
validation_files_location = "/tmp/backuprestore"
if info == "linux":
command = "rm -rf {0}".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
elif info == "windows":
remote_client.remove_directory_recursive(backup_directory)
if info == "linux":
command = "rm -rf /cbqe3043/entbackup".format(backup_directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if self.input.clusters:
for key in self.input.clusters.keys():
servers = self.input.clusters[key]
self.backup_reset_clusters(servers)
if os.path.exists(validation_files_location):
shutil.rmtree(validation_files_location)
@property
def cluster_to_backup(self):
return self.get_nodes_in_cluster(self.backupset.cluster_host)
@property
def cluster_to_restore(self):
return self.get_nodes_in_cluster(self.backupset.restore_cluster_host)
def number_of_processors(self):
remote_client = RemoteMachineShellConnection(self.input.clusters[1][0])
info = remote_client.extract_remote_info().type.lower()
if info == "linux" or info == "mac":
command = "nproc"
output, error = remote_client.execute_command(command)
if output:
return output[0]
else:
return error[0]
elif info == "windows":
sysinfo = remote_client.get_windows_system_info()
numprocs = sysinfo["Processor(s)"].split(" ")
return numprocs[0]
def backup_reset_clusters(self, servers):
BucketOperationHelper.delete_all_buckets_or_assert(servers, self)
ClusterOperationHelper.cleanup_cluster(servers, master=servers[0])
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self)
def store_vbucket_seqno(self):
vseqno = self.get_vbucket_seqnos(self.cluster_to_backup, self.buckets, self.skip_consistency, self.per_node)
self.vbucket_seqno.append(vseqno)
def backup_create(self, del_old_backup=True):
args = "config --archive {0} --repo {1}".format(self.backupset.directory, self.backupset.name)
if self.backupset.exclude_buckets:
args += ' --exclude-buckets "{0}"'.format(",".join(self.backupset.exclude_buckets))
if self.backupset.include_buckets:
args += ' --include-buckets "{0}"'.format(",".join(self.backupset.include_buckets))
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config"
if self.backupset.disable_views:
args += " --disable-views"
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes"
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes"
if self.backupset.disable_data:
args += " --disable-data"
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
if del_old_backup:
self.log.info("Remove any old dir before create new one")
remote_client.execute_command("rm -rf %s" % self.backupset.directory)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
return output, error
def backup_create_validate(self):
output, error = self.backup_create()
if error or "Backup repository `{0}` created successfully".format(self.backupset.name) not in output[0]:
self.fail("Creating backupset failed.")
status, msg = self.validation_helper.validate_backup_create()
if not status:
self.fail(msg)
self.log.info(msg)
def backup_cluster(self):
args = "backup --archive {0} --repo {1} {6} http://{2}:{3} --username {4} --password {5}".format(
self.backupset.directory,
self.backupset.name,
self.backupset.cluster_host.ip,
self.backupset.cluster_host.port,
self.backupset.cluster_host_username,
self.backupset.cluster_host_password,
self.cluster_flag,
)
if self.backupset.resume:
args += " --resume"
if self.backupset.purge:
args += " --purge"
if self.no_progress_bar:
args += " --no-progress-bar"
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Backup successfully completed" not in output[0]:
return output, error
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.append(o[0])
self.number_of_backups_taken += 1
self.log.info("Finished taking backup with args: {0}".format(args))
return output, error
def backup_cluster_validate(self):
output, error = self.backup_cluster()
if error or "Backup successfully completed" not in output[-1]:
self.fail("Taking cluster backup failed.")
status, msg = self.validation_helper.validate_backup()
if not status:
self.fail(msg)
self.log.info(msg)
self.store_vbucket_seqno()
self.validation_helper.store_keys(
self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location
)
self.validation_helper.store_latest(
self.cluster_to_backup, self.buckets, self.number_of_backups_taken, self.backup_validation_files_location
)
def backup_restore(self):
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = (
"restore --archive {0} --repo {1} {2} http://{3}:{4} --username {5} "
"--password {6} --start {7} --end {8}".format(
self.backupset.directory,
self.backupset.name,
self.cluster_flag,
self.backupset.restore_cluster_host.ip,
self.backupset.restore_cluster_host.port,
self.backupset.restore_cluster_host_username,
self.backupset.restore_cluster_host_password,
backup_start,
backup_end,
)
)
if self.backupset.exclude_buckets:
args += " --exclude-buckets {0}".format(self.backupset.exclude_buckets)
if self.backupset.include_buckets:
args += " --include-buckets {0}".format(self.backupset.include_buckets)
if self.backupset.disable_bucket_config:
args += " --disable-bucket-config {0}".format(self.backupset.disable_bucket_config)
if self.backupset.disable_views:
args += " --disable-views {0}".format(self.backupset.disable_views)
if self.backupset.disable_gsi_indexes:
args += " --disable-gsi-indexes {0}".format(self.backupset.disable_gsi_indexes)
if self.backupset.disable_ft_indexes:
args += " --disable-ft-indexes {0}".format(self.backupset.disable_ft_indexes)
if self.backupset.disable_data:
args += " --disable-data {0}".format(self.backupset.disable_data)
if self.backupset.disable_conf_res_restriction is not None:
args += " --disable-conf-res-restriction {0}".format(self.backupset.disable_conf_res_restriction)
if self.backupset.filter_keys:
args += " --filter_keys {0}".format(self.backupset.filter_keys)
if self.backupset.filter_values:
args += " --filter_values {0}".format(self.backupset.filter_values)
if self.backupset.force_updates:
args += " --force-updates"
if self.no_progress_bar:
args += " --no-progress-bar"
if not self.skip_buckets:
rest_conn = RestConnection(self.backupset.restore_cluster_host)
rest_helper = RestHelper(rest_conn)
for bucket in self.buckets:
if not rest_helper.bucket_exists(bucket.name):
self.log.info(
"Creating bucket {0} in restore host {1}".format(
bucket.name, self.backupset.restore_cluster_host.ip
)
)
rest_conn.create_bucket(
bucket=bucket.name,
ramQuotaMB=512,
authType=bucket.authType if bucket.authType else "none",
proxyPort=bucket.port,
saslPassword=bucket.saslPassword,
lww=self.lww_new,
)
bucket_ready = rest_helper.vbucket_map_ready(bucket.name)
if not bucket_ready:
self.fail("Bucket %s not created after 120 seconds." % bucket.name)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
res = output
res.extend(error)
error_str = "Error restoring cluster: Transfer failed. Check the logs for more information."
if error_str in res:
command = "cat " + self.backupset.directory + "/logs/backup.log | grep '" + error_str + "' -A 10 -B 100"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if "Required Flags:" in res:
self.fail("Command line failed. Please check test params.")
return output, error
def backup_restore_validate(
self, compare_uuid=False, seqno_compare_function="==", replicas=False, mode="memory", expected_error=None
):
output, error = self.backup_restore()
if expected_error:
output.extend(error)
error_found = False
if expected_error:
for line in output:
if line.find(expected_error) != -1:
error_found = True
break
self.assertTrue(error_found, "Expected error not found: %s" % expected_error)
return
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "grep 'Transfer plan finished successfully' " + self.backupset.directory + "/logs/backup.log"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if not output:
self.fail("Restoring backup failed.")
command = "grep 'Transfer failed' " + self.backupset.directory + "/logs/backup.log"
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if output:
self.fail("Restoring backup failed.")
self.log.info("Finished restoring backup")
current_vseqno = self.get_vbucket_seqnos(
self.cluster_to_restore, self.buckets, self.skip_consistency, self.per_node
)
self.log.info("*** Start to validate the restore ")
status, msg = self.validation_helper.validate_restore(
self.backupset.end,
self.vbucket_seqno,
current_vseqno,
compare_uuid=compare_uuid,
compare=seqno_compare_function,
get_replica=replicas,
mode=mode,
)
if not status:
self.fail(msg)
self.log.info(msg)
def backup_list(self):
args = "list --archive {0}".format(self.backupset.directory)
if self.backupset.backup_list_name:
args += " --repo {0}".format(self.backupset.backup_list_name)
if self.backupset.backup_incr_backup:
args += " --backup {0}".format(self.backupset.backup_incr_backup)
if self.backupset.bucket_backup:
args += " --bucket {0}".format(self.backupset.bucket_backup)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error:
return False, error, "Getting backup list failed."
else:
return True, output, "Backup list obtained"
def backup_compact(self):
args = "compact --archive {0} --repo {1} --backup {2}".format(
self.backupset.directory, self.backupset.name, self.backups[self.backupset.backup_to_compact]
)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if "Compaction succeeded," not in output[0]:
return False, output, "Compacting backup failed."
else:
return True, output, "Compaction of backup success"
def backup_remove(self):
args = "remove --archive {0} --repo {1}".format(self.backupset.directory, self.backupset.name)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
self.verify_cluster_stats()
if error:
return False, error, "Removing backup failed."
else:
return True, output, "Removing of backup success"
def backup_list_validate(self):
status, output, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_backup_list(output)
if not status:
self.fail(message)
self.log.info(message)
def backup_compact_validate(self):
self.log.info("Listing backup details before compact")
status, output_before_compact, message = self.backup_list()
if not status:
self.fail(message)
status, output, message = self.backup_compact()
if not status:
self.fail(message)
self.log.info("Listing backup details after compact")
status, output_after_compact, message = self.backup_list()
if not status:
self.fail(message)
status, message = self.validation_helper.validate_compact_lists(output_before_compact, output_after_compact)
if not status:
self.fail(message)
self.log.info(message)
def backup_merge(self):
self.log.info("backups before merge: " + str(self.backups))
self.log.info("number_of_backups_taken before merge: " + str(self.number_of_backups_taken))
try:
backup_start = self.backups[int(self.backupset.start) - 1]
except IndexError:
backup_start = "{0}{1}".format(self.backups[-1], self.backupset.start)
try:
backup_end = self.backups[int(self.backupset.end) - 1]
except IndexError:
backup_end = "{0}{1}".format(self.backups[-1], self.backupset.end)
args = "merge --archive {0} --repo {1} --start {2} --end {3}".format(
self.backupset.directory, self.backupset.name, backup_start, backup_end
)
remote_client = RemoteMachineShellConnection(self.backupset.backup_host)
command = "{0}/cbbackupmgr {1}".format(self.cli_command_location, args)
output, error = remote_client.execute_command(command)
remote_client.log_command_output(output, error)
if error or "Merge completed successfully" not in output[0]:
return False, error, "Merging backup failed"
del self.backups[self.backupset.start - 1 : self.backupset.end]
command = "ls -tr {0}/{1} | tail -1".format(self.backupset.directory, self.backupset.name)
o, e = remote_client.execute_command(command)
if o:
self.backups.insert(self.backupset.start - 1, o[0])
self.number_of_backups_taken -= self.backupset.end - self.backupset.start + 1
self.number_of_backups_taken += 1
self.log.info("backups after merge: " + str(self.backups))
self.log.info("number_of_backups_taken after merge: " + str(self.number_of_backups_taken))
return True, output, "Merging backup succeeded"
