class CliBaseTest(BaseTestCase): vbucketId = 0 def setUp(self): self.times_teardown_called = 1 super(CliBaseTest, self).setUp() self.r = random.Random() self.vbucket_count = 1024 self.cluster = Cluster() self.clusters_dic = self.input.clusters if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] elif len(self.clusters_dic) == 1: self.log.error("=== need 2 cluster to setup xdcr in ini file ===") else: self.log.error("**** Cluster config is setup in ini file. ****") self.shell = RemoteMachineShellConnection(self.master) if not self.skip_init_check_cbserver: self.rest = RestConnection(self.master) self.cb_version = self.rest.get_nodes_version() """ cli output message """ self.cli_bucket_create_msg = "SUCCESS: Bucket created" self.cli_rebalance_msg = "SUCCESS: Rebalance complete" if self.cb_version[:3] == "4.6": self.cli_bucket_create_msg = "SUCCESS: bucket-create" self.cli_rebalance_msg = "SUCCESS: rebalanced cluster" self.import_back = self.input.param("import_back", False) if self.import_back: if len(self.servers) < 3: self.fail("This test needs minimum of 3 vms to run ") self.test_type = self.input.param("test_type", "import") self.import_file = self.input.param("import_file", None) self.imex_type = self.input.param("imex_type", "json") self.format_type = self.input.param("format_type", "lines") self.import_method = self.input.param("import_method", "file://") self.force_failover = self.input.param("force_failover", False) self.json_invalid_errors = self.input.param("json-invalid-errors", None) self.field_separator = self.input.param("field-separator", "comma") self.key_gen = self.input.param("key-gen", True) self.skip_docs = self.input.param("skip-docs", None) self.limit_docs = self.input.param("limit-docs", None) self.limit_rows = self.input.param("limit-rows", None) self.skip_rows = self.input.param("skip-rows", None) self.omit_empty = self.input.param("omit-empty", None) self.infer_types = self.input.param("infer-types", None) self.fx_generator = self.input.param("fx-generator", None) self.fx_gen_start = self.input.param("fx-gen-start", None) self.secure_conn = self.input.param("secure-conn", False) self.no_cacert = self.input.param("no-cacert", False) self.no_ssl_verify = self.input.param("no-ssl-verify", False) self.verify_data = self.input.param("verify-data", False) self.field_substitutions = self.input.param("field-substitutions", None) self.check_preload_keys = self.input.param("check-preload-keys", True) self.debug_logs = self.input.param("debug-logs", False) self.should_fail = self.input.param("should-fail", False) info = self.shell.extract_remote_info() self.os_version = info.distribution_version.lower() self.deliverable_type = info.deliverable_type.lower() type = info.type.lower() self.excluded_commands = self.input.param("excluded_commands", None) self.os = 'linux' self.full_v = None self.short_v = None self.build_number = None cmd = 'curl -g {0}:8091/diag/eval -u {1}:{2} '.format(self.master.ip, self.master.rest_username, self.master.rest_password) cmd += '-d "path_config:component_path(bin)."' bin_path = subprocess.check_output(cmd, shell=True) if "bin" not in bin_path: self.fail("Check if cb server install on %s" % self.master.ip) else: self.cli_command_path = bin_path.replace('"','') + "/" self.root_path = LINUX_ROOT_PATH self.tmp_path = "/tmp/" self.tmp_path_raw = "/tmp/" self.cmd_backup_path = LINUX_BACKUP_PATH self.backup_path = LINUX_BACKUP_PATH self.cmd_ext = "" self.src_file = "" self.des_file = "" self.sample_files_path = LINUX_COUCHBASE_SAMPLE_PATH self.log_path = LINUX_COUCHBASE_LOGS_PATH self.base_cb_path = LINUX_CB_PATH """ non root path """ if self.nonroot: self.sample_files_path = "/home/%s%s" % (self.master.ssh_username, LINUX_COUCHBASE_SAMPLE_PATH) self.log_path = "/home/%s%s" % (self.master.ssh_username, LINUX_COUCHBASE_LOGS_PATH) self.base_cb_path = "/home/%s%s" % (self.master.ssh_username, LINUX_CB_PATH) self.root_path = "/home/%s/" % self.master.ssh_username if type == 'windows': self.os = 'windows' self.cmd_ext = ".exe" self.root_path = WIN_ROOT_PATH self.tmp_path = WIN_TMP_PATH self.tmp_path_raw = WIN_TMP_PATH_RAW self.cmd_backup_path = WIN_BACKUP_C_PATH self.backup_path = WIN_BACKUP_PATH self.sample_files_path = WIN_COUCHBASE_SAMPLE_PATH_C self.log_path = WIN_COUCHBASE_LOGS_PATH win_format = "C:/Program Files" cygwin_format = "/cygdrive/c/Program\ Files" if win_format in self.cli_command_path: self.cli_command_path = self.cli_command_path.replace(win_format, cygwin_format) self.base_cb_path = WIN_CB_PATH if info.distribution_type.lower() == 'mac': self.os = 'mac' self.full_v, self.short_v, self.build_number = self.shell.get_cbversion(type) self.couchbase_usrname = "%s" % (self.input.membase_settings.rest_username) self.couchbase_password = "******" % (self.input.membase_settings.rest_password) self.cb_login_info = "%s:%s" % (self.couchbase_usrname, self.couchbase_password) self.path_type = self.input.param("path_type", None) if self.path_type is None: self.log.info("Test command with absolute path ") elif self.path_type == "local": self.log.info("Test command at %s dir " % self.cli_command_path) self.cli_command_path = "cd %s; ./" % self.cli_command_path self.cli_command = self.input.param("cli_command", None) self.command_options = self.input.param("command_options", None) if self.command_options is not None: self.command_options = self.command_options.split(";") if str(self.__class__).find('couchbase_clitest.CouchbaseCliTest') == -1: if len(self.servers) > 1 and int(self.nodes_init) == 1: servers_in = [self.servers[i + 1] for i in range(self.num_servers - 1)] self.cluster.rebalance(self.servers[:1], servers_in, []) for bucket in self.buckets: testuser = [{'id': bucket.name, 'name': bucket.name, 'password': '******'}] rolelist = [{'id': bucket.name, 'name': bucket.name, 'roles': 'admin'}] self.add_built_in_server_user(testuser=testuser, rolelist=rolelist) def tearDown(self): if not self.input.param("skip_cleanup", True): if self.times_teardown_called > 1 : self.shell.disconnect() if self.input.param("skip_cleanup", True): if self.case_number > 1 or self.times_teardown_called > 1: self.shell.disconnect() self.times_teardown_called += 1 serverInfo = self.servers[0] rest = RestConnection(serverInfo) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone) self.clusters_dic = self.input.clusters if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] if self.dest_nodes and len(self.dest_nodes) > 1: self.log.info("======== clean up destination cluster =======") rest = RestConnection(self.dest_nodes[0]) rest.remove_all_remote_clusters() rest.remove_all_replications() BucketOperationHelper.delete_all_buckets_or_assert(self.dest_nodes, self) ClusterOperationHelper.cleanup_cluster(self.dest_nodes) elif len(self.clusters_dic) == 1: self.log.error("=== need 2 cluster to setup xdcr in ini file ===") else: self.log.info("**** If run xdcr test, need cluster config is setup in ini file. ****") super(CliBaseTest, self).tearDown() """ in sherlock, there is an extra value called runCmd in the 1st element """ def del_runCmd_value(self, output): if "runCmd" in output[0]: output = output[1:] return output def verifyCommandOutput(self, output, expect_error, message): """Inspects each line of the output and checks to see if the expected error was found Options: output - A list of output lines expect_error - Whether or not the command should have succeeded or failed message - The success or error message Returns a boolean indicating whether or not the error/success message was found in the output """ if expect_error: for line in output: if line == "ERROR: " + message: return True log.info("Did not receive expected error message `ERROR: %s`", message) return False else: for line in output: if line == "SUCCESS: " + message: return True log.info("Did not receive expected success message `SUCCESS: %s`", message) return False def verifyWarningOutput(self, output, message): for line in output: if line == "WARNING: " + message: return True log.info("Did not receive expected error message `WARNING: %s`", message) return False def verifyServices(self, server, expected_services): """Verifies that the services on a given node match the expected service Options: server - A TestInputServer object of the server to connect to expected_services - A comma separated list of services Returns a boolean corresponding to whether or not the expected services are available on the server. """ rest = RestConnection(server) hostname = "%s:%s" % (server.ip, server.port) expected_services = expected_services.replace("data", "kv") expected_services = expected_services.replace("query", "n1ql") expected_services = expected_services.split(",") nodes_services = rest.get_nodes_services() for node, services in nodes_services.iteritems(): if node.encode('ascii') == hostname: if len(services) != len(expected_services): log.info("Services on %s do not match expected services (%s vs. %s)", hostname, services, expected_services) return False for service in services: if service.encode("ascii") not in expected_services: log.info("Services on %s do not match expected services (%s vs. %s)", hostname, services, expected_services) return False return True log.info("Services on %s not found, the server may not exist", hostname) return False def verifyRamQuotas(self, server, data, index, fts): """Verifies that the RAM quotas for each service are set properly Options: server - A TestInputServer object of the server to connect to data - An int containing the data service RAM quota, None will skip the check index - An int containing the index service RAM quota, None will skip the check fts - An int containing the FTS service RAM quota, None will skip the check Returns a boolean corresponding to whether or not the RAM quotas were set properly """ rest = RestConnection(server) settings = rest.get_pools_default() if data: if "memoryQuota" not in settings: log.info("Unable to get data service ram quota") return False if int(settings["memoryQuota"]) != int(data): log.info("Data service memory quota does not match (%d vs %d)", int(settings["memoryQuota"]), int(data)) return False if index: if "indexMemoryQuota" not in settings: log.info("Unable to get index service ram quota") return False if int(settings["indexMemoryQuota"]) != int(index): log.info( "Index service memory quota does not match (%d vs %d)", int(settings["indexMemoryQuota"]), int(index)) return False if fts: if "ftsMemoryQuota" not in settings: log.info("Unable to get fts service ram quota") return False if int(settings["ftsMemoryQuota"]) != int(fts): log.info("FTS service memory quota does not match (%d vs %d)", int(settings["ftsMemoryQuota"]), int(fts)) return False return True def verifyBucketSettings(self, server, bucket_name, bucket_type, memory_quota, eviction_policy, replica_count, enable_index_replica, priority, enable_flush): rest = RestConnection(server) result = rest.get_bucket_json(bucket_name) if bucket_type == "couchbase": bucket_type = "membase" if bucket_type is not None and bucket_type != result["bucketType"]: log.info("Memory quota does not match (%s vs %s)", bucket_type, result["bucketType"]) return False quota = result["quota"]["rawRAM"] / 1024 / 1024 if memory_quota is not None and memory_quota != quota: log.info("Bucket quota does not match (%s vs %s)", memory_quota, quota) return False if eviction_policy is not None and eviction_policy != result[ "evictionPolicy"]: log.info("Eviction policy does not match (%s vs %s)", eviction_policy, result["evictionPolicy"]) return False if replica_count is not None and replica_count != result[ "replicaNumber"]: log.info("Replica count does not match (%s vs %s)", replica_count, result["replicaNumber"]) return False if enable_index_replica == 1: enable_index_replica = True elif enable_index_replica == 0: enable_index_replica = False if enable_index_replica is not None and enable_index_replica != result[ "replicaIndex"]: log.info("Replica index enabled does not match (%s vs %s)", enable_index_replica, result["replicaIndex"]) return False if priority == "high": priority = 8 elif priority == "low": priority = 3 if priority is not None and priority != result["threadsNumber"]: log.info("Bucket priority does not match (%s vs %s)", priority, result["threadsNumber"]) return False if enable_flush is not None: if enable_flush == 1 and "flush" not in result["controllers"]: log.info("Bucket flush is not enabled, but it should be") return False elif enable_flush == 0 and "flush" in result["controllers"]: log.info("Bucket flush is not enabled, but it should be") return False return True def verifyContainsBucket(self, server, name): rest = RestConnection(server) buckets = rest.get_buckets() for bucket in buckets: if bucket.name == name: return True return False def verifyClusterName(self, server, name): rest = RestConnection(server) settings = rest.get_pools_default("waitChange=0") if name is None: name = "" if name == "empty": name = " " if "clusterName" not in settings: log.info("Unable to get cluster name from server") return False if settings["clusterName"] != name: log.info("Cluster name does not match (%s vs %s)", settings["clusterName"], name) return False return True def isClusterInitialized(self, server): """Checks whether or not the server is initialized Options: server - A TestInputServer object of the server to connect to Checks to see whether or not the default pool was created in order to determine whether or no the server was initialized. Returns a boolean value to indicate initialization. """ rest = RestConnection(server) settings = rest.get_pools_info() if "pools" in settings and len(settings["pools"]) > 0: return True return False def verifyNotificationsEnabled(self, server): rest = RestConnection(server) enabled = rest.get_notifications() if enabled: return True return False def verifyIndexSettings(self, server, max_rollbacks, stable_snap_interval, mem_snap_interval, storage_mode, threads, log_level): rest = RestConnection(server) settings = rest.get_global_index_settings() if storage_mode == "default": storage_mode = "plasma" elif storage_mode == "memopt": storage_mode = "memory_optimized" if max_rollbacks and str(settings["maxRollbackPoints"]) != str( max_rollbacks): log.info("Max rollbacks does not match (%s vs. %s)", str(settings["maxRollbackPoints"]), str(max_rollbacks)) return False if stable_snap_interval and str( settings["stableSnapshotInterval"]) != str( stable_snap_interval): log.info("Stable snapshot interval does not match (%s vs. %s)", str(settings["stableSnapshotInterval"]), str(stable_snap_interval)) return False if mem_snap_interval and str( settings["memorySnapshotInterval"]) != str(mem_snap_interval): log.info("Memory snapshot interval does not match (%s vs. %s)", str(settings["memorySnapshotInterval"]), str(mem_snap_interval)) return False if storage_mode and str(settings["storageMode"]) != str(storage_mode): log.info("Storage mode does not match (%s vs. %s)", str(settings["storageMode"]), str(storage_mode)) return False if threads and str(settings["indexerThreads"]) != str(threads): log.info("Threads does not match (%s vs. %s)", str(settings["indexerThreads"]), str(threads)) return False if log_level and str(settings["logLevel"]) != str(log_level): log.info("Log level does not match (%s vs. %s)", str(settings["logLevel"]), str(log_level)) return False return True def verifyAutofailoverSettings(self, server, enabled, timeout): rest = RestConnection(server) settings = rest.get_autofailover_settings() if enabled and not ((str(enabled) == "1" and settings.enabled) or ( str(enabled) == "0" and not settings.enabled)): log.info("Enabled does not match (%s vs. %s)", str(enabled), str(settings.enabled)) return False if timeout and str(settings.timeout) != str(timeout): log.info("Timeout does not match (%s vs. %s)", str(timeout), str(settings.timeout)) return False return True def verifyAutoreprovisionSettings(self, server, enabled, max_nodes): rest = RestConnection(server) settings = rest.get_autoreprovision_settings() if enabled and not ((str(enabled) == "1" and settings.enabled) or ( str(enabled) == "0" and not settings.enabled)): log.info("Enabled does not match (%s vs. %s)", str(max_nodes), str(settings.enabled)) return False if max_nodes and str(settings.max_nodes) != str(max_nodes): log.info("max_nodes does not match (%s vs. %s)", str(max_nodes), str(settings.max_nodes)) return False return True def verifyAuditSettings(self, server, enabled, log_path, rotate_interval): rest = RestConnection(server) settings = rest.getAuditSettings() if enabled and not ( (str(enabled) == "1" and settings["auditdEnabled"]) or ( str(enabled) == "0" and not settings["auditdEnabled"])): log.info("Enabled does not match (%s vs. %s)", str(enabled), str(settings["auditdEnabled"])) return False if log_path and str(str(settings["logPath"])) != str(log_path): log.info("Log path does not match (%s vs. %s)", str(log_path), str(settings["logPath"])) return False if rotate_interval and str(str(settings["rotateInterval"])) != str( rotate_interval): log.info("Rotate interval does not match (%s vs. %s)", str(rotate_interval), str(settings["rotateInterval"])) return False return True def verifyPendingServer(self, server, server_to_add, group_name, services): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False expected_services = services.replace("data", "kv") expected_services = expected_services.replace("query", "n1ql") expected_services = expected_services.split(",") for group in settings["groups"]: for node in group["nodes"]: if node["hostname"] == server_to_add: if node["clusterMembership"] != "inactiveAdded": log.info("Node `%s` not in pending status", server_to_add) return False if group["name"] != group_name: log.info("Node `%s` not in correct group (%s vs %s)", node["hostname"], group_name, group["name"]) return False if len(node["services"]) != len(expected_services): log.info("Services do not match on %s (%s vs %s) ", node["hostname"], services, ",".join(node["services"])) return False for service in node["services"]: if service not in expected_services: log.info("Services do not match on %s (%s vs %s) ", node["hostname"], services, ",".join(node["services"])) return False return True log.info("Node `%s` not found in nodes list", server_to_add) return False def verifyPendingServerDoesNotExist(self, server, server_to_add): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False for group in settings["groups"]: for node in group["nodes"]: if node["hostname"] == server_to_add: return False log.info("Node `%s` not found in nodes list", server_to_add) return True def verifyActiveServers(self, server, expected_num_servers): return self._verifyServersByStatus(server, expected_num_servers, "active") def verifyFailedServers(self, server, expected_num_servers): return self._verifyServersByStatus(server, expected_num_servers, "inactiveFailed") def _verifyServersByStatus(self, server, expected_num_servers, status): rest = RestConnection(server) settings = rest.get_pools_default() count = 0 for node in settings["nodes"]: if node["clusterMembership"] == status: count += 1 return count == expected_num_servers def verifyRecoveryType(self, server, recovery_servers, recovery_type): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False if not recovery_servers: return True num_found = 0 recovery_servers = recovery_servers.split(",") for group in settings["groups"]: for node in group["nodes"]: for rs in recovery_servers: if node["hostname"] == rs: if node["recoveryType"] != recovery_type: log.info( "Node %s doesn't contain recovery type %s ", rs, recovery_type) return False else: num_found = num_found + 1 if num_found == len(recovery_servers): return True log.info("Node `%s` not found in nodes list", ",".join(recovery_servers)) return False def verifyUserRoles(self, server, username, roles): rest = RestConnection(server) status, content, header = rbacmain(server)._retrieve_user_roles() content = json.loads(content) temp = rbacmain()._parse_get_user_response(content, username, username, roles) return temp def verifyLdapSettings(self, server, admins, ro_admins, default, enabled): rest = RestConnection(server) settings = rest.ldapRestOperationGetResponse() if admins is None: admins = [] else: admins = admins.split(",") if ro_admins is None: ro_admins = [] else: ro_admins = ro_admins.split(",") if str(enabled) == "0": admins = [] ro_admins = [] if default == "admins" and str(enabled) == "1": if settings["admins"] != "asterisk": log.info("Admins don't match (%s vs asterisk)", settings["admins"]) return False elif not self._list_compare(settings["admins"], admins): log.info("Admins don't match (%s vs %s)", settings["admins"], admins) return False if default == "roadmins" and str(enabled) == "1": if settings["roAdmins"] != "asterisk": log.info("Read only admins don't match (%s vs asterisk)", settings["roAdmins"]) return False elif not self._list_compare(settings["roAdmins"], ro_admins): log.info("Read only admins don't match (%s vs %s)", settings["roAdmins"], ro_admins) return False return True def verifyAlertSettings(self, server, enabled, email_recipients, email_sender, email_username, email_password, email_host, email_port, encrypted, alert_af_node, alert_af_max_reached, alert_af_node_down, alert_af_small, alert_af_disable, alert_ip_changed, alert_disk_space, alert_meta_overhead, alert_meta_oom, alert_write_failed, alert_audit_dropped): rest = RestConnection(server) settings = rest.get_alerts_settings() print settings if not enabled: if not settings["enabled"]: return True else: log.info("Alerts should be disabled") return False if encrypted is None or encrypted == "0": encrypted = False else: encrypted = True if email_recipients is not None and not self._list_compare( email_recipients.split(","), settings["recipients"]): log.info("Email recipients don't match (%s vs %s)", email_recipients.split(","), settings["recipients"]) return False if email_sender is not None and email_sender != settings["sender"]: log.info("Email sender does not match (%s vs %s)", email_sender, settings["sender"]) return False if email_username is not None and email_username != \ settings["emailServer"]["user"]: log.info("Email username does not match (%s vs %s)", email_username, settings["emailServer"]["user"]) return False if email_host is not None and email_host != settings["emailServer"][ "host"]: log.info("Email host does not match (%s vs %s)", email_host, settings["emailServer"]["host"]) return False if email_port is not None and email_port != settings["emailServer"][ "port"]: log.info("Email port does not match (%s vs %s)", email_port, settings["emailServer"]["port"]) return False if encrypted is not None and encrypted != settings["emailServer"][ "encrypt"]: log.info("Email encryption does not match (%s vs %s)", encrypted, settings["emailServer"]["encrypt"]) return False alerts = list() if alert_af_node: alerts.append('auto_failover_node') if alert_af_max_reached: alerts.append('auto_failover_maximum_reached') if alert_af_node_down: alerts.append('auto_failover_other_nodes_down') if alert_af_small: alerts.append('auto_failover_cluster_too_small') if alert_af_disable: alerts.append('auto_failover_disabled') if alert_ip_changed: alerts.append('ip') if alert_disk_space: alerts.append('disk') if alert_meta_overhead: alerts.append('overhead') if alert_meta_oom: alerts.append('ep_oom_errors') if alert_write_failed: alerts.append('ep_item_commit_failed') if alert_audit_dropped: alerts.append('audit_dropped_events') if not self._list_compare(alerts, settings["alerts"]): log.info("Alerts don't match (%s vs %s)", alerts, settings["alerts"]) return False return True def verify_node_settings(self, server, data_path, index_path, hostname): rest = RestConnection(server) node_settings = rest.get_nodes_self() if data_path != node_settings.storage[0].path: log.info("Data path does not match (%s vs %s)", data_path, node_settings.storage[0].path) return False if index_path != node_settings.storage[0].index_path: log.info("Index path does not match (%s vs %s)", index_path, node_settings.storage[0].index_path) return False if hostname is not None: if hostname != node_settings.hostname: log.info("Hostname does not match (%s vs %s)", hostname, node_settings.hostname) return True return True def verifyCompactionSettings(self, server, db_frag_perc, db_frag_size, view_frag_perc, view_frag_size, from_period, to_period, abort_outside, parallel_compact, purgeInt): rest = RestConnection(server) settings = rest.get_auto_compaction_settings() ac = settings["autoCompactionSettings"] if db_frag_perc is not None and str(db_frag_perc) != str( ac["databaseFragmentationThreshold"]["percentage"]): log.info("DB frag perc does not match (%s vs %s)", str(db_frag_perc), str(ac["databaseFragmentationThreshold"]["percentage"])) return False if db_frag_size is not None and str(db_frag_size * 1024 ** 2) != str( ac["databaseFragmentationThreshold"]["size"]): log.info("DB frag size does not match (%s vs %s)", str(db_frag_size * 1024 ** 2), str(ac["databaseFragmentationThreshold"]["size"])) return False if view_frag_perc is not None and str(view_frag_perc) != str( ac["viewFragmentationThreshold"]["percentage"]): log.info("View frag perc does not match (%s vs %s)", str(view_frag_perc), str(ac["viewFragmentationThreshold"]["percentage"])) return False if view_frag_size is not None and str( view_frag_size * 1024 ** 2) != str( ac["viewFragmentationThreshold"]["size"]): log.info("View frag size does not match (%s vs %s)", str(view_frag_size * 1024 ** 2), str(ac["viewFragmentationThreshold"]["size"])) return False print from_period, to_period if from_period is not None: fromHour, fromMin = from_period.split(":", 1) if int(fromHour) != int(ac["allowedTimePeriod"]["fromHour"]): log.info("From hour does not match (%s vs %s)", str(fromHour), str(ac["allowedTimePeriod"]["fromHour"])) return False if int(fromMin) != int(ac["allowedTimePeriod"]["fromMinute"]): log.info("From minute does not match (%s vs %s)", str(fromMin), str(ac["allowedTimePeriod"]["fromMinute"])) return False if to_period is not None: toHour, toMin = to_period.split(":", 1) if int(toHour) != int(ac["allowedTimePeriod"]["toHour"]): log.info("To hour does not match (%s vs %s)", str(toHour), str(ac["allowedTimePeriod"]["toHour"])) return False if int(toMin) != int(ac["allowedTimePeriod"]["toMinute"]): log.info("To minute does not match (%s vs %s)", str(toMin), str(ac["allowedTimePeriod"]["toMinute"])) return False if str(abort_outside) == "1": abort_outside = True elif str(abort_outside) == "0": abort_outside = False if abort_outside is not None and abort_outside != \ ac["allowedTimePeriod"]["abortOutside"]: log.info("Abort outside does not match (%s vs %s)", abort_outside, ac["allowedTimePeriod"]["abortOutside"]) return False if str(parallel_compact) == "1": parallel_compact = True elif str(parallel_compact) == "0": parallel_compact = False if parallel_compact is not None and parallel_compact != ac[ "parallelDBAndViewCompaction"]: log.info("Parallel compact does not match (%s vs %s)", str(parallel_compact), str(ac["parallelDBAndViewCompaction"])) return False if purgeInt is not None and str(purgeInt) != str( settings["purgeInterval"]): log.info("Purge interval does not match (%s vs %s)", str(purgeInt), str(settings["purgeInterval"])) return False return True def verify_gsi_compact_settings(self, compact_mode, compact_percent, compact_interval, from_period, to_period, enable_abort): rest = RestConnection(self.master) settings = rest.get_auto_compaction_settings() ac = settings["autoCompactionSettings"]["indexFragmentationThreshold"] cc = settings["autoCompactionSettings"]["indexCircularCompaction"] if compact_mode is not None: if compact_mode == "append": self.log.info("append compactino settings %s " % ac) if compact_percent is not None and \ compact_percent != ac["percentage"]: raise Exception( "setting percent does not match. Set: %s vs %s :Actual" % (compact_percent, ac["percentage"])) if compact_mode == "circular": self.log.info("circular compaction settings %s " % cc) if enable_abort and not cc["interval"]["abortOutside"]: raise Exception("setting enable abort failed") if compact_interval is not None: if compact_interval != cc["daysOfWeek"]: raise Exception( "Failed to set compaction on %s " % compact_interval) elif from_period is None and int( cc["interval"]["fromHour"]) != 0 and \ int(cc["interval"]["fromMinute"]) != 0: raise Exception( "fromHour and fromMinute should be zero") if compact_interval is None: if (from_period != str(cc["interval"][ "fromHour"]) + ":" + str( cc["interval"]["fromMinute"])) \ and (to_period != str(cc["interval"]["toHour"]) + ":" + str( cc["interval"]["toMinute"])): raise Exception( "fromHour and fromMinute do not set correctly") return True def verifyGroupExists(self, server, name): rest = RestConnection(server) groups = rest.get_zone_names() print groups for gname, _ in groups.iteritems(): if name == gname: return True return False def _list_compare(self, list1, list2): if len(list1) != len(list2): return False for elem1 in list1: found = False for elem2 in list2: if elem1 == elem2: found = True break if not found: return False return True def waitForItemCount(self, server, bucket_name, count, timeout=30): rest = RestConnection(server) for sec in range(timeout): items = int( rest.get_bucket_json(bucket_name)["basicStats"]["itemCount"]) if items != count: time.sleep(1) else: return True log.info("Waiting for item count to be %d timed out", count) return False
class Capi(XDCRNewBaseTest, NewUpgradeBaseTest): def setUp(self): super(Capi, self).setUp() self.cluster = Cluster() self.src_cluster = self.get_cb_cluster_by_name('C1') self.src_master = self.src_cluster.get_master_node() self.dest_cluster = self.get_cb_cluster_by_name('C2') self.dest_master = self.dest_cluster.get_master_node() self.use_hostnames = self._input.param("use_hostnames", False) self.src_init = self._input.param('src_init', 2) self.dest_init = self._input.param('dest_init', 1) self.product = self._input.param('product', 'couchbase-server') self.initial_version = self._input.param('initial_version', '2.5.1-1083') self.initial_vbuckets = self._input.param('initial_vbuckets', 1024) self.init_nodes = self._input.param('init_nodes', True) self.initial_build_type = self._input.param('initial_build_type', None) self.upgrade_build_type = self._input.param('upgrade_build_type', self.initial_build_type) self.master = self.src_master self.rest = RestConnection(self.src_master) def tearDown(self): super(Capi, self).tearDown() def _start_es_replication(self, bucket='default', xdcr_params={}): rest_conn = RestConnection(self.src_cluster.get_master_node()) if bucket == 'default': self.log.info("Creating default bucket") rest_conn.create_bucket(bucket='default', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3, flushEnabled=1, lww=False) self.src_cluster.add_bucket(ramQuotaMB=100, bucket='default', authType='none', saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase', evictionPolicy='valueOnly') elif bucket == 'sasl': self.log.info("Creating sasl bucket") rest_conn.create_bucket(bucket='sasl', ramQuotaMB=100, authType='sasl', saslPassword='******', replicaNumber=1, proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3, flushEnabled=1, lww=False) self.src_cluster.add_bucket(ramQuotaMB=100, bucket='sasl', authType='sasl', saslPassword='******', replicaNumber=1, proxyPort=11211, bucketType='membase', evictionPolicy='valueOnly') elif bucket == 'standard': self.log.info("Creating standard bucket") rest_conn.create_bucket(bucket='standard', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1, proxyPort=STANDARD_BUCKET_PORT, bucketType='membase', replica_index=1, threadsNumber=3, flushEnabled=1, lww=False) self.src_cluster.add_bucket(ramQuotaMB=100, bucket='standard', authType='none', saslPassword='', replicaNumber=1, proxyPort=STANDARD_BUCKET_PORT, bucketType='membase', evictionPolicy='valueOnly') elif bucket== 'lww': self.log.info("Creating lww bucket") rest_conn.create_bucket(bucket='lww', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3, flushEnabled=1, lww=True) self.src_cluster.add_bucket(ramQuotaMB=100, bucket='lww', authType='none', saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase', evictionPolicy='valueOnly') esrest_conn = EsRestConnection(self.dest_cluster.get_master_node()) esrest_conn.create_index(bucket) rest_conn.add_remote_cluster(remoteIp=self.dest_master.ip, remotePort=9091, username='******', password='******', name='es') self.src_cluster.get_remote_clusters().append(XDCRRemoteClusterRef(self.src_cluster, self.dest_cluster, Utility.get_rc_name(self.src_cluster.get_name(), self.dest_cluster.get_name()))) repl_id = rest_conn.start_replication(replicationType='continuous', fromBucket=bucket, toCluster='es', rep_type='capi', toBucket=bucket, xdcr_params=xdcr_params) return repl_id def _verify_es_results(self, bucket='default'): esrest_conn = EsRestConnection(self.dest_master) es_docs = esrest_conn.all_docs() self.log.info("Retrieved ES Docs") rest_conn = RestConnection(self.src_master) memcached_conn = VBucketAwareMemcached(rest_conn, bucket) self.log.info("Comparing CB and ES data") for doc in es_docs: es_data = doc['doc'] mc_active = memcached_conn.memcached(str(es_data['_id'])) cb_flags, cb_cas, cb_data = mc_active.get(str(es_data['_id'])) self.assertDictEqual(es_data, json.loads(cb_data), "Data mismatch found - es data: {0} cb data: {1}". format(str(es_data), str(cb_data))) self.log.info("Data verified") def test_crud_ops_from_cb_to_es(self): bucket = self._input.param("bucket", 'default') repl_id = self._start_es_replication(bucket=bucket) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results(bucket=bucket) def test_incr_crud_ops_from_cb_to_es(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.async_perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_pause_resume(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.async_load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') self.sleep(30) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_checkpointing(self): repl_id = self._start_es_replication(xdcr_params={"checkpointInterval":"60"}) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.sleep(120) vb0_node = None nodes = self.src_cluster.get_nodes() ip = VBucketAwareMemcached(rest_conn,'default').vBucketMap[0].split(':')[0] for node in nodes: if ip == node.ip: vb0_node = node if not vb0_node: raise XDCRCheckpointException("Error determining the node containing active vb0") vb0_conn = RestConnection(vb0_node) try: checkpoint_record = vb0_conn.get_recent_xdcr_vb_ckpt(repl_id) self.log.info("Checkpoint record : {0}".format(checkpoint_record)) except Exception as e: raise XDCRCheckpointException("Error retrieving last checkpoint document - {0}".format(e)) self._verify_es_results() def test_capi_with_optimistic_replication(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.set_xdcr_param('default', 'default', 'optimisticReplicationThreshold', self._optimistic_threshold) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_filter(self): repl_id = self._start_es_replication(xdcr_params={'filterExpression':'es-5*'}) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_advanced_settings(self): batch_count = self._input.param("batch_count", 10) batch_size = self._input.param("batch_size", 2048) source_nozzle = self._input.param("source_nozzle", 2) target_nozzle = self._input.param("target_nozzle", 2) enable_firewall = self._input.param("enable_firewall", False) capi_data_chan_size_multi = self._input.param("capi_data_chan_size_multi", None) if capi_data_chan_size_multi: shell = RemoteMachineShellConnection(self.src_master) command = "curl -X POST -u Administrator:password http://127.0.0.1:9998/xdcr/internalSettings " + \ "-d CapiDataChanSizeMultiplier=" + str(capi_data_chan_size_multi) output, error = shell.execute_command(command) shell.log_command_output(output, error) repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.set_xdcr_param('default', 'default', 'workerBatchSize', batch_count) rest_conn.set_xdcr_param('default', 'default', 'docBatchSizeKb', batch_size) rest_conn.set_xdcr_param('default', 'default', 'sourceNozzlePerNode', source_nozzle) rest_conn.set_xdcr_param('default', 'default', 'targetNozzlePerNode', target_nozzle) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') if enable_firewall: NodeHelper.enable_firewall(self.dest_cluster.get_master_node()) self.sleep(120) NodeHelper.disable_firewall(self.dest_cluster.get_master_node()) self._verify_es_results() def test_capi_with_rebalance_in(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.src_cluster.rebalance_in() self._wait_for_es_replication_to_catchup(timeout=900) self._verify_es_results() def test_capi_with_rebalance_out(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.src_cluster.rebalance_out() self._wait_for_es_replication_to_catchup(timeout=900) self._verify_es_results() def test_capi_with_swap_rebalance(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.src_cluster.swap_rebalance() self._wait_for_es_replication_to_catchup(timeout=600) self._verify_es_results() def test_capi_with_failover(self): repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') graceful = self._input.param("graceful", False) self.recoveryType = self._input.param("recoveryType", None) self.src_cluster.failover(graceful=graceful) self.sleep(30) if self.recoveryType: server_nodes = rest_conn.node_statuses() for node in server_nodes: if node.ip == self._input.servers[1].ip: rest_conn.set_recovery_type(otpNode=node.id, recoveryType=self.recoveryType) self.sleep(30) rest_conn.add_back_node(otpNode=node.id) rebalance = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], []) rebalance.result() self._verify_es_results() def test_capi_with_malformed_http_resp(self): repl_id = self._start_es_replication(xdcr_params={'workerBatchSize':'2000', 'docBatchSizeKb':'8096', 'targetNozzlePerNode':'64'}) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() goxdcr_log = NodeHelper.get_goxdcr_log_dir(self.src_master)\ + '/goxdcr.log*' for node in self.src_cluster.get_nodes(): count = NodeHelper.check_goxdcr_log( node, "malformed HTTP response", goxdcr_log) self.assertEqual(count, 0, "malformed HTTP response error message found in " + str(node.ip)) self.log.info("malformed HTTP response error message not found in " + str(node.ip)) self._verify_es_results() def test_capi_with_offline_upgrade(self): self._install(self._input.servers[:self.src_init + self.dest_init]) upgrade_nodes = self.src_cluster.get_nodes() upgrade_version = self._input.param("upgrade_version", "5.0.0-1797") repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') self._install(servers=upgrade_nodes, version=upgrade_version) self.log.info("######### Upgrade of CB cluster completed ##########") repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value"}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_online_upgrade(self): self._install(self._input.servers[:self.src_init + self.dest_init]) upgrade_version = self._input.param("upgrade_version", "5.0.0-1797") upgrade_nodes = self.src_cluster.get_nodes() extra_nodes = self._input.servers[self.src_init + self.dest_init:] repl_id = self._start_es_replication() rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') RestConnection(upgrade_nodes[0]).get_nodes_versions() added_versions = RestConnection(extra_nodes[0]).get_nodes_versions() self.cluster.rebalance(upgrade_nodes + extra_nodes, extra_nodes, []) self.log.info("Rebalance in all {0} nodes completed".format(added_versions[0])) RestConnection(upgrade_nodes[0]).get_nodes_versions() self.sleep(15) status, content = ClusterOperationHelper.find_orchestrator(upgrade_nodes[0]) self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\ format(status, content)) self.log.info("after rebalance in the master is {0}".format(content)) find_master = False for new_server in extra_nodes: if content.find(new_server.ip) >= 0: find_master = True self.log.info("{0} Node {1} becomes the master".format(added_versions[0], new_server.ip)) break if not find_master: raise Exception("After rebalance in {0} Nodes, one of them doesn't become the master". format(added_versions[0])) self.log.info("Rebalancing out all old version nodes") self.cluster.rebalance(upgrade_nodes + extra_nodes, [], upgrade_nodes) self.src_master = self._input.servers[self.src_init + self.dest_init] self._install(self.src_cluster.get_nodes(), version=upgrade_version) upgrade_nodes = self._input.servers[self.src_init + self.dest_init:] extra_nodes = self.src_cluster.get_nodes() RestConnection(upgrade_nodes[0]).get_nodes_versions() added_versions = RestConnection(extra_nodes[0]).get_nodes_versions() self.cluster.rebalance(upgrade_nodes + extra_nodes, extra_nodes, []) self.log.info("Rebalance in all {0} nodes completed".format(added_versions[0])) RestConnection(upgrade_nodes[0]).get_nodes_versions() self.sleep(15) status, content = ClusterOperationHelper.find_orchestrator(upgrade_nodes[0]) self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\ format(status, content)) self.log.info("after rebalance in the master is {0}".format(content)) self.log.info("Rebalancing out all old version nodes") self.cluster.rebalance(upgrade_nodes + extra_nodes, [], upgrade_nodes) self.src_master = self._input.servers[0] self.log.info("######### Upgrade of CB cluster completed ##########") rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value"}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.perform_update_delete() rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self._wait_for_es_replication_to_catchup() self._verify_es_results() def test_capi_with_cb_stop_and_start(self): bucket = self._input.param("bucket", 'default') repl_id = self._start_es_replication(bucket=bucket) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.async_perform_update_delete() conn = RemoteMachineShellConnection(self.src_master) conn.stop_couchbase() conn.start_couchbase() self.sleep(30) self._wait_for_es_replication_to_catchup() self._verify_es_results(bucket=bucket) def test_capi_with_erlang_crash(self): bucket = self._input.param("bucket", 'default') repl_id = self._start_es_replication(bucket=bucket) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.async_perform_update_delete() conn = RemoteMachineShellConnection(self.src_master) conn.kill_erlang() conn.start_couchbase() self.sleep(30) self._wait_for_es_replication_to_catchup() self._verify_es_results(bucket=bucket) def test_capi_with_memcached_crash(self): bucket = self._input.param("bucket", 'default') repl_id = self._start_es_replication(bucket=bucket) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.async_perform_update_delete() conn = RemoteMachineShellConnection(self.src_master) conn.pause_memcached() conn.unpause_memcached() self.sleep(30) self._wait_for_es_replication_to_catchup() self._verify_es_results(bucket=bucket)
class BucketConfig(BaseTestCase): def setUp(self): super(BucketConfig, self).setUp() self.testcase = '2' self.log = logger.Logger.get_logger() self.input = TestInputSingleton.input self.servers = self.input.servers #self.time_synchronization = self.input.param("time_sync", "enabledWithoutDrift") self.lww = self.input.param("lww", True) self.drift = self.input.param("drift", False) self.bucket='bucket-1' self.master = self.servers[0] self.rest = RestConnection(self.master) self.cluster = Cluster() self.skip_rebalance = self.input.param("skip_rebalance", False) node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers) mem_quota = int(self.rest.get_nodes_self().mcdMemoryReserved * node_ram_ratio) if not self.skip_rebalance: self.rest.init_cluster(self.master.rest_username, self.master.rest_password) self.rest.init_cluster_memoryQuota(self.master.rest_username, self.master.rest_password, memoryQuota=mem_quota) for server in self.servers: ClusterOperationHelper.cleanup_cluster([server]) ClusterOperationHelper.wait_for_ns_servers_or_assert( [self.master], self.testcase) try: rebalanced = ClusterOperationHelper.add_and_rebalance( self.servers) except Exception as e: self.fail(e, 'cluster is not rebalanced') self._create_bucket(self.lww, self.drift) def tearDown(self): super(BucketConfig, self).tearDown() return if not "skip_cleanup" in TestInputSingleton.input.test_params: BucketOperationHelper.delete_all_buckets_or_assert( self.servers, self.testcase) ClusterOperationHelper.cleanup_cluster(self.servers) ClusterOperationHelper.wait_for_ns_servers_or_assert( self.servers, self.testcase) def test_modify_bucket_params(self): try: self.log.info("Modifying timeSynchronization value after bucket creation .....") self._modify_bucket() except Exception as e: traceback.print_exc() self.fail('[ERROR] Modify testcase failed .., {0}'.format(e)) def test_restart(self): try: self.log.info("Restarting the servers ..") self._restart_server(self.servers[:]) self.log.info("Verifying bucket settings after restart ..") self._check_config() except Exception as e: traceback.print_exc() self.fail("[ERROR] Check data after restart failed with exception {0}".format(e)) def test_failover(self): num_nodes=1 self.cluster.failover(self.servers, self.servers[1:num_nodes]) try: self.log.info("Failing over 1 of the servers ..") self.cluster.rebalance(self.servers, [], self.servers[1:num_nodes]) self.log.info("Verifying bucket settings after failover ..") self._check_config() except Exception as e: traceback.print_exc() self.fail('[ERROR]Failed to failover .. , {0}'.format(e)) def test_rebalance_in(self): try: self.log.info("Rebalancing 1 of the servers ..") ClusterOperationHelper.add_and_rebalance( self.servers) self.log.info("Verifying bucket settings after rebalance ..") self._check_config() except Exception as e: self.fail('[ERROR]Rebalance failed .. , {0}'.format(e)) def test_backup_same_cluster(self): self.shell = RemoteMachineShellConnection(self.master) self.buckets = RestConnection(self.master).get_buckets() self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.backup_location = "/tmp/backup" self.command_options = self.input.param("command_options", '') try: shell = RemoteMachineShellConnection(self.master) self.shell.execute_cluster_backup(self.couchbase_login_info, self.backup_location, self.command_options) time.sleep(5) shell.restore_backupFile(self.couchbase_login_info, self.backup_location, [bucket.name for bucket in self.buckets]) finally: self._check_config() def test_backup_diff_bucket(self): self.shell = RemoteMachineShellConnection(self.master) self.buckets = RestConnection(self.master).get_buckets() self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.backup_location = "/tmp/backup" self.command_options = self.input.param("command_options", '') try: shell = RemoteMachineShellConnection(self.master) self.shell.execute_cluster_backup(self.couchbase_login_info, self.backup_location, self.command_options) time.sleep(5) self._create_bucket(lww=False, name="new_bucket") self.buckets = RestConnection(self.master).get_buckets() shell.restore_backupFile(self.couchbase_login_info, self.backup_location, ["new_bucket"]) finally: self._check_config() ''' Helper functions for above testcases ''' #create a bucket if it doesn't exist. The drift parameter is currently unused def _create_bucket(self, lww=True, drift=False, name=None): if lww: self.lww=lww if name: self.bucket=name helper = RestHelper(self.rest) if not helper.bucket_exists(self.bucket): node_ram_ratio = BucketOperationHelper.base_bucket_ratio( self.servers) info = self.rest.get_nodes_self() self.rest.create_bucket(bucket=self.bucket, ramQuotaMB=512, authType='sasl', lww=self.lww) try: ready = BucketOperationHelper.wait_for_memcached(self.master, self.bucket) except Exception as e: self.fail('unable to create bucket') # KETAKI tochange this def _modify_bucket(self): helper = RestHelper(self.rest) node_ram_ratio = BucketOperationHelper.base_bucket_ratio( self.servers) info = self.rest.get_nodes_self() status, content = self.rest.change_bucket_props(bucket=self.bucket, ramQuotaMB=512, authType='sasl', timeSynchronization='enabledWithOutDrift') if re.search('TimeSyncronization not allowed in update bucket', content): self.log.info('[PASS]Expected modify bucket to disallow Time Synchronization.') else: self.fail('[ERROR] Not expected to allow modify bucket for Time Synchronization') def _restart_server(self, servers): for server in servers: shell = RemoteMachineShellConnection(server) shell.stop_couchbase() time.sleep(10) shell.start_couchbase() shell.disconnect() ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self, wait_if_warmup=True) # REBOOT def _reboot_server(self): try: for server in self.servers[:]: shell = RemoteMachineShellConnection(server) if shell.extract_remote_info().type.lower() == 'windows': o, r = shell.execute_command("shutdown -r -f -t 0") shell.log_command_output(o, r) shell.disconnect() self.log.info("Node {0} is being stopped".format(server.ip)) elif shell.extract_remote_info().type.lower() == 'linux': o, r = shell.execute_command("reboot") shell.log_command_output(o, r) shell.disconnect() self.log.info("Node {0} is being stopped".format(server.ip)) ClusterOperationHelper.wait_for_ns_servers_or_assert([server], self, wait_if_warmup=True) shell = RemoteMachineShellConnection(server) command = "/sbin/iptables -F" o, r = shell.execute_command(command) shell.log_command_output(o, r) shell.disconnect() self.log.info("Node {0} backup".format(server.ip)) finally: self.log.info("Warming-up servers ..") ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self, wait_if_warmup=True) def _check_config(self): rc = self.rest.get_bucket_json(self.bucket) if 'conflictResolution' in rc: conflictResolution = self.rest.get_bucket_json(self.bucket)['conflictResolutionType'] self.assertTrue(conflictResolution == 'lww', 'Expected conflict resolution of lww but got {0}'.format(conflictResolution)) """ drift is disabled in 4.6, commenting out for now as it may come back later
class BaseTestCase(unittest.TestCase, bucket_utils, cluster_utils, failover_utils, node_utils, views_utils): def setUp(self): self.failover_util = failover_utils() self.node_util = node_utils() self.views_util = views_utils() self.log = logger.Logger.get_logger() self.input = TestInputSingleton.input self.primary_index_created = False self.use_sdk_client = self.input.param("use_sdk_client", False) self.analytics = self.input.param("analytics", False) if self.input.param("log_level", None): log.setLevel(level=0) for hd in log.handlers: if str(hd.__class__).find('FileHandler') != -1: hd.setLevel(level=logging.DEBUG) else: hd.setLevel(level=getattr( logging, self.input.param("log_level", None))) self.servers = self.input.servers if str(self.__class__).find('moxitests') != -1: self.moxi_server = self.input.moxis[0] self.servers = [ server for server in self.servers if server.ip != self.moxi_server.ip ] self.buckets = [] self.bucket_base_params = {} self.bucket_base_params['membase'] = {} self.master = self.servers[0] self.bucket_util = bucket_utils(self.master) self.cluster_util = cluster_utils(self.master) self.indexManager = self.servers[0] if not hasattr(self, 'cluster'): self.cluster = Cluster() self.pre_warmup_stats = {} self.cleanup = False self.nonroot = False shell = RemoteMachineShellConnection(self.master) self.os_info = shell.extract_remote_info().type.lower() if self.os_info != 'windows': if self.master.ssh_username != "root": self.nonroot = True shell.disconnect() """ some tests need to bypass checking cb server at set up to run installation """ self.skip_init_check_cbserver = \ self.input.param("skip_init_check_cbserver", False) self.data_collector = DataCollector() self.data_analyzer = DataAnalyzer() self.result_analyzer = DataAnalysisResultAnalyzer() # self.set_testrunner_client() self.change_bucket_properties = False self.cbas_node = self.input.cbas self.cbas_servers = [] self.kv_servers = [] self.otpNodes = [] for server in self.servers: if "cbas" in server.services: self.cbas_servers.append(server) if "kv" in server.services: self.kv_servers.append(server) if not self.cbas_node and len(self.cbas_servers) >= 1: self.cbas_node = self.cbas_servers[0] try: self.skip_setup_cleanup = self.input.param("skip_setup_cleanup", False) self.vbuckets = self.input.param("vbuckets", 1024) self.upr = self.input.param("upr", None) self.index_quota_percent = self.input.param( "index_quota_percent", None) self.targetIndexManager = self.input.param("targetIndexManager", False) self.targetMaster = self.input.param("targetMaster", False) self.reset_services = self.input.param("reset_services", False) self.auth_mech = self.input.param("auth_mech", "PLAIN") self.wait_timeout = self.input.param("wait_timeout", 60) # number of case that is performed from testrunner( increment each time) self.case_number = self.input.param("case_number", 0) self.default_bucket = self.input.param("default_bucket", True) self.parallelism = self.input.param("parallelism", False) if self.default_bucket: self.default_bucket_name = "default" self.standard_buckets = self.input.param("standard_buckets", 0) self.sasl_buckets = self.input.param("sasl_buckets", 0) self.num_buckets = self.input.param("num_buckets", 0) self.verify_unacked_bytes = self.input.param( "verify_unacked_bytes", False) self.memcached_buckets = self.input.param("memcached_buckets", 0) self.enable_flow_control = self.input.param( "enable_flow_control", False) self.total_buckets = self.sasl_buckets + self.default_bucket + self.standard_buckets + self.memcached_buckets self.num_servers = self.input.param("servers", len(self.servers)) # initial number of items in the cluster self.nodes_init = self.input.param("nodes_init", 1) self.nodes_in = self.input.param("nodes_in", 1) self.nodes_out = self.input.param("nodes_out", 1) self.services_init = self.input.param("services_init", None) self.services_in = self.input.param("services_in", None) self.forceEject = self.input.param("forceEject", False) self.force_kill_memcached = TestInputSingleton.input.param( 'force_kill_memcached', False) self.num_items = self.input.param("items", 1000) self.value_size = self.input.param("value_size", 512) self.dgm_run = self.input.param("dgm_run", False) self.active_resident_threshold = int( self.input.param("active_resident_threshold", 0)) # max items number to verify in ValidateDataTask, None - verify all self.max_verify = self.input.param("max_verify", None) # we don't change consistent_view on server by default self.disabled_consistent_view = self.input.param( "disabled_consistent_view", None) self.rebalanceIndexWaitingDisabled = self.input.param( "rebalanceIndexWaitingDisabled", None) self.rebalanceIndexPausingDisabled = self.input.param( "rebalanceIndexPausingDisabled", None) self.maxParallelIndexers = self.input.param( "maxParallelIndexers", None) self.maxParallelReplicaIndexers = self.input.param( "maxParallelReplicaIndexers", None) self.quota_percent = self.input.param("quota_percent", None) self.port = None self.log_message = self.input.param("log_message", None) self.log_info = self.input.param("log_info", None) self.log_location = self.input.param("log_location", None) self.stat_info = self.input.param("stat_info", None) self.port_info = self.input.param("port_info", None) if not hasattr(self, 'skip_buckets_handle'): self.skip_buckets_handle = self.input.param( "skip_buckets_handle", False) self.nodes_out_dist = self.input.param("nodes_out_dist", None) self.absolute_path = self.input.param("absolute_path", True) self.test_timeout = self.input.param( "test_timeout", 3600) # kill hang test and jump to next one. self.enable_bloom_filter = self.input.param( "enable_bloom_filter", False) self.enable_time_sync = self.input.param("enable_time_sync", False) self.gsi_type = self.input.param("gsi_type", 'plasma') # bucket parameters go here, self.bucket_size = self.input.param("bucket_size", None) self.bucket_type = self.input.param("bucket_type", 'membase') self.num_replicas = self.input.param("replicas", 1) self.enable_replica_index = self.input.param("index_replicas", 1) self.eviction_policy = self.input.param( "eviction_policy", 'valueOnly') # or 'fullEviction' # for ephemeral bucket is can be noEviction or nruEviction if self.bucket_type == 'ephemeral' and self.eviction_policy == 'valueOnly': # use the ephemeral bucket default self.eviction_policy = 'noEviction' # for ephemeral buckets it self.sasl_password = self.input.param("sasl_password", 'password') self.lww = self.input.param( "lww", False ) # only applies to LWW but is here because the bucket is created here self.maxttl = self.input.param("maxttl", None) self.compression_mode = self.input.param("compression_mode", 'passive') self.sdk_compression = self.input.param("sdk_compression", True) self.sasl_bucket_name = "bucket" self.sasl_bucket_priority = self.input.param( "sasl_bucket_priority", None) self.standard_bucket_priority = self.input.param( "standard_bucket_priority", None) # end of bucket parameters spot (this is ongoing) if self.skip_setup_cleanup: self.buckets = BucketHelper(self.master).get_buckets() return if not self.skip_init_check_cbserver: self.cb_version = None if RestHelper(RestConnection( self.master)).is_ns_server_running(): """ since every new couchbase version, there will be new features that test code will not work on previous release. So we need to get couchbase version to filter out those tests. """ self.cb_version = RestConnection( self.master).get_nodes_version() else: log.info("couchbase server does not run yet") self.protocol = self.get_protocol_type() self.services_map = None if self.sasl_bucket_priority is not None: self.sasl_bucket_priority = self.sasl_bucket_priority.split( ":") if self.standard_bucket_priority is not None: self.standard_bucket_priority = self.standard_bucket_priority.split( ":") log.info("============== basetestcase setup was started for test #{0} {1}==============" \ .format(self.case_number, self._testMethodName)) if not self.skip_buckets_handle and not self.skip_init_check_cbserver: self._cluster_cleanup() shared_params = self._create_bucket_params( server=self.master, size=self.bucket_size, replicas=self.num_replicas, enable_replica_index=self.enable_replica_index, eviction_policy=self.eviction_policy, bucket_priority=None, lww=self.lww, maxttl=self.maxttl, compression_mode=self.compression_mode) membase_params = copy.deepcopy(shared_params) membase_params['bucket_type'] = 'membase' self.bucket_base_params['membase'][ 'non_ephemeral'] = membase_params membase_ephemeral_params = copy.deepcopy(shared_params) membase_ephemeral_params['bucket_type'] = 'ephemeral' self.bucket_base_params['membase'][ 'ephemeral'] = membase_ephemeral_params memcached_params = copy.deepcopy(shared_params) memcached_params['bucket_type'] = 'memcached' self.bucket_base_params['memcached'] = memcached_params # avoid any cluster operations in setup for new upgrade # & upgradeXDCR tests if str(self.__class__).find('newupgradetests') != -1 or \ str(self.__class__).find('upgradeXDCR') != -1 or \ str(self.__class__).find('Upgrade_EpTests') != -1 or \ hasattr(self, 'skip_buckets_handle') and \ self.skip_buckets_handle: log.info("any cluster operation in setup will be skipped") self.primary_index_created = True log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \ .format(self.case_number, self._testMethodName)) return # avoid clean up if the previous test has been tear down if self.case_number == 1 or self.case_number > 1000: if self.case_number > 1000: log.warn( "teardDown for previous test failed. will retry..") self.case_number -= 1000 self.cleanup = True if not self.skip_init_check_cbserver: self.tearDownEverything() self.cluster = Cluster() if not self.skip_init_check_cbserver: log.info("initializing cluster") self.reset_cluster() master_services = self.get_services(self.servers[:1], \ self.services_init, \ start_node=0) if master_services != None: master_services = master_services[0].split(",") self.quota = self._initialize_nodes(self.cluster, self.servers, \ self.disabled_consistent_view, \ self.rebalanceIndexWaitingDisabled, \ self.rebalanceIndexPausingDisabled, \ self.maxParallelIndexers, \ self.maxParallelReplicaIndexers, \ self.port, \ self.quota_percent, \ services=master_services) self.change_env_variables() self.change_checkpoint_params() # Add built-in user if not self.skip_init_check_cbserver: self.add_built_in_server_user(node=self.master) log.info("done initializing cluster") else: self.quota = "" if self.input.param("log_info", None): self.change_log_info() if self.input.param("log_location", None): self.change_log_location() if self.input.param("stat_info", None): self.change_stat_info() if self.input.param("port_info", None): self.change_port_info() if self.input.param("port", None): self.port = str(self.input.param("port", None)) try: if (str(self.__class__).find('rebalanceout.RebalanceOutTests') != -1) or \ (str(self.__class__).find('memorysanitytests.MemorySanity') != -1) or \ str(self.__class__).find('negativetests.NegativeTests') != -1 or \ str(self.__class__).find('warmuptest.WarmUpTests') != -1 or \ str(self.__class__).find('failover.failovertests.FailoverTests') != -1 or \ str(self.__class__).find('observe.observeseqnotests.ObserveSeqNoTests') != -1 or \ str(self.__class__).find('epengine.lwwepengine.LWW_EP_Engine') != -1: self.services = self.get_services(self.servers, self.services_init) # rebalance all nodes into the cluster before each test self.cluster.rebalance(self.servers[:self.num_servers], self.servers[1:self.num_servers], [], services=self.services) elif self.nodes_init > 1 and not self.skip_init_check_cbserver: self.services = self.get_services( self.servers[:self.nodes_init], self.services_init) self.cluster.rebalance(self.servers[:1], \ self.servers[1:self.nodes_init], \ [], services=self.services) elif str(self.__class__).find('ViewQueryTests') != -1 and \ not self.input.param("skip_rebalance", False): self.services = self.get_services(self.servers, self.services_init) self.cluster.rebalance(self.servers, self.servers[1:], [], services=self.services) self.setDebugLevel(service_type="index") except BaseException, e: # increase case_number to retry tearDown in setup for the next test self.case_number += 1000 self.fail(e) if self.dgm_run: self.quota = 256 if self.total_buckets > 10: log.info("================== changing max buckets from 10 to {0} =================" \ .format(self.total_buckets)) self.change_max_buckets(self, self.total_buckets) if self.total_buckets > 0 and not self.skip_init_check_cbserver: """ from sherlock, we have index service that could take some RAM quota from total RAM quota for couchbase server. We need to get the correct RAM quota available to create bucket(s) after all services were set """ node_info = RestConnection(self.master).get_nodes_self() if node_info.memoryQuota and int(node_info.memoryQuota) > 0: ram_available = node_info.memoryQuota else: ram_available = self.quota if self.bucket_size is None: if self.dgm_run: """ if dgm is set, we need to set bucket size to dgm setting """ self.bucket_size = self.quota else: self.bucket_size = self._get_bucket_size(ram_available, \ self.total_buckets) self.bucket_base_params['membase']['non_ephemeral'][ 'size'] = self.bucket_size self.bucket_base_params['membase']['ephemeral'][ 'size'] = self.bucket_size self.bucket_base_params['memcached']['size'] = self.bucket_size if str(self.__class__).find('upgrade_tests') == -1 and \ str(self.__class__).find('newupgradetests') == -1: self._bucket_creation() log.info("============== basetestcase setup was finished for test #{0} {1} ==============" \ .format(self.case_number, self._testMethodName)) if not self.skip_init_check_cbserver: self._log_start(self) self.sleep(10) except Exception, e: traceback.print_exc() self.cluster.shutdown(force=True) self.fail(e)
class AltAddrBaseTest(BaseTestCase): vbucketId = 0 def setUp(self): self.times_teardown_called = 1 super(AltAddrBaseTest, self).setUp() self.r = random.Random() self.cluster = Cluster() self.clusters_dic = self.input.clusters self.client_os = self.input.param("client_os", "linux") self.alt_addr_with_xdcr = self.input.param("alt_addr_with_xdcr", False) if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] elif len(self.clusters_dic) == 1: self.log.error( "=== need 2 cluster to setup xdcr in ini file ===") if self.alt_addr_with_xdcr: self.des_name = "des_cluster" self.delete_xdcr_reference(self.clusters_dic[0][0].ip, self.clusters_dic[1][0].ip) if self.skip_init_check_cbserver: for key in self.clusters_dic.keys(): servers = self.clusters_dic[key] try: self.backup_reset_clusters(servers) except: self.log.error( "was not able to cleanup cluster the first time" ) self.backup_reset_clusters(servers) else: self.log.error("**** Cluster config is setup in ini file. ****") self.shell = RemoteMachineShellConnection(self.master) if not self.skip_init_check_cbserver: self.rest = RestConnection(self.master) self.cb_version = self.rest.get_nodes_version() self.key_gen = self.input.param("key-gen", True) self.secure_conn = self.input.param("secure-conn", False) self.no_cacert = self.input.param("no-cacert", False) self.no_ssl_verify = self.input.param("no-ssl-verify", False) self.verify_data = self.input.param("verify-data", False) self.debug_logs = self.input.param("debug-logs", False) self.should_fail = self.input.param("should-fail", False) self.add_hostname_node = self.input.param("add_hostname_node", False) self.add_hostname_node_at_src = self.input.param( "add_hostname_node_at_src", False) self.add_hostname_node_at_des = self.input.param( "add_hostname_node_at_des", False) self.num_hostname_add = self.input.param("num_hostname_add", 1) self.alt_addr_services_in = self.input.param("alt_addr_services_in", "kv") self.alt_addr_rebalance_out = self.input.param( "alt_addr_rebalance_out", False) self.alt_addr_rebalance_in = self.input.param("alt_addr_rebalance_in", False) self.alt_addr_rebalance_in_services = self.input.param( "alt_addr_rebalance_in_services", "kv") self.alt_addr_use_public_dns = self.input.param( "alt_addr_use_public_dns", False) self.alt_addr_kv_loader = self.input.param("alt_addr_kv_loader", False) self.alt_addr_n1ql_query = self.input.param("alt_addr_n1ql_query", False) self.alt_addr_eventing_function = self.input.param( "alt_addr_eventing_function", False) self.alt_addr_fts_loader = self.input.param("alt_addr_fts_loader", False) self.run_alt_addr_loader = self.input.param("run_alt_addr_loader", False) self.all_alt_addr_set = False info = self.shell.extract_remote_info() self.os_version = info.distribution_version.lower() self.deliverable_type = info.deliverable_type.lower() type = info.type.lower() self.excluded_commands = self.input.param("excluded_commands", None) self.os = 'linux' self.full_v = None self.short_v = None self.build_number = None cmd = 'curl -g {0}:8091/diag/eval -u {1}:{2} '.format( self.master.ip, self.master.rest_username, self.master.rest_password) cmd += '-d "path_config:component_path(bin)."' bin_path = check_output(cmd, shell=True) bin_path = bin_path.decode() if "bin" not in bin_path: self.fail("Check if cb server install on %s" % self.master.ip) else: self.cli_command_path = bin_path.replace('"', '') + "/" self.root_path = LINUX_ROOT_PATH self.tmp_path = "/tmp/" self.tmp_path_raw = "/tmp/" self.cmd_ext = "" self.src_file = "" self.des_file = "" self.log_path = LINUX_COUCHBASE_LOGS_PATH self.base_cb_path = LINUX_CB_PATH """ non root path """ if self.nonroot: self.log_path = "/home/%s%s" % (self.master.ssh_username, LINUX_COUCHBASE_LOGS_PATH) self.base_cb_path = "/home/%s%s" % (self.master.ssh_username, LINUX_CB_PATH) self.root_path = "/home/%s/" % self.master.ssh_username if type == 'windows': self.os = 'windows' self.cmd_ext = ".exe" self.root_path = WIN_ROOT_PATH self.tmp_path = WIN_TMP_PATH self.tmp_path_raw = WIN_TMP_PATH_RAW win_format = "C:/Program Files" cygwin_format = "/cygdrive/c/Program\ Files" if win_format in self.cli_command_path: self.cli_command_path = self.cli_command_path.replace( win_format, cygwin_format) self.base_cb_path = WIN_CB_PATH if info.distribution_type.lower() == 'mac': self.os = 'mac' self.full_v, self.short_v, self.build_number = self.shell.get_cbversion( type) self.couchbase_usrname = "%s" % ( self.input.membase_settings.rest_username) self.couchbase_password = "******" % ( self.input.membase_settings.rest_password) self.cb_login_info = "%s:%s" % (self.couchbase_usrname, self.couchbase_password) self.path_type = self.input.param("path_type", None) if self.path_type is None: self.log.info("Test command with absolute path ") elif self.path_type == "local": self.log.info("Test command at %s dir " % self.cli_command_path) self.cli_command_path = "cd %s; ./" % self.cli_command_path self.cli_command = self.input.param("cli_command", None) self.start_with_cluster = self.input.param("start_with_cluster", True) if str(self.__class__).find( 'couchbase_clitest.CouchbaseCliTest') == -1: if len(self.servers) > 1 and int( self.nodes_init) == 1 and self.start_with_cluster: servers_in = [ self.servers[i + 1] for i in range(self.num_servers - 1) ] self.cluster.rebalance(self.servers[:1], servers_in, []) for bucket in self.buckets: testuser = [{ 'id': bucket.name, 'name': bucket.name, 'password': '******' }] rolelist = [{ 'id': bucket.name, 'name': bucket.name, 'roles': 'admin' }] self.add_built_in_server_user(testuser=testuser, rolelist=rolelist) def tearDown(self): self.times_teardown_called += 1 serverInfo = self.servers[0] rest = RestConnection(serverInfo) self.clusters_dic = self.input.clusters if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] if self.dest_nodes and len(self.dest_nodes) > 1: self.log.info( "======== clean up destination cluster =======") rest = RestConnection(self.dest_nodes[0]) rest.remove_all_remote_clusters() rest.remove_all_replications() BucketOperationHelper.delete_all_buckets_or_assert( self.dest_nodes, self) ClusterOperationHelper.cleanup_cluster(self.dest_nodes) elif len(self.clusters_dic) == 1: self.log.error( "=== need 2 cluster to setup xdcr in ini file ===") else: self.log.info( "**** If run xdcr test, need cluster config is setup in ini file. ****" ) super(AltAddrBaseTest, self).tearDown() def _list_compare(self, list1, list2): if len(list1) != len(list2): return False for elem1 in list1: found = False for elem2 in list2: if elem1 == elem2: found = True break if not found: return False return True def get_internal_IP(self, server): shell = RemoteMachineShellConnection(server) internal_IP = shell.get_ip_address() internal_IP = [x for x in internal_IP if x != "127.0.0.1"] shell.disconnect() if internal_IP: return internal_IP[0] else: self.fail("Fail to get internal IP") def backup_reset_clusters(self, servers): BucketOperationHelper.delete_all_buckets_or_assert(servers, self) ClusterOperationHelper.cleanup_cluster(servers, master=servers[0]) #ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self) def get_external_IP(self, internal_IP): found = False external_IP = "" for server in self.servers: internalIP = self.get_internal_IP(server) if internal_IP == internalIP: found = True external_IP = server.ip break if not found: self.fail("Could not find server which matches internal IP") else: return external_IP def setup_xdcr_cluster(self): if not self.input.clusters[0] and not self.input.clusters[1]: self.fail("This test needs ini set with cluster config") self.log.info("Create source cluster") self.create_xdcr_cluster(self.input.clusters[0]) self.log.info("Create destination cluster") self.create_xdcr_cluster(self.input.clusters[1]) def create_xdcr_cluster(self, cluster_servers): num_hostname_add = 1 add_host_name = False if self.add_hostname_node_at_src: add_host_name = True if self.add_hostname_node_at_des: add_host_name = True shell = RemoteMachineShellConnection(cluster_servers[0]) services_in = self.alt_addr_services_in if "-" in services_in: set_services = services_in.split("-") else: set_services = services_in.split(",") for server in cluster_servers[1:]: add_node_IP = self.get_internal_IP(server) node_services = "kv" if len(set_services) == 1: node_services = set_services[0] elif len(set_services) > 1: if len(set_services) == len(cluster_servers): node_services = set_services[i] i += 1 if add_host_name and num_hostname_add <= self.num_hostname_add: add_node_IP = server.ip num_hostname_add += 1 try: shell.alt_addr_add_node(main_server=cluster_servers[0], internal_IP=add_node_IP, server_add=server, services=node_services, cmd_ext=self.cmd_ext) except Exception as e: if e: self.fail("Error: {0}".format(e)) rest = RestConnection(cluster_servers[0]) rest.rebalance(otpNodes=[node.id for node in rest.node_statuses()], ejectedNodes=[]) rest.monitorRebalance() def create_xdcr_reference(self, src_IP, des_IP): cmd = "curl -u Administrator:password " cmd += "http://{0}:8091/pools/default/remoteClusters ".format(src_IP) cmd += "-d username=Administrator -d password=password " cmd += "-d name={0} -d demandEncryption=0 ".format(self.des_name) cmd += "-d hostname={0}:8091 ".format(des_IP) mesg = "\n **** Create XDCR cluster remote reference from cluster {0} ".format( src_IP) mesg += "to cluster {0}".format(des_IP) self.log.info(mesg) print("command to run: {0}".format(cmd)) try: output = check_output(cmd, shell=True, stderr=STDOUT) except CalledProcessError as e: if e.output: self.fail("\n Error: ".format(e.output)) def delete_xdcr_reference(self, src_IP, des_IP): cmd = "curl -X DELETE -u Administrator:password " cmd += "http://{0}:8091/pools/default/remoteClusters/{1}".format( src_IP, self.des_name) print("command to run: {0}".format(cmd)) try: output = check_output(cmd, shell=True, stderr=STDOUT) except CalledProcessError as e: if e.output: self.fail("Error: ".format(e.output)) def create_xdcr_replication(self, src_IP, des_IP, bucket_name): cmd = "curl -X POST -u Administrator:password " cmd += "http://{0}:8091/controller/createReplication ".format(src_IP) cmd += "-d fromBucket={0} ".format(bucket_name) cmd += "-d toCluster={0} ".format(self.des_name) cmd += "-d toBucket={0} ".format(bucket_name) cmd += "-d replicationType=continuous -d enableCompression=1 " print("command to run: {0}".format(cmd)) try: output = check_output(cmd, shell=True, stderr=STDOUT) return output except CalledProcessError as e: if e.output: self.fail("Error: ".format(e.output)) def delete_xdcr_replication(self, src_IP, replication_id): replication_id = urllib.quote(replication_id, safe='') cmd = "curl -X DELETE -u Administrator:password " cmd += " http://{0}:8091/controller/cancelXDCR/{1} ".format( src_IP, replication_id) print("command to run: {0}".format(cmd)) try: output = check_output(cmd, shell=True, stderr=STDOUT) except CalledProcessError as e: if e.output: self.fail("Error: ".format(e.output)) def set_xdcr_checkpoint(self, src_IP, check_time): cmd = "curl -u Administrator:password " cmd += "http://{0}:8091/settings/replications ".format(src_IP) cmd += "-d goMaxProcs=10 " cmd += "-d checkpointInterval={0} ".format(check_time) print("command to run: {0}".format(cmd)) try: self.log.info( "Set xdcr checkpoint to {0} seconds".format(check_time)) output = check_output(cmd, shell=True, stderr=STDOUT) except CalledProcessError as e: if e.output: self.fail("Error: ".format(e.output)) def waitForItemCount(self, server, bucket_name, count, timeout=30): rest = RestConnection(server) for sec in range(timeout): items = int( rest.get_bucket_json(bucket_name)["basicStats"]["itemCount"]) if items != count: time.sleep(1) else: return True log.info("Waiting for item count to be %d timed out", count) return False def _check_output(self, word_check, output): found = False if len(output) >= 1: if isinstance(word_check, list): for ele in word_check: for x in output: if ele.lower() in x.lower(): self.log.info( "Found '{0} in CLI output".format(ele)) found = True break elif isinstance(word_check, str): for x in output: if word_check.lower() in x.lower(): self.log.info( "Found '{0}' in CLI output".format(word_check)) found = True break else: self.log.error("invalid {0}".format(word_check)) return found
class CliBaseTest(BaseTestCase): vbucketId = 0 def setUp(self): self.times_teardown_called = 1 super(CliBaseTest, self).setUp() self.r = random.Random() self.vbucket_count = 1024 self.cluster = Cluster() self.clusters_dic = self.input.clusters if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] elif len(self.clusters_dic) == 1: self.log.error("=== need 2 cluster to setup xdcr in ini file ===") else: self.log.error("**** Cluster config is setup in ini file. ****") self.shell = RemoteMachineShellConnection(self.master) if not self.skip_init_check_cbserver: self.rest = RestConnection(self.master) self.cb_version = self.rest.get_nodes_version() """ cli output message """ self.cli_bucket_create_msg = "SUCCESS: Bucket created" self.cli_rebalance_msg = "SUCCESS: Rebalance complete" if self.cb_version[:3] == "4.6": self.cli_bucket_create_msg = "SUCCESS: bucket-create" self.cli_rebalance_msg = "SUCCESS: rebalanced cluster" self.import_back = self.input.param("import_back", False) if self.import_back: if len(self.servers) < 3: self.fail("This test needs minimum of 3 vms to run ") self.test_type = self.input.param("test_type", "import") self.import_file = self.input.param("import_file", None) self.imex_type = self.input.param("imex_type", "json") self.format_type = self.input.param("format_type", "lines") self.import_method = self.input.param("import_method", "file://") self.force_failover = self.input.param("force_failover", False) self.json_invalid_errors = self.input.param("json-invalid-errors", None) self.field_separator = self.input.param("field-separator", "comma") self.key_gen = self.input.param("key-gen", True) self.skip_docs = self.input.param("skip-docs", None) self.limit_docs = self.input.param("limit-docs", None) self.limit_rows = self.input.param("limit-rows", None) self.skip_rows = self.input.param("skip-rows", None) self.omit_empty = self.input.param("omit-empty", None) self.infer_types = self.input.param("infer-types", None) self.fx_generator = self.input.param("fx-generator", None) self.fx_gen_start = self.input.param("fx-gen-start", None) self.secure_conn = self.input.param("secure-conn", False) self.no_cacert = self.input.param("no-cacert", False) self.no_ssl_verify = self.input.param("no-ssl-verify", False) self.verify_data = self.input.param("verify-data", False) self.field_substitutions = self.input.param("field-substitutions", None) self.check_preload_keys = self.input.param("check-preload-keys", True) self.debug_logs = self.input.param("debug-logs", False) self.should_fail = self.input.param("should-fail", False) info = self.shell.extract_remote_info() self.os_version = info.distribution_version.lower() type = info.type.lower() self.excluded_commands = self.input.param("excluded_commands", None) self.os = 'linux' self.full_v = None self.short_v = None self.build_number = None cmd = 'curl -g %s:8091/diag/eval -u Administrator:password ' % self.master.ip cmd += '-d "path_config:component_path(bin)."' bin_path = subprocess.check_output(cmd, shell=True) if "bin" not in bin_path: self.fail("Check if cb server install on %s" % self.master.ip) else: self.cli_command_path = bin_path.replace('"','') + "/" self.root_path = LINUX_ROOT_PATH self.tmp_path = "/tmp/" self.tmp_path_raw = "/tmp/" self.cmd_backup_path = LINUX_BACKUP_PATH self.backup_path = LINUX_BACKUP_PATH self.cmd_ext = "" self.src_file = "" self.des_file = "" self.sample_files_path = LINUX_COUCHBASE_SAMPLE_PATH self.log_path = LINUX_COUCHBASE_LOGS_PATH self.base_cb_path = LINUX_CB_PATH """ non root path """ if self.nonroot: self.sample_files_path = "/home/%s%s" % (self.master.ssh_username, LINUX_COUCHBASE_SAMPLE_PATH) self.log_path = "/home/%s%s" % (self.master.ssh_username, LINUX_COUCHBASE_LOGS_PATH) self.base_cb_path = "/home/%s%s" % (self.master.ssh_username, LINUX_CB_PATH) self.root_path = "/home/%s/" % self.master.ssh_username if type == 'windows': self.os = 'windows' self.cmd_ext = ".exe" self.root_path = WIN_ROOT_PATH self.tmp_path = WIN_TMP_PATH self.tmp_path_raw = WIN_TMP_PATH_RAW self.cmd_backup_path = WIN_BACKUP_C_PATH self.backup_path = WIN_BACKUP_PATH self.sample_files_path = WIN_COUCHBASE_SAMPLE_PATH_C self.log_path = WIN_COUCHBASE_LOGS_PATH win_format = "C:/Program Files" cygwin_format = "/cygdrive/c/Program\ Files" if win_format in self.cli_command_path: self.cli_command_path = self.cli_command_path.replace(win_format, cygwin_format) if info.distribution_type.lower() == 'mac': self.os = 'mac' self.full_v, self.short_v, self.build_number = self.shell.get_cbversion(type) self.couchbase_usrname = "%s" % (self.input.membase_settings.rest_username) self.couchbase_password = "******" % (self.input.membase_settings.rest_password) self.cb_login_info = "%s:%s" % (self.couchbase_usrname, self.couchbase_password) self.path_type = self.input.param("path_type", None) if self.path_type is None: self.log.info("Test command with absolute path ") elif self.path_type == "local": self.log.info("Test command at %s dir " % self.cli_command_path) self.cli_command_path = "cd %s; ./" % self.cli_command_path self.cli_command = self.input.param("cli_command", None) self.command_options = self.input.param("command_options", None) if self.command_options is not None: self.command_options = self.command_options.split(";") if str(self.__class__).find('couchbase_clitest.CouchbaseCliTest') == -1: if len(self.servers) > 1 and int(self.nodes_init) == 1: servers_in = [self.servers[i + 1] for i in range(self.num_servers - 1)] self.cluster.rebalance(self.servers[:1], servers_in, []) for bucket in self.buckets: testuser = [{'id': bucket.name, 'name': bucket.name, 'password': '******'}] rolelist = [{'id': bucket.name, 'name': bucket.name, 'roles': 'admin'}] self.add_built_in_server_user(testuser=testuser, rolelist=rolelist) def tearDown(self): if not self.input.param("skip_cleanup", True): if self.times_teardown_called > 1 : self.shell.disconnect() if self.input.param("skip_cleanup", True): if self.case_number > 1 or self.times_teardown_called > 1: self.shell.disconnect() self.times_teardown_called += 1 serverInfo = self.servers[0] rest = RestConnection(serverInfo) zones = rest.get_zone_names() for zone in zones: if zone != "Group 1": rest.delete_zone(zone) self.clusters_dic = self.input.clusters if self.clusters_dic: if len(self.clusters_dic) > 1: self.dest_nodes = self.clusters_dic[1] self.dest_master = self.dest_nodes[0] if self.dest_nodes and len(self.dest_nodes) > 1: self.log.info("======== clean up destination cluster =======") rest = RestConnection(self.dest_nodes[0]) rest.remove_all_remote_clusters() rest.remove_all_replications() BucketOperationHelper.delete_all_buckets_or_assert(self.dest_nodes, self) ClusterOperationHelper.cleanup_cluster(self.dest_nodes) elif len(self.clusters_dic) == 1: self.log.error("=== need 2 cluster to setup xdcr in ini file ===") else: self.log.info("**** If run xdcr test, need cluster config is setup in ini file. ****") super(CliBaseTest, self).tearDown() """ in sherlock, there is an extra value called runCmd in the 1st element """ def del_runCmd_value(self, output): if "runCmd" in output[0]: output = output[1:] return output def verifyCommandOutput(self, output, expect_error, message): """Inspects each line of the output and checks to see if the expected error was found Options: output - A list of output lines expect_error - Whether or not the command should have succeeded or failed message - The success or error message Returns a boolean indicating whether or not the error/success message was found in the output """ if expect_error: for line in output: if line == "ERROR: " + message: return True log.info("Did not receive expected error message `ERROR: %s`", message) return False else: for line in output: if line == "SUCCESS: " + message: return True log.info("Did not receive expected success message `SUCCESS: %s`", message) return False def verifyWarningOutput(self, output, message): for line in output: if line == "WARNING: " + message: return True log.info("Did not receive expected error message `WARNING: %s`", message) return False def verifyServices(self, server, expected_services): """Verifies that the services on a given node match the expected service Options: server - A TestInputServer object of the server to connect to expected_services - A comma separated list of services Returns a boolean corresponding to whether or not the expected services are available on the server. """ rest = RestConnection(server) hostname = "%s:%s" % (server.ip, server.port) expected_services = expected_services.replace("data", "kv") expected_services = expected_services.replace("query", "n1ql") expected_services = expected_services.split(",") nodes_services = rest.get_nodes_services() for node, services in nodes_services.iteritems(): if node.encode('ascii') == hostname: if len(services) != len(expected_services): log.info("Services on %s do not match expected services (%s vs. %s)", hostname, services, expected_services) return False for service in services: if service.encode("ascii") not in expected_services: log.info("Services on %s do not match expected services (%s vs. %s)", hostname, services, expected_services) return False return True log.info("Services on %s not found, the server may not exist", hostname) return False def verifyRamQuotas(self, server, data, index, fts): """Verifies that the RAM quotas for each service are set properly Options: server - A TestInputServer object of the server to connect to data - An int containing the data service RAM quota, None will skip the check index - An int containing the index service RAM quota, None will skip the check fts - An int containing the FTS service RAM quota, None will skip the check Returns a boolean corresponding to whether or not the RAM quotas were set properly """ rest = RestConnection(server) settings = rest.get_pools_default() if data: if "memoryQuota" not in settings: log.info("Unable to get data service ram quota") return False if int(settings["memoryQuota"]) != int(data): log.info("Data service memory quota does not match (%d vs %d)", int(settings["memoryQuota"]), int(data)) return False if index: if "indexMemoryQuota" not in settings: log.info("Unable to get index service ram quota") return False if int(settings["indexMemoryQuota"]) != int(index): log.info( "Index service memory quota does not match (%d vs %d)", int(settings["indexMemoryQuota"]), int(index)) return False if fts: if "ftsMemoryQuota" not in settings: log.info("Unable to get fts service ram quota") return False if int(settings["ftsMemoryQuota"]) != int(fts): log.info("FTS service memory quota does not match (%d vs %d)", int(settings["ftsMemoryQuota"]), int(fts)) return False return True def verifyBucketSettings(self, server, bucket_name, bucket_type, memory_quota, eviction_policy, replica_count, enable_index_replica, priority, enable_flush): rest = RestConnection(server) result = rest.get_bucket_json(bucket_name) if bucket_type == "couchbase": bucket_type = "membase" if bucket_type is not None and bucket_type != result["bucketType"]: log.info("Memory quota does not match (%s vs %s)", bucket_type, result["bucketType"]) return False quota = result["quota"]["rawRAM"] / 1024 / 1024 if memory_quota is not None and memory_quota != quota: log.info("Bucket quota does not match (%s vs %s)", memory_quota, quota) return False if eviction_policy is not None and eviction_policy != result[ "evictionPolicy"]: log.info("Eviction policy does not match (%s vs %s)", eviction_policy, result["evictionPolicy"]) return False if replica_count is not None and replica_count != result[ "replicaNumber"]: log.info("Replica count does not match (%s vs %s)", replica_count, result["replicaNumber"]) return False if enable_index_replica == 1: enable_index_replica = True elif enable_index_replica == 0: enable_index_replica = False if enable_index_replica is not None and enable_index_replica != result[ "replicaIndex"]: log.info("Replica index enabled does not match (%s vs %s)", enable_index_replica, result["replicaIndex"]) return False if priority == "high": priority = 8 elif priority == "low": priority = 3 if priority is not None and priority != result["threadsNumber"]: log.info("Bucket priority does not match (%s vs %s)", priority, result["threadsNumber"]) return False if enable_flush is not None: if enable_flush == 1 and "flush" not in result["controllers"]: log.info("Bucket flush is not enabled, but it should be") return False elif enable_flush == 0 and "flush" in result["controllers"]: log.info("Bucket flush is not enabled, but it should be") return False return True def verifyContainsBucket(self, server, name): rest = RestConnection(server) buckets = rest.get_buckets() for bucket in buckets: if bucket.name == name: return True return False def verifyClusterName(self, server, name): rest = RestConnection(server) settings = rest.get_pools_default("waitChange=0") if name is None: name = "" if "clusterName" not in settings: log.info("Unable to get cluster name from server") return False if settings["clusterName"] != name: log.info("Cluster name does not match (%s vs %s)", settings["clusterName"], name) return False return True def isClusterInitialized(self, server): """Checks whether or not the server is initialized Options: server - A TestInputServer object of the server to connect to Checks to see whether or not the default pool was created in order to determine whether or no the server was initialized. Returns a boolean value to indicate initialization. """ rest = RestConnection(server) settings = rest.get_pools_info() if "pools" in settings and len(settings["pools"]) > 0: return True return False def verifyNotificationsEnabled(self, server): rest = RestConnection(server) enabled = rest.get_notifications() if enabled: return True return False def verifyIndexSettings(self, server, max_rollbacks, stable_snap_interval, mem_snap_interval, storage_mode, threads, log_level): rest = RestConnection(server) settings = rest.get_global_index_settings() if storage_mode == "default": storage_mode = "plasma" elif storage_mode == "memopt": storage_mode = "memory_optimized" if max_rollbacks and str(settings["maxRollbackPoints"]) != str( max_rollbacks): log.info("Max rollbacks does not match (%s vs. %s)", str(settings["maxRollbackPoints"]), str(max_rollbacks)) return False if stable_snap_interval and str( settings["stableSnapshotInterval"]) != str( stable_snap_interval): log.info("Stable snapshot interval does not match (%s vs. %s)", str(settings["stableSnapshotInterval"]), str(stable_snap_interval)) return False if mem_snap_interval and str( settings["memorySnapshotInterval"]) != str(mem_snap_interval): log.info("Memory snapshot interval does not match (%s vs. %s)", str(settings["memorySnapshotInterval"]), str(mem_snap_interval)) return False if storage_mode and str(settings["storageMode"]) != str(storage_mode): log.info("Storage mode does not match (%s vs. %s)", str(settings["storageMode"]), str(storage_mode)) return False if threads and str(settings["indexerThreads"]) != str(threads): log.info("Threads does not match (%s vs. %s)", str(settings["indexerThreads"]), str(threads)) return False if log_level and str(settings["logLevel"]) != str(log_level): log.info("Log level does not match (%s vs. %s)", str(settings["logLevel"]), str(log_level)) return False return True def verifyAutofailoverSettings(self, server, enabled, timeout): rest = RestConnection(server) settings = rest.get_autofailover_settings() if enabled and not ((str(enabled) == "1" and settings.enabled) or ( str(enabled) == "0" and not settings.enabled)): log.info("Enabled does not match (%s vs. %s)", str(enabled), str(settings.enabled)) return False if timeout and str(settings.timeout) != str(timeout): log.info("Timeout does not match (%s vs. %s)", str(timeout), str(settings.timeout)) return False return True def verifyAutoreprovisionSettings(self, server, enabled, max_nodes): rest = RestConnection(server) settings = rest.get_autoreprovision_settings() if enabled and not ((str(enabled) == "1" and settings.enabled) or ( str(enabled) == "0" and not settings.enabled)): log.info("Enabled does not match (%s vs. %s)", str(max_nodes), str(settings.enabled)) return False if max_nodes and str(settings.max_nodes) != str(max_nodes): log.info("max_nodes does not match (%s vs. %s)", str(max_nodes), str(settings.max_nodes)) return False return True def verifyAuditSettings(self, server, enabled, log_path, rotate_interval): rest = RestConnection(server) settings = rest.getAuditSettings() if enabled and not ( (str(enabled) == "1" and settings["auditdEnabled"]) or ( str(enabled) == "0" and not settings["auditdEnabled"])): log.info("Enabled does not match (%s vs. %s)", str(enabled), str(settings["auditdEnabled"])) return False if log_path and str(str(settings["logPath"])) != str(log_path): log.info("Log path does not match (%s vs. %s)", str(log_path), str(settings["logPath"])) return False if rotate_interval and str(str(settings["rotateInterval"])) != str( rotate_interval): log.info("Rotate interval does not match (%s vs. %s)", str(rotate_interval), str(settings["rotateInterval"])) return False return True def verifyPendingServer(self, server, server_to_add, group_name, services): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False expected_services = services.replace("data", "kv") expected_services = expected_services.replace("query", "n1ql") expected_services = expected_services.split(",") for group in settings["groups"]: for node in group["nodes"]: if node["hostname"] == server_to_add: if node["clusterMembership"] != "inactiveAdded": log.info("Node `%s` not in pending status", server_to_add) return False if group["name"] != group_name: log.info("Node `%s` not in correct group (%s vs %s)", node["hostname"], group_name, group["name"]) return False if len(node["services"]) != len(expected_services): log.info("Services do not match on %s (%s vs %s) ", node["hostname"], services, ",".join(node["services"])) return False for service in node["services"]: if service not in expected_services: log.info("Services do not match on %s (%s vs %s) ", node["hostname"], services, ",".join(node["services"])) return False return True log.info("Node `%s` not found in nodes list", server_to_add) return False def verifyPendingServerDoesNotExist(self, server, server_to_add): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False for group in settings["groups"]: for node in group["nodes"]: if node["hostname"] == server_to_add: return False log.info("Node `%s` not found in nodes list", server_to_add) return True def verifyActiveServers(self, server, expected_num_servers): return self._verifyServersByStatus(server, expected_num_servers, "active") def verifyFailedServers(self, server, expected_num_servers): return self._verifyServersByStatus(server, expected_num_servers, "inactiveFailed") def _verifyServersByStatus(self, server, expected_num_servers, status): rest = RestConnection(server) settings = rest.get_pools_default() count = 0 for node in settings["nodes"]: if node["clusterMembership"] == status: count += 1 return count == expected_num_servers def verifyRecoveryType(self, server, recovery_servers, recovery_type): rest = RestConnection(server) settings = rest.get_all_zones_info() if not settings or "groups" not in settings: log.info("Group settings payload appears to be invalid") return False if not recovery_servers: return True num_found = 0 recovery_servers = recovery_servers.split(",") for group in settings["groups"]: for node in group["nodes"]: for rs in recovery_servers: if node["hostname"] == rs: if node["recoveryType"] != recovery_type: log.info( "Node %s doesn't contain recovery type %s ", rs, recovery_type) return False else: num_found = num_found + 1 if num_found == len(recovery_servers): return True log.info("Node `%s` not found in nodes list", ",".join(recovery_servers)) return False def verifyUserRoles(self, server, username, roles): rest = RestConnection(server) status, content, header = rbacmain(server)._retrieve_user_roles() content = json.loads(content) temp = rbacmain()._parse_get_user_response(content, username, username, roles) return temp def verifyLdapSettings(self, server, admins, ro_admins, default, enabled): rest = RestConnection(server) settings = rest.ldapRestOperationGetResponse() if admins is None: admins = [] else: admins = admins.split(",") if ro_admins is None: ro_admins = [] else: ro_admins = ro_admins.split(",") if str(enabled) == "0": admins = [] ro_admins = [] if default == "admins" and str(enabled) == "1": if settings["admins"] != "asterisk": log.info("Admins don't match (%s vs asterisk)", settings["admins"]) return False elif not self._list_compare(settings["admins"], admins): log.info("Admins don't match (%s vs %s)", settings["admins"], admins) return False if default == "roadmins" and str(enabled) == "1": if settings["roAdmins"] != "asterisk": log.info("Read only admins don't match (%s vs asterisk)", settings["roAdmins"]) return False elif not self._list_compare(settings["roAdmins"], ro_admins): log.info("Read only admins don't match (%s vs %s)", settings["roAdmins"], ro_admins) return False return True def verifyAlertSettings(self, server, enabled, email_recipients, email_sender, email_username, email_password, email_host, email_port, encrypted, alert_af_node, alert_af_max_reached, alert_af_node_down, alert_af_small, alert_af_disable, alert_ip_changed, alert_disk_space, alert_meta_overhead, alert_meta_oom, alert_write_failed, alert_audit_dropped): rest = RestConnection(server) settings = rest.get_alerts_settings() print settings if not enabled: if not settings["enabled"]: return True else: log.info("Alerts should be disabled") return False if encrypted is None or encrypted == "0": encrypted = False else: encrypted = True if email_recipients is not None and not self._list_compare( email_recipients.split(","), settings["recipients"]): log.info("Email recipients don't match (%s vs %s)", email_recipients.split(","), settings["recipients"]) return False if email_sender is not None and email_sender != settings["sender"]: log.info("Email sender does not match (%s vs %s)", email_sender, settings["sender"]) return False if email_username is not None and email_username != \ settings["emailServer"]["user"]: log.info("Email username does not match (%s vs %s)", email_username, settings["emailServer"]["user"]) return False if email_host is not None and email_host != settings["emailServer"][ "host"]: log.info("Email host does not match (%s vs %s)", email_host, settings["emailServer"]["host"]) return False if email_port is not None and email_port != settings["emailServer"][ "port"]: log.info("Email port does not match (%s vs %s)", email_port, settings["emailServer"]["port"]) return False if encrypted is not None and encrypted != settings["emailServer"][ "encrypt"]: log.info("Email encryption does not match (%s vs %s)", encrypted, settings["emailServer"]["encrypt"]) return False alerts = list() if alert_af_node: alerts.append('auto_failover_node') if alert_af_max_reached: alerts.append('auto_failover_maximum_reached') if alert_af_node_down: alerts.append('auto_failover_other_nodes_down') if alert_af_small: alerts.append('auto_failover_cluster_too_small') if alert_af_disable: alerts.append('auto_failover_disabled') if alert_ip_changed: alerts.append('ip') if alert_disk_space: alerts.append('disk') if alert_meta_overhead: alerts.append('overhead') if alert_meta_oom: alerts.append('ep_oom_errors') if alert_write_failed: alerts.append('ep_item_commit_failed') if alert_audit_dropped: alerts.append('audit_dropped_events') if not self._list_compare(alerts, settings["alerts"]): log.info("Alerts don't match (%s vs %s)", alerts, settings["alerts"]) return False return True def verify_node_settings(self, server, data_path, index_path, hostname): rest = RestConnection(server) node_settings = rest.get_nodes_self() if data_path != node_settings.storage[0].path: log.info("Data path does not match (%s vs %s)", data_path, node_settings.storage[0].path) return False if index_path != node_settings.storage[0].index_path: log.info("Index path does not match (%s vs %s)", index_path, node_settings.storage[0].index_path) return False if hostname is not None: if hostname != node_settings.hostname: log.info("Hostname does not match (%s vs %s)", hostname, node_settings.hostname) return True return True def verifyCompactionSettings(self, server, db_frag_perc, db_frag_size, view_frag_perc, view_frag_size, from_period, to_period, abort_outside, parallel_compact, purgeInt): rest = RestConnection(server) settings = rest.get_auto_compaction_settings() ac = settings["autoCompactionSettings"] if db_frag_perc is not None and str(db_frag_perc) != str( ac["databaseFragmentationThreshold"]["percentage"]): log.info("DB frag perc does not match (%s vs %s)", str(db_frag_perc), str(ac["databaseFragmentationThreshold"]["percentage"])) return False if db_frag_size is not None and str(db_frag_size * 1024 ** 2) != str( ac["databaseFragmentationThreshold"]["size"]): log.info("DB frag size does not match (%s vs %s)", str(db_frag_size * 1024 ** 2), str(ac["databaseFragmentationThreshold"]["size"])) return False if view_frag_perc is not None and str(view_frag_perc) != str( ac["viewFragmentationThreshold"]["percentage"]): log.info("View frag perc does not match (%s vs %s)", str(view_frag_perc), str(ac["viewFragmentationThreshold"]["percentage"])) return False if view_frag_size is not None and str( view_frag_size * 1024 ** 2) != str( ac["viewFragmentationThreshold"]["size"]): log.info("View frag size does not match (%s vs %s)", str(view_frag_size * 1024 ** 2), str(ac["viewFragmentationThreshold"]["size"])) return False print from_period, to_period if from_period is not None: fromHour, fromMin = from_period.split(":", 1) if int(fromHour) != int(ac["allowedTimePeriod"]["fromHour"]): log.info("From hour does not match (%s vs %s)", str(fromHour), str(ac["allowedTimePeriod"]["fromHour"])) return False if int(fromMin) != int(ac["allowedTimePeriod"]["fromMinute"]): log.info("From minute does not match (%s vs %s)", str(fromMin), str(ac["allowedTimePeriod"]["fromMinute"])) return False if to_period is not None: toHour, toMin = to_period.split(":", 1) if int(toHour) != int(ac["allowedTimePeriod"]["toHour"]): log.info("To hour does not match (%s vs %s)", str(toHour), str(ac["allowedTimePeriod"]["toHour"])) return False if int(toMin) != int(ac["allowedTimePeriod"]["toMinute"]): log.info("To minute does not match (%s vs %s)", str(toMin), str(ac["allowedTimePeriod"]["toMinute"])) return False if str(abort_outside) == "1": abort_outside = True elif str(abort_outside) == "0": abort_outside = False if abort_outside is not None and abort_outside != \ ac["allowedTimePeriod"]["abortOutside"]: log.info("Abort outside does not match (%s vs %s)", abort_outside, ac["allowedTimePeriod"]["abortOutside"]) return False if str(parallel_compact) == "1": parallel_compact = True elif str(parallel_compact) == "0": parallel_compact = False if parallel_compact is not None and parallel_compact != ac[ "parallelDBAndViewCompaction"]: log.info("Parallel compact does not match (%s vs %s)", str(parallel_compact), str(ac["parallelDBAndViewCompaction"])) return False if purgeInt is not None and str(purgeInt) != str( settings["purgeInterval"]): log.info("Purge interval does not match (%s vs %s)", str(purgeInt), str(settings["purgeInterval"])) return False return True def verify_gsi_compact_settings(self, compact_mode, compact_percent, compact_interval, from_period, to_period, enable_abort): rest = RestConnection(self.master) settings = rest.get_auto_compaction_settings() ac = settings["autoCompactionSettings"]["indexFragmentationThreshold"] cc = settings["autoCompactionSettings"]["indexCircularCompaction"] if compact_mode is not None: if compact_mode == "append": self.log.info("append compactino settings %s " % ac) if compact_percent is not None and \ compact_percent != ac["percentage"]: raise Exception( "setting percent does not match. Set: %s vs %s :Actual" % (compact_percent, ac["percentage"])) if compact_mode == "circular": self.log.info("circular compaction settings %s " % cc) if enable_abort and not cc["interval"]["abortOutside"]: raise Exception("setting enable abort failed") if compact_interval is not None: if compact_interval != cc["daysOfWeek"]: raise Exception( "Failed to set compaction on %s " % compact_interval) elif from_period is None and int( cc["interval"]["fromHour"]) != 0 and \ int(cc["interval"]["fromMinute"]) != 0: raise Exception( "fromHour and fromMinute should be zero") if compact_interval is None: if (from_period != str(cc["interval"][ "fromHour"]) + ":" + str( cc["interval"]["fromMinute"])) \ and (to_period != str(cc["interval"]["toHour"]) + ":" + str( cc["interval"]["toMinute"])): raise Exception( "fromHour and fromMinute do not set correctly") return True def verifyGroupExists(self, server, name): rest = RestConnection(server) groups = rest.get_zone_names() print groups for gname, _ in groups.iteritems(): if name == gname: return True return False def _list_compare(self, list1, list2): if len(list1) != len(list2): return False for elem1 in list1: found = False for elem2 in list2: if elem1 == elem2: found = True break if not found: return False return True def waitForItemCount(self, server, bucket_name, count, timeout=30): rest = RestConnection(server) for sec in range(timeout): items = int( rest.get_bucket_json(bucket_name)["basicStats"]["itemCount"]) if items != count: time.sleep(1) else: return True log.info("Waiting for item count to be %d timed out", count) return False