def common_setup(input, testcase):
log.info("============== common_setup was started for test #{0} {1}==============" \
.format(testcase.case_number, testcase._testMethodName))
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
# Add built-in user
testuser = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'password': '******'
}]
RbacBase().create_user_source(testuser, 'builtin', servers[0])
# Assign user to role
role_list = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'roles': 'admin'
}]
RbacBase().add_user_role(role_list, RestConnection(servers[0]),
'builtin')
log.info("============== common_setup was finished for test #{0} {1} ==============" \
.format(testcase.case_number, testcase._testMethodName))
def common_setup(input, testcase):
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
for server in servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
def tearDown(self):
try:
self._cluster_helper.shutdown()
log = logger.Logger.get_logger()
log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
for server in self._cleanup_nodes:
shell = RemoteMachineShellConnection(server)
o, r = shell.execute_command("iptables -F")
shell.log_command_output(o, r)
o, r = shell.execute_command(
"/sbin/iptables -A INPUT -p tcp -i eth0 --dport 1000:60000 -j ACCEPT"
)
shell.log_command_output(o, r)
o, r = shell.execute_command(
"/sbin/iptables -A OUTPUT -p tcp -o eth0 --dport 1000:60000 -j ACCEPT"
)
shell.log_command_output(o, r)
o, r = shell.execute_command(
"/etc/init.d/couchbase-server start")
shell.log_command_output(o, r)
shell.disconnect()
BucketOperationHelper.delete_all_buckets_or_assert(
self._servers, self)
ClusterOperationHelper.cleanup_cluster(self._servers)
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
pass
def common_setup(input, testcase):
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
for server in servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
def common_tearDown(servers, testcase):
log = logger.Logger.get_logger()
log.info(
"============== common_tearDown was started for test #{0} {1} ==============".format(
testcase.case_number, testcase._testMethodName
)
)
RemoteUtilHelper.common_basic_setup(servers)
log.info("10 seconds delay to wait for couchbase-server to start")
time.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(
servers, testcase, wait_time=AutoFailoverBaseTest.MAX_FAIL_DETECT_TIME * 15, wait_if_warmup=True
)
try:
rest = RestConnection(self._servers[0])
buckets = rest.get_buckets()
for bucket in buckets:
MemcachedClientHelper.flush_bucket(servers[0], bucket.name)
except Exception:
pass
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
log.info(
"============== common_tearDown was finished for test #{0} {1} ==============".format(
testcase.case_number, testcase._testMethodName
)
)
def tearDown(self):
try:
self._cluster_helper.shutdown()
log = logger.Logger.get_logger()
log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
for server in self._cleanup_nodes:
shell = RemoteMachineShellConnection(server)
o, r = shell.execute_command("iptables -F")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A INPUT -p tcp -i eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A OUTPUT -p tcp -o eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/etc/init.d/couchbase-server start")
shell.log_command_output(o, r)
shell.disconnect()
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
ClusterOperationHelper.cleanup_cluster(self._servers)
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
pass
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params and \
str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
#supported starting with python2.7
log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown()
self._log_finish(self)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.servers)
self.log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
for server in self._cleanup_nodes:
shell = RemoteMachineShellConnection(server)
o, r = shell.execute_command("iptables -F")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A INPUT -p tcp -i eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A OUTPUT -p tcp -o eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/etc/init.d/couchbase-server start")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
super(FailoverBaseTest, self).tearDown()
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params and \
str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
#supported starting with python2.7
log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown()
self._log_finish(self)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.servers)
self.log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
for server in self._cleanup_nodes:
shell = RemoteMachineShellConnection(server)
o, r = shell.execute_command("iptables -F")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A INPUT -p tcp -i eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A OUTPUT -p tcp -o eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/etc/init.d/couchbase-server start")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
super(FailoverBaseTest, self).tearDown()
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params and \
str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
# supported starting with python2.7
log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown(force=True)
self._log_finish(self)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.servers)
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for node in self.servers:
master = node
try:
ClusterOperationHelper.cleanup_cluster(self.servers,
master=master)
except:
continue
self.log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
super(FailoverBaseTest, self).tearDown()
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params and \
str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
# supported starting with python2.7
log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown(force=True)
self._log_finish(self)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.servers)
BucketOperationHelper.delete_all_buckets_or_assert(
self.servers, self)
for node in self.servers:
master = node
try:
ClusterOperationHelper.cleanup_cluster(self.servers,
master=master)
except:
continue
self.log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
super(FailoverBaseTest, self).tearDown()
def common_tearDown(servers, testcase):
RemoteUtilHelper.common_basic_setup(servers)
log = logger.Logger.get_logger()
log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
def common_tearDown(servers, testcase):
RemoteUtilHelper.common_basic_setup(servers)
log = logger.Logger.get_logger()
log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
def common_setup(input, testcase):
log.info("============== common_setup was started for test #{0} {1}=============="\
.format(testcase.case_number, testcase._testMethodName))
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
log.info("============== common_setup was finished for test #{0} {1} =============="\
.format(testcase.case_number, testcase._testMethodName))
def common_setup(input, testcase):
log.info("============== common_setup was started for test #{0} {1}=============="\
.format(testcase.case_number, testcase._testMethodName))
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
log.info("============== common_setup was finished for test #{0} {1} =============="\
.format(testcase.case_number, testcase._testMethodName))
def common_tearDown(servers, testcase):
RemoteUtilHelper.common_basic_setup(servers)
log = logger.Logger.get_logger()
log.info("10 seconds delay to wait for couchbase-server to start")
time.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
try:
MemcachedClientHelper.flush_bucket(servers[0], 'default')
except Exception:
pass
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
def setUp(self):
log = logger.Logger.get_logger()
self._input = TestInputSingleton.input
self._keys_count = self._input.param("keys_count", DEFAULT_KEY_COUNT)
self._num_replicas = self._input.param("replica", DEFAULT_REPLICA)
self.bidirectional = self._input.param("bidirectional", False)
self.case_number = self._input.param("case_number", 0)
self._value_size = self._input.param("value_size", 256)
self.wait_timeout = self._input.param("wait_timeout", 60)
self._servers = self._input.servers
self.master = self._servers[0]
self._failed_nodes = []
num_buckets = 0
self.buckets = []
self.default_bucket = self._input.param("default_bucket", True)
if self.default_bucket:
self.default_bucket_name = "default"
num_buckets += 1
self._standard_buckets = self._input.param("standard_buckets", 0)
self._sasl_buckets = self._input.param("sasl_buckets", 0)
num_buckets += self._standard_buckets + self._sasl_buckets
self.dgm_run = self._input.param("dgm_run", True)
self.log = logger.Logger().get_logger()
self._cluster_helper = Cluster()
self.disabled_consistent_view = self._input.param(
"disabled_consistent_view", None)
self._quota = self._initialize_nodes(self._cluster_helper,
self._servers,
self.disabled_consistent_view)
if self.dgm_run:
self.quota = 256
self.bucket_size = int(
(2.0 / 3.0) / float(num_buckets) * float(self._quota))
self.gen_create = BlobGenerator('loadOne',
'loadOne_',
self._value_size,
end=self._keys_count)
self.add_back_flag = False
self._cleanup_nodes = []
log.info("============== setup was started for test #{0} {1}=============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
for server in self._servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
self._setup_cluster()
self._create_buckets_()
log.info("============== setup was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') \
and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params \
and str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
# supported starting with python2.7
self.log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown(force=True)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.cluster.servers)
self.cluster_util.check_for_panic_and_mini_dumps(self.servers)
finally:
super(FailoverBaseTest, self).tearDown()
def common_tearDown(servers, testcase):
RemoteUtilHelper.common_basic_setup(servers)
log = logger.Logger.get_logger()
log.info("10 seconds delay to wait for couchbase-server to start")
time.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
try:
rest = RestConnection(self._servers[0])
buckets = rest.get_buckets()
for bucket in buckets:
MemcachedClientHelper.flush_bucket(servers[0], bucket.name)
except Exception:
pass
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
def setUp(self):
log = logger.Logger.get_logger()
self._input = TestInputSingleton.input
self._keys_count = self._input.param("keys_count", DEFAULT_KEY_COUNT)
self._num_replicas = self._input.param("replica", DEFAULT_REPLICA)
self.bidirectional = self._input.param("bidirectional", False)
self.case_number = self._input.param("case_number", 0)
self._value_size = self._input.param("value_size", 256)
self.wait_timeout = self._input.param("wait_timeout", 60)
self._servers = self._input.servers
self.master = self._servers[0]
self._failed_nodes = []
num_buckets = 0
self.buckets = []
self.default_bucket = self._input.param("default_bucket", True)
if self.default_bucket:
self.default_bucket_name = "default"
num_buckets += 1
self._standard_buckets = self._input.param("standard_buckets", 0)
self._sasl_buckets = self._input.param("sasl_buckets", 0)
num_buckets += self._standard_buckets + self._sasl_buckets
self.dgm_run = self._input.param("dgm_run", True)
self.log = logger.Logger().get_logger()
self._cluster_helper = Cluster()
self.disabled_consistent_view = self._input.param("disabled_consistent_view", None)
self._quota = self._initialize_nodes(self._cluster_helper, self._servers, self.disabled_consistent_view)
if self.dgm_run:
self.quota = 256
self.bucket_size = int((2.0 / 3.0) / float(num_buckets) * float(self._quota))
self.gen_create = BlobGenerator('loadOne', 'loadOne_', self._value_size, end=self._keys_count)
self.add_back_flag = False
self._cleanup_nodes = []
log.info("============== setup was started for test #{0} {1}=============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
for server in self._servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
self._setup_cluster()
self._create_buckets_()
log.info("============== setup was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
def common_tearDown(servers, testcase):
log.info("============== common_tearDown was started for test #{0} {1} =============="\
.format(testcase.case_number, testcase._testMethodName))
RemoteUtilHelper.common_basic_setup(servers)
log.info("10 seconds delay to wait for couchbase-server to start")
time.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase, \
wait_time=AutoFailoverBaseTest.MAX_FAIL_DETECT_TIME * 10, wait_if_warmup=True)
try:
rest = RestConnection(servers[0])
buckets = rest.get_buckets()
for bucket in buckets:
MemcachedClientHelper.flush_bucket(servers[0], bucket.name)
except Exception:
pass
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
log.info("============== common_tearDown was finished for test #{0} {1} =============="\
.format(testcase.case_number, testcase._testMethodName))
def common_setup(input, testcase):
log.info("============== common_setup was started for test #{0} {1}==============" \
.format(testcase.case_number, testcase._testMethodName))
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, testcase)
# Add built-in user
testuser = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'password': '******'}]
RbacBase().create_user_source(testuser, 'builtin', servers[0])
time.sleep(10)
# Assign user to role
role_list = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'roles': 'admin'}]
RbacBase().add_user_role(role_list, RestConnection(servers[0]), 'builtin')
time.sleep(10)
log.info("============== common_setup was finished for test #{0} {1} ==============" \
.format(testcase.case_number, testcase._testMethodName))
def test_node_memcached_failure(self):
timeout = self.timeout / 2
status = self.rest.update_autoreprovision_settings(True, 1)
if not status:
self.fail('failed to change autoreprovision_settings!')
self.sleep(5)
self._pause_couchbase(self.server_fail)
self.sleep(5)
AutoReprovisionBaseTest.wait_for_warmup_or_assert(self.master, 1,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
RemoteUtilHelper.common_basic_setup([self.server_fail])
AutoReprovisionBaseTest.wait_for_failover_or_assert(self.master, 0,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
helper = RestHelper(self.rest)
self.assertTrue(helper.is_cluster_healthy(), "cluster status is not healthy")
self.assertTrue(helper.is_cluster_rebalanced(), "cluster is not balanced")
buckets = self.rest.get_buckets()
for bucket in buckets:
self.verify_loaded_data(self.master, bucket.name, self.loaded_items[bucket.name])
def test_node_memcached_failure_in_series(self):
timeout = self.timeout / 2
status = self.rest.update_autoreprovision_settings(True, 1)
if not status:
self.fail('failed to change autoreprovision_settings!')
self.sleep(5)
data_lost = False
for i in reversed(xrange(len(self.servers))):
print self.servers[i]
operation = random.choice(['stop', 'memcached_failure', 'restart', 'failover', 'reboot'])
shell = RemoteMachineShellConnection(self.servers[i])
print "operation", operation
if i == 0:
self.master = self.servers[1]
if operation == 'stop':
self._stop_couchbase(self.servers[i])
elif operation == 'memcached_failure':
self._pause_couchbase(self.servers[i])
elif operation == 'restart':
shell.restart_couchbase()
elif operation == 'failover':
RemoteUtilHelper.enable_firewall(self.servers[i])
elif operation == 'reboot':
if shell.extract_remote_info().type.lower() == 'windows':
o, r = shell.execute_command("shutdown -r -f -t 0")
self.sleep(200)
elif shell.extract_remote_info().type.lower() == 'linux':
o, r = shell.execute_command("reboot")
shell.log_command_output(o, r)
self.sleep(60)
self.sleep(40)
if operation == 'memcached_failure':
AutoReprovisionBaseTest.wait_for_warmup_or_assert(self.master, 1,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
if operation != 'restart' and operation != 'memcached_failure' and operation != 'reboot':
AutoReprovisionBaseTest.wait_for_failover_or_assert(self.master, 1,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
if operation != 'restart':
RemoteUtilHelper.common_basic_setup([self.servers[i]])
AutoReprovisionBaseTest.wait_for_failover_or_assert(self.master, 0,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
helper = RestHelper(RestConnection(self.master))
self.assertTrue(helper.is_cluster_healthy(), "cluster status is not healthy")
self.sleep(40)
if operation == 'memcached_failure' or operation == 'failover':
self.assertTrue(helper.is_cluster_rebalanced(), "cluster is not balanced")
else:
if 'kv' in self.servers[i].services and self.replicas > 0:
self.assertFalse(helper.is_cluster_rebalanced(), "cluster is balanced")
self.rest.rebalance(otpNodes=[node.id for node in self.rest.node_statuses()], ejectedNodes=[])
self.assertTrue(self.rest.monitorRebalance())
else:
self.assertTrue(helper.is_cluster_rebalanced(), "cluster is not balanced")
buckets = self.rest.get_buckets()
if self.replicas == 0 and (operation == 'restart' or operation == 'reboot'):
data_lost = True
for bucket in buckets:
if not data_lost:
self.verify_loaded_data(self.master, bucket.name, self.loaded_items[bucket.name])
def test_node_memcached_failure_in_series(self):
timeout = self.timeout / 2
status = self.rest.update_autoreprovision_settings(True, 1)
if not status:
self.fail('failed to change autoreprovision_settings!')
self.sleep(5)
data_lost = False
for i in reversed(xrange(len(self.servers))):
print self.servers[i]
operation = random.choice(
['stop', 'memcached_failure', 'restart', 'failover', 'reboot'])
shell = RemoteMachineShellConnection(self.servers[i])
print "operation", operation
if i == 0:
self.master = self.servers[1]
if operation == 'stop':
self._stop_couchbase(self.servers[i])
elif operation == 'memcached_failure':
self._pause_couchbase(self.servers[i])
elif operation == 'restart':
shell.restart_couchbase()
elif operation == 'failover':
RemoteUtilHelper.enable_firewall(self.servers[i])
elif operation == 'reboot':
if shell.extract_remote_info().type.lower() == 'windows':
o, r = shell.execute_command("shutdown -r -f -t 0")
self.sleep(200)
elif shell.extract_remote_info().type.lower() == 'linux':
o, r = shell.execute_command("reboot")
shell.log_command_output(o, r)
self.sleep(60)
self.sleep(40)
if operation == 'memcached_failure':
AutoReprovisionBaseTest.wait_for_warmup_or_assert(
self.master, 1,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
if operation != 'restart' and operation != 'memcached_failure' and operation != 'reboot':
AutoReprovisionBaseTest.wait_for_failover_or_assert(
self.master, 1,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME,
self)
if operation != 'restart':
RemoteUtilHelper.common_basic_setup([self.servers[i]])
AutoReprovisionBaseTest.wait_for_failover_or_assert(
self.master, 0,
timeout + AutoReprovisionBaseTest.MAX_FAIL_DETECT_TIME, self)
helper = RestHelper(RestConnection(self.master))
self.assertTrue(helper.is_cluster_healthy(),
"cluster status is not healthy")
self.sleep(40)
if operation == 'memcached_failure' or operation == 'failover':
self.assertTrue(helper.is_cluster_rebalanced(),
"cluster is not balanced")
else:
if 'kv' in self.servers[i].services and self.replicas > 0:
self.assertFalse(helper.is_cluster_rebalanced(),
"cluster is balanced")
self.rest.rebalance(otpNodes=[
node.id for node in self.rest.node_statuses()
],
ejectedNodes=[])
self.assertTrue(self.rest.monitorRebalance())
else:
self.assertTrue(helper.is_cluster_rebalanced(),
"cluster is not balanced")
buckets = self.rest.get_buckets()
if self.replicas == 0 and (operation == 'restart'
or operation == 'reboot'):
data_lost = True
for bucket in buckets:
if not data_lost:
self.verify_loaded_data(self.master, bucket.name,
self.loaded_items[bucket.name])
