def setUp_bucket(self, unittest):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
unittest.assertTrue(self.input, msg="input parameters missing...")
self.test = unittest
self.master = self.input.servers[0]
rest = RestConnection(self.master)
rest.init_cluster(username=self.master.rest_username, password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
ClusterOperationHelper.cleanup_cluster([self.master])
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self.test)
serverInfo = self.master
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.memoryQuota)
# Add built-in user
testuser = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'password': '******'}]
RbacBase().create_user_source(testuser, 'builtin', self.master)
time.sleep(10)
# Assign user to role
role_list = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'roles': 'admin'}]
RbacBase().add_user_role(role_list, RestConnection(self.master), 'builtin')
time.sleep(10)
def cleanup(self):
rest = RestConnection(self.master)
rest.stop_rebalance()
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self)
def setUp(self):
self.log = logger.Logger.get_logger()
self.master = TestInputSingleton.input.servers[0]
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.num_of_docs = self.input.param("num_of_docs", 1000)
rest = RestConnection(self.master)
for server in self.servers:
rest.init_cluster(server.rest_username, server.rest_password)
info = rest.get_nodes_self()
for server in self.servers:
rest.init_cluster_memoryQuota(
server.rest_username, server.rest_password, memoryQuota=info.mcdMemoryReserved
)
ClusterOperationHelper.cleanup_cluster(self.servers)
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
self._create_default_bucket()
# Rebalance the nodes
ClusterOperationHelper.begin_rebalance_in(self.master, self.servers)
ClusterOperationHelper.end_rebalance(self.master)
self._log_start()
def setUp(self):
self.log = logger.Logger().get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.master = self.servers[0]
self.ip = self.master.ip
self.finished = False
self.keys = []
self.keycount = 0
self.failure_string = ""
self.cleanup()
rest = RestConnection(self.master)
info = rest.get_nodes_self()
self.port = info.moxi+1
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
created = BucketOperationHelper.create_multiple_buckets(self.master,
replica=1,
bucket_ram_ratio=(2.0 / 3.0),
howmany=10,
sasl=False)
self.assertTrue(created, "bucket creation failed")
ready = BucketOperationHelper.wait_for_memcached(self.master, "bucket-0")
self.assertTrue(ready, "wait_for_memcached failed")
def tearDown(self):
try:
test_failed = len(self._resultForDoCleanups.errors)
if self.driver and test_failed:
BaseHelper(self).create_screenshot()
if self.driver:
self.driver.close()
if test_failed and TestInputSingleton.input.param("stop-on-failure", False):
print "test fails, teardown will be skipped!!!"
return
rest = RestConnection(self.servers[0])
try:
reb_status = rest._rebalance_progress_status()
except ValueError as e:
if e.message == 'No JSON object could be decoded':
print "cluster not initialized!!!"
return
if reb_status == 'running':
stopped = rest.stop_rebalance()
self.assertTrue(stopped, msg="unable to stop rebalance")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self)
except Exception as e:
raise e
finally:
if self.driver:
self.shell.disconnect()
def _create_plasma_buckets(self):
for bucket in self.buckets:
if bucket.name.startswith("standard"):
BucketOperationHelper.delete_bucket_or_assert(
serverInfo=self.dgmServer, bucket=bucket.name)
self.buckets = [bu for bu in self.buckets if not bu.name.startswith("standard")]
buckets = []
for i in range(self.num_plasma_buckets):
name = "plasma_dgm_" + str(i)
buckets.append(name)
bucket_size = self._get_bucket_size(self.quota,
len(self.buckets)+len(buckets))
self._create_buckets(server=self.master, bucket_list=buckets,
bucket_size=bucket_size)
testuser = []
rolelist = []
for bucket in buckets:
testuser.append({'id': bucket, 'name': bucket, 'password': '******'})
rolelist.append({'id': bucket, 'name': bucket, 'roles': 'admin'})
self.add_built_in_server_user(testuser=testuser, rolelist=rolelist)
buckets = []
for bucket in self.buckets:
if bucket.name.startswith("plasma_dgm"):
buckets.append(bucket)
return buckets
def tearDown(self):
if not self.input.param("skip_cleanup", True):
if self.times_teardown_called > 1 :
self.shell.disconnect()
if self.input.param("skip_cleanup", True):
if self.case_number > 1 or self.times_teardown_called > 1:
self.shell.disconnect()
self.times_teardown_called += 1
serverInfo = self.servers[0]
rest = RestConnection(serverInfo)
zones = rest.get_zone_names()
for zone in zones:
if zone != "Group 1":
rest.delete_zone(zone)
self.clusters_dic = self.input.clusters
if self.clusters_dic:
if len(self.clusters_dic) > 1:
self.dest_nodes = self.clusters_dic[1]
self.dest_master = self.dest_nodes[0]
if self.dest_nodes and len(self.dest_nodes) > 1:
self.log.info("======== clean up destination cluster =======")
rest = RestConnection(self.dest_nodes[0])
rest.remove_all_remote_clusters()
rest.remove_all_replications()
BucketOperationHelper.delete_all_buckets_or_assert(self.dest_nodes, self)
ClusterOperationHelper.cleanup_cluster(self.dest_nodes)
elif len(self.clusters_dic) == 1:
self.log.error("=== need 2 cluster to setup xdcr in ini file ===")
else:
self.log.info("**** If run xdcr test, need cluster config is setup in ini file. ****")
super(CliBaseTest, self).tearDown()
def test_oom_delete_bucket(self):
"""
1. Get OOM
2. Delete a bucket
3. Verify if state of indexes is changed
:return:
"""
self.assertTrue(self._push_indexer_off_the_cliff(), "OOM Can't be achieved")
for i in range(len(self.buckets)):
log.info("Deleting bucket {0}...".format(self.buckets[i].name))
BucketOperationHelper.delete_bucket_or_assert(serverInfo=self.oomServer, bucket=self.buckets[i].name)
self.sleep(120)
check = self._validate_indexer_status_oom()
if not check:
if i < len(self.buckets):
self.buckets = self.buckets[i+1:]
else:
#TODO: Pras: Need better solution here
self.buckets = []
break
log.info("Indexer Still in OOM...")
self.sleep(120)
self.assertFalse(self._validate_indexer_status_oom(), "Indexer still in OOM")
self._verify_bucket_count_with_index_count(self.load_query_definitions)
self.multi_query_using_index(buckets=self.buckets,
query_definitions=self.load_query_definitions)
def tearDown(self):
try:
self._cluster_helper.shutdown()
log = logger.Logger.get_logger()
log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self._servers)
log.info("10 seconds delay to wait for membase-server to start")
time.sleep(10)
for server in self._cleanup_nodes:
shell = RemoteMachineShellConnection(server)
o, r = shell.execute_command("iptables -F")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A INPUT -p tcp -i eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/sbin/iptables -A OUTPUT -p tcp -o eth0 --dport 1000:60000 -j ACCEPT")
shell.log_command_output(o, r)
o, r = shell.execute_command("/etc/init.d/couchbase-server start")
shell.log_command_output(o, r)
shell.disconnect()
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
ClusterOperationHelper.cleanup_cluster(self._servers)
ClusterHelper.wait_for_ns_servers_or_assert(self._servers, self)
log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
pass
def test_backup_with_spatial_data(self):
num_docs = self.helper.input.param("num-docs", 5000)
self.log.info("description : Make limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init(data_set)
if not self.command_options:
self.command_options = []
options = self.command_options + [' -m full']
self.total_backups = 1
self.shell.execute_cluster_backup(self.couchbase_login_info, self.backup_location, options)
time.sleep(2)
self.buckets = RestConnection(self.master).get_buckets()
bucket_names = [bucket.name for bucket in self.buckets]
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
gc.collect()
self.helper._create_default_bucket()
self.shell.restore_backupFile(self.couchbase_login_info, self.backup_location, bucket_names)
SimpleDataSet(self.helper, num_docs)._create_views()
self._query_test_init(data_set)
def common_setup(input, testcase):
servers = input.servers
RemoteUtilHelper.common_basic_setup(servers)
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
for server in servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
def common_tearDown(servers, testcase):
log = logger.Logger.get_logger()
log.info(
"============== common_tearDown was started for test #{0} {1} ==============".format(
testcase.case_number, testcase._testMethodName
)
)
RemoteUtilHelper.common_basic_setup(servers)
log.info("10 seconds delay to wait for couchbase-server to start")
time.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(
servers, testcase, wait_time=AutoFailoverBaseTest.MAX_FAIL_DETECT_TIME * 15, wait_if_warmup=True
)
try:
rest = RestConnection(self._servers[0])
buckets = rest.get_buckets()
for bucket in buckets:
MemcachedClientHelper.flush_bucket(servers[0], bucket.name)
except Exception:
pass
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
log.info(
"============== common_tearDown was finished for test #{0} {1} ==============".format(
testcase.case_number, testcase._testMethodName
)
)
def cleanup_cluster(self):
if not "skip_cleanup" in TestInputSingleton.input.test_params:
BucketOperationHelper.delete_all_buckets_or_assert(
self.servers, self.testcase)
ClusterOperationHelper.cleanup_cluster(self.servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers, self.testcase)
def _cluster_setup(self):
replicas = self.input.param("replicas", 1)
keys_count = self.input.param("keys-count", 0)
num_buckets = self.input.param("num-buckets", 1)
bucket_name = "default"
master = self.servers[0]
credentials = self.input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.reset_autofailover()
ClusterOperationHelper.add_and_rebalance(self.servers, True)
if num_buckets == 1:
bucket_ram = info.memoryQuota * 2 / 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replicas,
proxyPort=info.moxi)
else:
created = BucketOperationHelper.create_multiple_buckets(self.master, replicas, howmany=num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
for bucket in buckets:
inserted_keys_cnt = self.load_data(self.master, bucket.name, keys_count)
log.info('inserted {0} keys'.format(inserted_keys_cnt))
def setUp(self):
super(SGConfigTests, self).setUp()
for server in self.servers:
if self.case_number == 1:
with open('pytests/sg/resources/gateway_config_walrus_template.json', 'r') as file:
filedata = file.read()
filedata = filedata.replace('LOCAL_IP', server.ip)
with open('pytests/sg/resources/gateway_config_walrus.json', 'w') as file:
file.write(filedata)
shell = RemoteMachineShellConnection(server)
shell.execute_command("rm -rf {0}/tmp/*".format(self.folder_prefix))
shell.copy_files_local_to_remote('pytests/sg/resources', '{0}/tmp'.format(self.folder_prefix))
# will install sg only the first time
self.install(shell)
pid = self.is_sync_gateway_process_running(shell)
self.assertNotEqual(pid, 0)
exist = shell.file_exists('{0}/tmp/'.format(self.folder_prefix), 'gateway.log')
self.assertTrue(exist)
shell.disconnect()
if self.case_number == 1:
shutil.copy2('pytests/sg/resources/gateway_config_backup.json', 'pytests/sg/resources/gateway_config.json')
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
self.cluster = Cluster()
self.cluster.create_default_bucket(self.master, 150)
task = self.cluster.async_create_sasl_bucket(self.master, 'test_%E-.5', 'password', 150, 1)
task.result()
task = self.cluster.async_create_standard_bucket(self.master, 'db', 11219, 150, 1)
task.result()
def setUp(self):
super(XDCRTests, self).setUp()
self.bucket = Bucket()
self._initialize_nodes()
self.master = self.servers[0]
for server in self.servers:
rest=RestConnection(server)
cluster_status = rest.cluster_status()
self.log.info("Initial status of {0} cluster is {1}".format(server.ip,
cluster_status['nodes'][0]['status']))
while cluster_status['nodes'][0]['status'] == 'warmup':
self.log.info("Waiting for cluster to become healthy")
self.sleep(5)
cluster_status = rest.cluster_status()
self.log.info("current status of {0} is {1}".format(server.ip,
cluster_status['nodes'][0]['status']))
# Delete all buckets before creating new buckets
self.log.info("Deleting all existing buckets")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
self.log.info("Creating new buckets")
src_bucket = self.input.param('src_bucket', self.bucket)
dest_bucket = self.input.param('dest_bucket', self.bucket)
if src_bucket:
RestConnection(self.servers[0]).create_bucket(bucket='default', ramQuotaMB=500)
if dest_bucket:
RestConnection(self.servers[1]).create_bucket(bucket='default', ramQuotaMB=500)
helper = BaseHelper(self)
helper.login()
def setUp(self):
self.log = logger.Logger.get_logger()
self.master = TestInputSingleton.input.servers[0]
ClusterOperationHelper.cleanup_cluster([self.master])
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
self._bucket_name = 'default'
serverInfo = self.master
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
self._bucket_port = info.moxi
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.memoryQuota * 2 / 3
# Add built-in user
testuser = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'password': '******'}]
RbacBase().create_user_source(testuser, 'builtin', self.master)
time.sleep(10)
# Assign user to role
role_list = [{'id': 'cbadminbucket', 'name': 'cbadminbucket', 'roles': 'admin'}]
RbacBase().add_user_role(role_list, RestConnection(self.master), 'builtin')
time.sleep(10)
rest.create_bucket(bucket=self._bucket_name,
ramQuotaMB=bucket_ram,
proxyPort=info.memcached)
msg = 'create_bucket succeeded but bucket "default" does not exist'
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(self._bucket_name, rest), msg=msg)
ready = BucketOperationHelper.wait_for_memcached(serverInfo, self._bucket_name)
self.assertTrue(ready, "wait_for_memcached failed")
self._log_start()
def setUp(self):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.cluster = Cluster()
self.servers = self.input.servers
self.buckets = {}
self.default_bucket = self.input.param("default_bucket", True)
self.standard_buckets = self.input.param("standard_buckets", 0)
self.sasl_buckets = self.input.param("sasl_buckets", 0)
self.total_buckets = self.sasl_buckets + self.default_bucket + self.standard_buckets
self.num_servers = self.input.param("servers", len(self.servers))
self.num_replicas = self.input.param("replicas", 1)
self.num_items = self.input.param("items", 1000)
self.dgm_run = self.input.param("dgm_run", False)
if not self.input.param("skip_cleanup", False):
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterOperationHelper.wait_for_ns_servers_or_assert([self.servers[0]], self)
self.quota = self._initialize_nodes(self.cluster, self.servers)
if self.dgm_run:
self.quota = 256
self.bucket_size = self._get_bucket_size(self.quota, self.total_buckets)
if self.default_bucket:
self.cluster.create_default_bucket(self.servers[0], self.bucket_size, self.num_replicas)
self.buckets['default'] = {1 : KVStore()}
self._create_sasl_buckets(self.servers[0], self.sasl_buckets)
def reset(self):
self.log.info(
"============== SwapRebalanceBase cleanup was started for test #{0} {1} ==============".format(
self.case_number, self._testMethodName
)
)
self.log.info("Stopping load in Teardown")
SwapRebalanceBase.stop_load(self.loaders)
for server in self.servers:
rest = RestConnection(server)
if rest._rebalance_progress_status() == "running":
self.log.warning("rebalancing is still running, test should be verified")
stopped = rest.stop_rebalance()
self.assertTrue(stopped, msg="unable to stop rebalance")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
if server.data_path:
rest = RestConnection(server)
rest.set_data_path(data_path=server.data_path)
ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self)
self.log.info(
"============== SwapRebalanceBase cleanup was finished for test #{0} {1} ==============".format(
self.case_number, self._testMethodName
)
)
def tearDown(self):
if hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and 'stop-on-failure' in TestInputSingleton.input.test_params and \
str(TestInputSingleton.input.test_params['stop-on-failure']).lower() == 'true':
# supported starting with python2.7
log.warn("CLEANUP WAS SKIPPED")
self.cluster.shutdown(force=True)
self._log_finish(self)
else:
try:
self.log.info("============== tearDown was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
RemoteUtilHelper.common_basic_setup(self.servers)
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for node in self.servers:
master = node
try:
ClusterOperationHelper.cleanup_cluster(self.servers,
master=master)
except:
continue
self.log.info("============== tearDown was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
super(FailoverBaseTest, self).tearDown()
def tearDown(self):
try:
if self.driver:
path_screen = self.input.ui_conf['screenshots'] or 'logs/screens'
full_path = '{1}/screen_{0}.png'.format(time.time(), path_screen)
self.log.info('screenshot is available: %s' % full_path)
if not os.path.exists(path_screen):
os.mkdir(path_screen)
self.driver.get_screenshot_as_file(os.path.abspath(full_path))
rest = RestConnection(self.servers[0])
if rest._rebalance_progress_status() == 'running':
stopped = rest.stop_rebalance()
self.assertTrue(stopped, msg="unable to stop rebalance")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self)
if self.driver:
self.driver.close()
except Exception as e:
raise e
finally:
if self.driver:
self.shell.disconnect()
self.cluster.shutdown()
def test_valid_bucket_name(self, password='******'):
tasks = []
if self.bucket_type == 'sasl':
self.cluster.create_sasl_bucket(self.server, self.bucket_name, password, self.num_replicas, self.bucket_size)
self.buckets.append(Bucket(name=self.bucket_name, authType="sasl", saslPassword=password, num_replicas=self.num_replicas,
bucket_size=self.bucket_size, master_id=self.server))
elif self.bucket_type == 'standard':
self.cluster.create_standard_bucket(self.server, self.bucket_name, STANDARD_BUCKET_PORT + 1, self.bucket_size, self.num_replicas)
self.buckets.append(Bucket(name=self.bucket_name, authType=None, saslPassword=None, num_replicas=self.num_replicas,
bucket_size=self.bucket_size, port=STANDARD_BUCKET_PORT + 1, master_id=self.server))
elif self.bucket_type == "memcached":
tasks.append(self.cluster.async_create_memcached_bucket(self.server, self.bucket_name, STANDARD_BUCKET_PORT + 1,
self.bucket_size, self.num_replicas))
self.buckets.append(Bucket(name=self.bucket_name, authType=None, saslPassword=None, num_replicas=self.num_replicas,
bucket_size=self.bucket_size, port=STANDARD_BUCKET_PORT + 1 , master_id=self.server, type='memcached'))
for task in tasks:
task.result()
else:
self.log.error('Bucket type not specified')
return
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(self.bucket_name, self.rest),
msg='failed to start up bucket with name "{0}'.format(self.bucket_name))
gen_load = BlobGenerator('buckettest', 'buckettest-', self.value_size, start=0, end=self.num_items)
self._load_all_buckets(self.server, gen_load, "create", 0)
self.cluster.bucket_delete(self.server, self.bucket_name)
self.assertTrue(BucketOperationHelper.wait_for_bucket_deletion(self.bucket_name, self.rest, timeout_in_seconds=60),
msg='bucket "{0}" was not deleted even after waiting for 30 seconds'.format(self.bucket_name))
def tearDown(self):
# super(Rebalance, self).tearDown()
try:
self.log.info("============== XDCRbasetests stats for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
self._end_replication_flag = 1
if hasattr(self, '_stats_thread1'): self._stats_thread1.join()
if hasattr(self, '_stats_thread2'): self._stats_thread2.join()
if hasattr(self, '_stats_thread3'): self._stats_thread3.join()
if self._replication_direction_str in "bidirection":
if hasattr(self, '_stats_thread4'): self._stats_thread4.join()
if hasattr(self, '_stats_thread5'): self._stats_thread5.join()
if hasattr(self, '_stats_thread6'): self._stats_thread6.join()
if self._replication_direction_str in "bidirection":
self.log.info("Type of run: BIDIRECTIONAL XDCR")
else:
self.log.info("Type of run: UNIDIRECTIONAL XDCR")
self._print_stats(self.src_master)
if self._replication_direction_str in "bidirection":
self._print_stats(self.dest_master)
self.log.info("============== = = = = = = = = END = = = = = = = = = = ==============")
self.log.info("============== rebalanceXDCR cleanup was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
for nodes in [self.src_nodes, self.dest_nodes]:
for node in nodes:
BucketOperationHelper.delete_all_buckets_or_assert([node], self)
ClusterOperationHelper.cleanup_cluster([node], self)
ClusterOperationHelper.wait_for_ns_servers_or_assert([node], self)
self.log.info("============== rebalanceXDCR cleanup was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
finally:
self.cluster.shutdown(force=True)
self._log_finish(self)
def test_backup_restore(self):
self._load_all_buckets()
self.shell.execute_command("rm -rf /tmp/backups")
output, error = self.shell.execute_command("/opt/couchbase/bin/cbbackupmgr config "
"--archive /tmp/backups --repo example")
self.log.info(output)
self.assertEquals('Backup repository `example` created successfully in archive `/tmp/backups`', output[0])
output, error = self.shell.execute_command(
"/opt/couchbase/bin/cbbackupmgr backup --archive /tmp/backups --repo example "
"--cluster couchbase://127.0.0.1 --username Administrator --password password")
self.log.info(output)
self.assertEquals('Backup successfully completed', output[1])
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
imp_rest = RestConnection(self.master)
info = imp_rest.get_nodes_self()
if info.memoryQuota and int(info.memoryQuota) > 0:
self.quota = info.memoryQuota
bucket_params = self._create_bucket_params(server=self.master, size=250, bucket_type='ephemeral',
replicas=self.num_replicas,
enable_replica_index=self.enable_replica_index,
eviction_policy=self.eviction_policy)
self.cluster.create_default_bucket(bucket_params)
output, error = self.shell.execute_command('ls /tmp/backups/example')
output, error = self.shell.execute_command("/opt/couchbase/bin/cbbackupmgr restore --archive /tmp/backups"
" --repo example --cluster couchbase://127.0.0.1 "
"--username Administrator --password password --start %s" % output[
0])
self.log.info(output)
self.assertEquals('Restore completed successfully', output[1])
self._verify_all_buckets(self.master)
def common_setup(self, replica):
self._input = TestInputSingleton.input
self._servers = self._input.servers
first = self._servers[0]
self.log = logger.Logger().get_logger()
self.log.info(self._input)
rest = RestConnection(first)
for server in self._servers:
RestHelper(RestConnection(server)).is_ns_server_running()
ClusterOperationHelper.cleanup_cluster(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
ClusterOperationHelper.add_all_nodes_or_assert(self._servers[0], self._servers, self._input.membase_settings, self)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
info = rest.get_nodes_self()
bucket_ram = info.mcdMemoryReserved * 3 / 4
rest.create_bucket(bucket="default",
ramQuotaMB=int(bucket_ram),
replicaNumber=replica,
proxyPort=rest.get_nodes_self().moxi)
msg = "wait_for_memcached fails"
ready = BucketOperationHelper.wait_for_memcached(first, "default"),
self.assertTrue(ready, msg)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(rebalanceStarted,
"unable to start rebalance on master node {0}".format(first.ip))
self.log.info('started rebalance operation on master node {0}'.format(first.ip))
rebalanceSucceeded = rest.monitorRebalance()
# without a bucket this seems to fail
self.assertTrue(rebalanceSucceeded,
"rebalance operation for nodes: {0} was not successful".format(otpNodeIds))
self.awareness = VBucketAwareMemcached(rest, "default")
def tearDown(self):
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
rest = RestConnection(self.master)
# Remove rbac user in teardown
role_del = ['cbadminbucket']
temp = RbacBase().remove_user_role(role_del, rest)
self._log_finish()
def setUp(self):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.assertTrue(self.input, msg="input parameters missing...")
self.servers = self.input.servers
self.master = self.servers[0]
rest = RestConnection(self.master)
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
info = rest.get_nodes_self()
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
BucketOperationHelper.delete_all_buckets_or_assert(servers=self.servers, test_case=self)
ClusterOperationHelper.cleanup_cluster(servers=self.servers)
credentials = self.input.membase_settings
ClusterOperationHelper.add_all_nodes_or_assert(master=self.master, all_servers=self.servers, rest_settings=credentials, test_case=self)
rest = RestConnection(self.master)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(rebalanceStarted,
"unable to start rebalance on master node {0}".format(self.master.ip))
self.log.info('started rebalance operation on master node {0}'.format(self.master.ip))
rebalanceSucceeded = rest.monitorRebalance()
def setUp(self):
self.log = logger.Logger.get_logger()
self.input = TestInput.TestInputSingleton.input
self.assertTrue(self.input, msg="input parameters missing...")
self.servers = self.input.servers
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, test_case=self)
self._log_start()
def tearDown(self):
try:
if (hasattr(self, '_resultForDoCleanups') and len(self._resultForDoCleanups.failures) > 0 \
and TestInputSingleton.input.param("stop-on-failure", False))\
or self.input.param("skip_cleanup", False):
self.log.warn("CLEANUP WAS SKIPPED")
else:
self.log.info("============== basetestcase cleanup was started for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
rest = RestConnection(self.master)
alerts = rest.get_alerts()
if alerts is not None and len(alerts) != 0:
self.log.warn("Alerts were found: {0}".format(alerts))
if rest._rebalance_progress_status() == 'running':
self.log.warning("rebalancing is still running, test should be verified")
stopped = rest.stop_rebalance()
self.assertTrue(stopped, msg="unable to stop rebalance")
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
ClusterOperationHelper.cleanup_cluster(self.servers)
self.sleep(10)
ClusterOperationHelper.wait_for_ns_servers_or_assert(self.servers, self)
self.log.info("============== basetestcase cleanup was finished for test #{0} {1} =============="\
.format(self.case_number, self._testMethodName))
except BaseException:
# increase case_number to retry tearDown in setup for the next test
self.case_number += 1000
finally:
# stop all existing task manager threads
self.cluster.shutdown()
self._log_finish(self)
def common_setUp(self, with_buckets):
ClusterOperationHelper.cleanup_cluster(self.servers)
server = self.servers[0]
if with_buckets:
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, test_case=self)
ok = BucketOperationHelper.create_multiple_buckets(server, 1)
if not ok:
self.fail("unable to create multiple buckets on this node : {0}".format(server))
def common_setup(input, testcase, bucket_ram_ratio=(2.8 / 3.0), replica=0):
log = logger.Logger.get_logger()
servers = input.servers
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
serverInfo = servers[0]
log.info('picking server : {0} as the master'.format(serverInfo))
#if all nodes are on the same machine let's have the bucket_ram_ratio as bucket_ram_ratio * 1/len(servers)
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(servers)
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved *
node_ram_ratio))
if "ascii" in TestInputSingleton.input.test_params\
and TestInputSingleton.input.test_params["ascii"].lower() == "true":
BucketOperationHelper.create_multiple_buckets(serverInfo,
replica,
node_ram_ratio *
bucket_ram_ratio,
howmany=1,
sasl=False)
else:
BucketOperationHelper.create_multiple_buckets(serverInfo,
replica,
node_ram_ratio *
bucket_ram_ratio,
howmany=1,
sasl=True)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(
serverInfo, bucket.name)
testcase.assertTrue(ready, "wait_for_memcached failed")
def _test_backup_add_restore_bucket_with_expiration_key(self, replica):
bucket = "default"
rest = RestConnection(self.master)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi, replicaNumber=replica)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
client = MemcachedClientHelper.direct_client(self.master, bucket)
expiry = 60
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(5000)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, key)
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
client.close()
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
ready = RebalanceHelper.wait_for_persistence(self.master, bucket, bucket_type=self.bucket_type)
self.assertTrue(ready, "not all items persisted. see logs")
node = RestConnection(self.master).get_nodes_self()
output, error = self.shell.execute_command(self.perm_command)
self.shell.log_command_output(output, error)
backupHelper = BackupHelper(self.master, self)
backupHelper.backup(bucket, node, self.remote_tmp_folder)
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket, self)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
backupHelper.restore(self.remote_tmp_folder)
time.sleep(60)
client = MemcachedClientHelper.direct_client(self.master, bucket)
self.log.info('verifying that all those keys have expired...')
for key in keys:
try:
client.get(key=key)
msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
self.fail(msg.format(expiry, key, expiry))
except mc_bin_client.MemcachedError as error:
self.assertEqual(error.status, 1,
msg="expected error code {0} but saw error code {1}".format(1, error.status))
client.close()
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def test_non_default_case_sensitive_same_port(self):
postfix = uuid.uuid4()
name = 'uppercase_{0}'.format(postfix)
for serverInfo in self.servers:
rest = RestConnection(serverInfo)
proxyPort = rest.get_nodes_self().moxi + 100
rest.create_bucket(bucket=name,
ramQuotaMB=200,
proxyPort=proxyPort)
msg = 'create_bucket succeeded but bucket {0} does not exist'.format(name)
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(name, rest), msg=msg)
self.log.info("user should not be able to create a new bucket on a an already used port")
name = 'UPPERCASE{0}'.format(postfix)
try:
rest.create_bucket(bucket=name,
ramQuotaMB=200,
proxyPort=proxyPort)
self.fail('create-bucket did not throw exception while creating a new bucket on an already used port')
#make sure it raises bucketcreateexception
except BucketCreationException as ex:
self.log.error(ex)
def test_max_buckets(self):
log = logger.Logger.get_logger()
serverInfo = self.servers[0]
log.info('picking server : {0} as the master'.format(serverInfo))
rest = RestConnection(serverInfo)
proxyPort = rest.get_nodes_self().moxi
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_num = rest.get_internalSettings("maxBucketCount")
bucket_ram = 100
for i in range(bucket_num):
bucket_name = 'max_buckets-{0}'.format(uuid.uuid4())
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
authType='sasl', proxyPort=proxyPort)
ready = BucketOperationHelper.wait_for_memcached(serverInfo, bucket_name)
self.assertTrue(ready, "wait_for_memcached failed")
buckets = rest.get_buckets()
if len(buckets) != bucket_num:
msg = 'tried to create {0} buckets, only created {1}'.format(bucket_count, len(buckets))
self.fail(msg)
try:
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
authType='sasl', proxyPort=proxyPort)
msg = 'bucket creation did not fail even though system was overcommited'
self.fail(msg)
except BucketCreationException as ex:
self.log.info('BucketCreationException was thrown as expected when we try to create {0} buckets'.
format(bucket_num + 1))
buckets = rest.get_buckets()
if len(buckets) != bucket_num:
msg = 'tried to create {0} buckets, only created {1}'.format(bucket_num + 1, len(buckets))
self.fail(msg)
def setUp(self):
super(BucketConfig, self).setUp()
self.testcase = '2'
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
#self.time_synchronization = self.input.param("time_sync", "enabledWithoutDrift")
self.lww = self.input.param("lww", True)
self.drift = self.input.param("drift", False)
self.bucket='bucket-1'
self.master = self.servers[0]
self.rest = RestConnection(self.master)
self.cluster = Cluster()
self.skip_rebalance = self.input.param("skip_rebalance", False)
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
mem_quota = int(self.rest.get_nodes_self().mcdMemoryReserved *
node_ram_ratio)
if not self.skip_rebalance:
self.rest.init_cluster(self.master.rest_username,
self.master.rest_password)
self.rest.init_cluster_memoryQuota(self.master.rest_username,
self.master.rest_password,
memoryQuota=mem_quota)
for server in self.servers:
ClusterOperationHelper.cleanup_cluster([server])
ClusterOperationHelper.wait_for_ns_servers_or_assert(
[self.master], self.testcase)
try:
rebalanced = ClusterOperationHelper.add_and_rebalance(
self.servers)
except Exception as e:
self.fail(e, 'cluster is not rebalanced')
self._create_bucket(self.lww, self.drift)
def test_non_default_case_sensitive_different_port(self):
postfix = uuid.uuid4()
lowercase_name = 'uppercase_{0}'.format(postfix)
for serverInfo in self.servers:
rest = RestConnection(serverInfo)
proxyPort = rest.get_nodes_self().moxi + 500
rest.create_bucket(bucket=lowercase_name,
ramQuotaMB=200,
proxyPort=proxyPort)
msg = 'create_bucket succeeded but bucket {0} does not exist'.format(lowercase_name)
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(lowercase_name, rest), msg=msg)
uppercase_name = 'UPPERCASE_{0}'.format(postfix)
try:
rest.create_bucket(bucket=uppercase_name,
ramQuotaMB=200,
proxyPort=proxyPort + 1000)
msg = "create_bucket created two buckets in different case : {0},{1}".format(lowercase_name,
uppercase_name)
self.fail(msg)
except BucketCreationException as ex:
#check if 'default' and 'Default' buckets exist
self.log.info('BucketCreationException was thrown as expected')
self.log.info(ex._message)
def test_bucket_edit_password(self,
bucket_name='secretsbucket',
num_replicas=1,
bucket_size=100):
updated_pass = "******"
rest = RestConnection(self.master)
for servers in self.servers:
self.secretmgmt_base_obj.setup_pass_node(servers, self.password)
bucket_type = self.input.param("bucket_type", 'standard')
tasks = []
if bucket_type == 'sasl':
self.cluster.create_sasl_bucket(self.master, bucket_name,
self.password, num_replicas,
bucket_size)
self.sleep(10)
rest.change_bucket_props(bucket_name, saslPassword=updated_pass)
else:
self.log.error('Bucket type not specified')
return
self.assertTrue(
BucketOperationHelper.wait_for_bucket_creation(
bucket_name, RestConnection(self.master)),
msg='failed to start up bucket with name "{0}'.format(bucket_name))
gen_load = BlobGenerator('buckettest',
'buckettest-',
self.value_size,
start=0,
end=self.num_items)
self._load_all_buckets(self.master, gen_load, "create", 0)
install_path = self.secretmgmt_base_obj._get_install_path(self.master)
temp_result = self.secretmgmt_base_obj.check_config_files(
self.master, install_path, '/config/config.dat', updated_pass)
self.assertTrue(temp_result, "Password found in config.dat")
temp_result = self.secretmgmt_base_obj.check_config_files(
self.master, install_path, 'isasl.pw', updated_pass)
self.assertTrue(temp_result, "Password found in isasl.pw")
def test_valid_length(self):
max_len = 100
name_len = self.input.param('name_length', 100)
name = 'a' * name_len
master = self.servers[0]
rest = RestConnection(master)
try:
rest.create_bucket(bucket=name,
ramQuotaMB=256,
storageBackend=self.bucket_storage)
if name_len <= max_len:
msg = 'failed to start up bucket with valid length'
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(
name, rest),
msg=msg)
else:
self.fail('Bucket with invalid length created')
except BucketCreationException as ex:
self.log.error(ex)
if name_len <= max_len:
self.fail('could not create bucket with valid length')
else:
self.log.info(
'bucket with invalid length not created as expected')
def _test_backup_and_restore_bucket_overwriting_body(
self, overwrite_flag=True):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master,
test_case=self)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(
len(keys), expiry))
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node,
self.remote_tmp_folder)
shell.disconnect()
for key in keys:
try:
client.replace(key, expiry, 0, "2")
except mc_bin_client.MemcachedError as error:
msg = "unable to replace key : {0} in bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("replaced {0} keys with expiry set to {1}".format(
len(keys), expiry))
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder,
overwrite_flag)
time.sleep(10)
self.log.info('verifying that all those keys...')
for key in keys:
if overwrite_flag:
self.assertEqual("2", client.get(key=key),
key + " should has value = 2")
else:
self.assertNotEqual("2", client.get(key=key),
key + " should not has value = 2")
self.log.info(
"verified that those keys inserted with expiry set to {0} have expired"
.format(expiry))
def _create_default_bucket(self, master):
BucketOperationHelper.create_default_buckets(servers=[master],
assert_on_test=self)
ready = BucketOperationHelper.wait_for_memcached(master, "default")
self.assertTrue(ready, "wait_for_memcached failed")
def _test_backup_add_restore_bucket_body(self, bucket,
delay_after_data_load,
startup_flag, single_node):
server = self.master
rest = RestConnection(server)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
if bucket == "default":
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
else:
proxyPort = info.moxi + 500
rest.create_bucket(bucket,
ramQuotaMB=size,
proxyPort=proxyPort,
authType="sasl",
saslPassword="******")
ready = BucketOperationHelper.wait_for_memcached(server, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
if not single_node:
self.add_nodes_and_rebalance()
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_keys, rejected_keys = MemcachedClientHelper.load_bucket_and_return_the_keys(
servers=[self.master],
name=bucket,
ram_load_ratio=1,
value_size_distribution=distribution,
moxi=True,
write_only=True,
number_of_threads=2)
if not single_node:
rest = RestConnection(self.master)
self.assertTrue(RebalanceHelper.wait_for_replication(
rest.get_nodes(), timeout=180),
msg="replication did not complete")
self.log.info(
"Sleep {0} seconds after data load".format(delay_after_data_load))
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
node = RestConnection(self.master).get_nodes_self()
if not startup_flag:
for server in self.servers:
shell = RemoteMachineShellConnection(server)
shell.stop_membase()
shell.stop_couchbase()
shell.disconnect()
output, error = self.shell.execute_command(self.perm_command)
self.shell.log_command_output(output, error)
#now let's back up
BackupHelper(self.master, self).backup(bucket, node,
self.remote_tmp_folder)
if not startup_flag:
for server in self.servers:
shell = RemoteMachineShellConnection(server)
shell.start_membase()
shell.start_couchbase()
RestHelper(RestConnection(server)).is_ns_server_running()
shell.disconnect()
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket,
self)
if bucket == "default":
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
else:
proxyPort = info.moxi + 500
rest.create_bucket(bucket,
ramQuotaMB=size,
proxyPort=proxyPort,
authType="sasl",
saslPassword="******")
BucketOperationHelper.wait_for_memcached(self.master, bucket)
if bucket == "default":
BackupHelper(self.master,
self).restore(backup_location=self.remote_tmp_folder,
moxi_port=info.moxi)
else:
BackupHelper(self.master,
self).restore(backup_location=self.remote_tmp_folder,
moxi_port=info.moxi,
username=bucket,
password='******')
keys_exist = BucketOperationHelper.keys_exist_or_assert_in_parallel(
inserted_keys, self.master, bucket, self, concurrency=4)
self.assertTrue(keys_exist, msg="unable to verify keys after restore")
def test_negative_auto_retry_of_failed_rebalance_where_rebalance_will_not_be_cancelled(
self):
during_rebalance_failure = self.input.param("during_rebalance_failure",
"stop_server")
post_failure_operation = self.input.param("post_failure_operation",
"create_delete_buckets")
zone_name = "Group_{0}_{1}".format(random.randint(1, 1000000000),
self._testMethodName)
zone_name = zone_name[0:60]
default_zone = "Group 1"
moved_node = []
moved_node.append(self.servers[1].ip)
try:
operation = self._rebalance_operation(self.rebalance_operation)
self.sleep(self.sleep_time)
# induce the failure during the rebalance
self._induce_error(during_rebalance_failure)
operation.result()
except Exception as e:
self.log.info("Rebalance failed with : {0}".format(str(e)))
# Recover from the error
self._recover_from_error(during_rebalance_failure)
result = json.loads(self.rest.get_pending_rebalance_info())
self.log.info(result)
retry_rebalance = result["retry_rebalance"]
if retry_rebalance != "pending":
self.fail("Auto-retry of failed rebalance is not triggered")
if post_failure_operation == "create_delete_buckets":
# delete buckets and create new one
BucketOperationHelper.delete_all_buckets_or_assert(
servers=self.servers, test_case=self)
self.sleep(self.sleep_time)
BucketOperationHelper.create_bucket(self.master,
test_case=self)
elif post_failure_operation == "change_replica_count":
# change replica count
self.log.info("Changing replica count of buckets")
for bucket in self.buckets:
self.rest.change_bucket_props(bucket, replicaNumber=2)
elif post_failure_operation == "change_server_group":
# change server group
self.log.info("Creating new zone " + zone_name)
self.rest.add_zone(zone_name)
self.log.info("Moving {0} to new zone {1}".format(
moved_node, zone_name))
status = self.rest.shuffle_nodes_in_zones(
moved_node, default_zone, zone_name)
else:
self.fail("Invalid post_failure_operation option")
# In these failure scenarios while the retry is pending, then the retry will be attempted but fail
try:
self.check_retry_rebalance_succeeded()
except Exception as e:
self.log.info(e)
if "Retrying of rebalance still did not help. All the retries exhausted" not in str(
e):
self.fail(
"Auto retry of failed rebalance succeeded when it was expected to fail"
)
else:
self.fail(
"Rebalance did not fail as expected. Hence could not validate auto-retry feature.."
)
finally:
if post_failure_operation == "change_server_group":
status = self.rest.shuffle_nodes_in_zones(
moved_node, zone_name, default_zone)
self.log.info(
"Shuffle the node back to default group . Status : {0}".
format(status))
self.sleep(self.sleep_time)
self.log.info("Deleting new zone " + zone_name)
try:
self.rest.delete_zone(zone_name)
except:
self.log.info("Errors in deleting zone")
if self.disable_auto_failover:
self.rest.update_autofailover_settings(True, 120)
self.start_server(self.servers[1])
self.stop_firewall_on_node(self.servers[1])
def _install_and_upgrade(self,
initial_version='1.6.5.3',
initialize_cluster=False,
create_buckets=False,
insert_data=False):
input = TestInputSingleton.input
rest_settings = input.membase_settings
servers = input.servers
server = servers[0]
save_upgrade_config = False
if initial_version.startswith(
"1.7") and input.test_params['version'].startswith("1.8"):
save_upgrade_config = True
is_amazon = False
if input.test_params.get('amazon', False):
is_amazon = True
if initial_version.startswith("1.6") or initial_version.startswith(
"1.7"):
product = 'membase-server-enterprise'
else:
product = 'couchbase-server-enterprise'
remote = RemoteMachineShellConnection(server)
rest = RestConnection(server)
info = remote.extract_remote_info()
remote.membase_uninstall()
remote.couchbase_uninstall()
builds, changes = BuildQuery().get_all_builds()
# check to see if we are installing from latestbuilds or releases
# note: for newer releases (1.8.0) even release versions can have the
# form 1.8.0r-55
if re.search('r', initial_version):
builds, changes = BuildQuery().get_all_builds()
older_build = BuildQuery().find_membase_build(
builds,
deliverable_type=info.deliverable_type,
os_architecture=info.architecture_type,
build_version=initial_version,
product=product,
is_amazon=is_amazon)
else:
older_build = BuildQuery().find_membase_release_build(
deliverable_type=info.deliverable_type,
os_architecture=info.architecture_type,
build_version=initial_version,
product=product,
is_amazon=is_amazon)
remote.stop_membase()
remote.stop_couchbase()
remote.download_build(older_build)
#now let's install ?
remote.membase_install(older_build)
RestHelper(rest).is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)
rest.init_cluster_port(rest_settings.rest_username,
rest_settings.rest_password)
bucket_data = {}
if initialize_cluster:
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
if create_buckets:
_create_load_multiple_bucket(self,
server,
bucket_data,
howmany=2)
version = input.test_params['version']
appropriate_build = _get_build(servers[0],
version,
is_amazon=is_amazon)
self.assertTrue(appropriate_build.url,
msg="unable to find build {0}".format(version))
remote.download_build(appropriate_build)
remote.membase_upgrade(appropriate_build,
save_upgrade_config=save_upgrade_config)
remote.disconnect()
RestHelper(rest).is_ns_server_running(testconstants.NS_SERVER_TIMEOUT)
pools_info = rest.get_pools_info()
rest.init_cluster_port(rest_settings.rest_username,
rest_settings.rest_password)
time.sleep(TIMEOUT_SECS)
#verify admin_creds still set
self.assertTrue(pools_info['implementationVersion'],
appropriate_build.product_version)
if initialize_cluster:
#TODO: how can i verify that the cluster init config is preserved
if create_buckets:
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(
'bucket-0', rest),
msg="bucket 'default' does not exist..")
if insert_data:
buckets = rest.get_buckets()
for bucket in buckets:
BucketOperationHelper.keys_exist_or_assert(
bucket_data[bucket.name]["inserted_keys"], server,
bucket.name, self)
def _test_backup_and_restore_from_to_different_buckets(self):
bucket_before_backup = "bucket_before_backup"
bucket_after_backup = "bucket_after_backup"
BucketOperationHelper.create_bucket(serverInfo=self.master,
name=bucket_before_backup,
port=11212,
test_case=self)
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket_before_backup)
self.assertTrue(ready, "wait_for_memcached failed")
self.add_nodes_and_rebalance()
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_keys, rejected_keys = MemcachedClientHelper.load_bucket_and_return_the_keys(
servers=[self.master],
name=bucket_before_backup,
ram_load_ratio=20,
value_size_distribution=distribution,
write_only=True,
moxi=True,
number_of_threads=2)
self.log.info("Sleep after data load")
ready = RebalanceHelper.wait_for_stats_on_all(self.master,
bucket_before_backup,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master,
bucket_before_backup,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket_before_backup, node,
self.remote_tmp_folder)
shell.disconnect()
BucketOperationHelper.delete_bucket_or_assert(self.master,
bucket_before_backup,
self)
BucketOperationHelper.create_bucket(serverInfo=self.master,
name=bucket_after_backup,
port=11212,
test_case=self)
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket_after_backup)
self.assertTrue(ready, "wait_for_memcached failed")
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder,
moxi_port=11212)
time.sleep(10)
ready = RebalanceHelper.wait_for_stats_on_all(self.master,
bucket_after_backup,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master,
bucket_after_backup,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
self.assertTrue(
BucketOperationHelper.verify_data(self.master,
inserted_keys,
False,
False,
11212,
debug=False,
bucket=bucket_after_backup),
"Missing keys")
def common_test_body(self, keys_count, replica, load_ratio,
failover_reason):
log = logger.Logger.get_logger()
log.info("keys_count : {0}".format(keys_count))
log.info("replica : {0}".format(replica))
log.info("load_ratio : {0}".format(load_ratio))
log.info("failover_reason : {0}".format(failover_reason))
master = self._servers[0]
log.info('picking server : {0} as the master'.format(master))
rest = RestConnection(master)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.memoryQuota * 2 / 3
bucket = 'default'
rest.create_bucket(bucket=bucket,
ramQuotaMB=bucket_ram,
replicaNumber=replica,
proxyPort=info.moxi)
ready = BucketOperationHelper.wait_for_memcached(master, bucket)
self.assertTrue(ready, "wait_for_memcached_failed")
credentials = self._input.membase_settings
ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers,
credentials, self)
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
msg = "rebalance failed after adding these nodes {0}".format(nodes)
self.assertTrue(rest.monitorRebalance(), msg=msg)
inserted_keys = FailoverBaseTest.load_data(master, bucket, keys_count,
load_ratio)
inserted_count = len(inserted_keys)
log.info('inserted {0} keys'.format(inserted_count))
nodes = rest.node_statuses()
while (len(nodes) - replica) > 1:
final_replication_state = RestHelper(rest).wait_for_replication(
900)
msg = "replication state after waiting for up to 15 minutes : {0}"
self.log.info(msg.format(final_replication_state))
chosen = RebalanceHelper.pick_nodes(master, howmany=replica)
for node in chosen:
#let's do op
if failover_reason == 'stop_server':
self.stop_server(node)
log.info(
"10 seconds delay to wait for membase-server to shutdown"
)
#wait for 5 minutes until node is down
self.assertTrue(
RestHelper(rest).wait_for_node_status(
node, "unhealthy", 300),
msg=
"node status is not unhealthy even after waiting for 5 minutes"
)
elif failover_reason == "firewall":
RemoteUtilHelper.enable_firewall(
self._servers, node, bidirectional=self.bidirectional)
self.assertTrue(
RestHelper(rest).wait_for_node_status(
node, "unhealthy", 300),
msg=
"node status is not unhealthy even after waiting for 5 minutes"
)
failed_over = rest.fail_over(node.id)
if not failed_over:
self.log.info(
"unable to failover the node the first time. try again in 60 seconds.."
)
#try again in 60 seconds
time.sleep(75)
failed_over = rest.fail_over(node.id)
self.assertTrue(
failed_over, "unable to failover node after {0}".format(
failover_reason))
log.info("failed over node : {0}".format(node.id))
self._failed_nodes.append(node.ip)
log.info(
"10 seconds sleep after failover before invoking rebalance...")
time.sleep(10)
rest.rebalance(otpNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in chosen])
msg = "rebalance failed while removing failover nodes {0}".format(
chosen)
self.assertTrue(rest.monitorRebalance(stop_if_loop=True), msg=msg)
FailoverBaseTest.replication_verification(master, bucket, replica,
inserted_count, self)
nodes = rest.node_statuses()
FailoverBaseTest.verify_data(master, inserted_keys, bucket, self)
def common_setup(self):
self.cluster_helper = Cluster()
self.log = logger.Logger.get_logger()
self.cluster_run = False
self.input = TestInputSingleton.input
self.servers = self.input.servers
serverInfo = self.servers[0]
rest = RestConnection(serverInfo)
if len(set([server.ip for server in self.servers])) == 1:
ip = rest.get_nodes_self().ip
for server in self.servers:
server.ip = ip
self.cluster_run = True
self.case_number = self.input.param("case_number", 0)
self.replica = self.input.param("replica", 1)
self.keys_count = self.input.param("keys-count", 1000)
self.load_ratio = self.input.param("load-ratio", 1)
self.ratio_expiry = self.input.param("ratio-expiry", 0.03)
self.ratio_deletes = self.input.param("ratio-deletes", 0.13)
self.num_buckets = self.input.param("num-buckets", 1)
self.failover_factor = self.num_swap = self.input.param("num-swap", 1)
self.num_initial_servers = self.input.param("num-initial-servers", 3)
self.fail_orchestrator = self.swap_orchestrator = self.input.param(
"swap-orchestrator", False)
self.do_access = self.input.param("do-access", True)
self.load_started = False
self.loaders = []
try:
# Clear the state from Previous invalid run
if rest._rebalance_progress_status() == 'running':
self.log.warning(
"rebalancing is still running, previous test should be verified"
)
stopped = rest.stop_rebalance()
self.assertTrue(stopped, msg="unable to stop rebalance")
self.log.info("============== SwapRebalanceBase setup was started for test #{0} {1}=============="\
.format(self.case_number, self._testMethodName))
SwapRebalanceBase.reset(self)
# Make sure the test is setup correctly
min_servers = int(self.num_initial_servers) + int(self.num_swap)
msg = "minimum {0} nodes required for running swap rebalance"
self.assertTrue(len(self.servers) >= min_servers,
msg=msg.format(min_servers))
self.log.info(
'picking server : {0} as the master'.format(serverInfo))
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(
self.servers)
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
SwapRebalanceBase.enable_diag_eval_on_non_local_hosts(
self, serverInfo)
# Add built-in user
testuser = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'password': '******'
}]
RbacBase().create_user_source(testuser, 'builtin', self.servers[0])
# Assign user to role
role_list = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'roles': 'admin'
}]
RbacBase().add_user_role(role_list,
RestConnection(self.servers[0]),
'builtin')
if self.num_buckets > 10:
BaseTestCase.change_max_buckets(self, self.num_buckets)
self.log.info(
"============== SwapRebalanceBase setup was finished for test #{0} {1} =============="
.format(self.case_number, self._testMethodName))
SwapRebalanceBase._log_start(self)
except Exception, e:
self.cluster_helper.shutdown()
self.fail(e)
def setUp(self):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.autocompaction_value = TestInputSingleton.input.param("autocompaction_value", 0)
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
def _create_bucket(self, bucketname):
self.rest.create_bucket(bucket=bucketname, ramQuotaMB=100, authType="sasl",
saslPassword="******")
ready = BucketOperationHelper.wait_for_memcached(self.master, bucketname)
self.assertTrue(ready, msg="wait_for_memcached failed")
if "bucket_quota" in params:
bucket_quota = params["bucket_quota"]
if "moxi" in params:
if params["moxi"].lower() == "true":
moxi = True
if "get" in params:
if params["get"].lower() == "true":
get = True
if "prefix" in params:
prefix = params["prefix"]
if "delete" in params:
if params["delete"].lower() == "true":
delete = True
if delete:
BucketOperationHelper.delete_all_buckets_or_assert([server], None)
create_buckets(server, count, prefix, bucket_quota)
if run_load:
rest = RestConnection(server)
buckets = rest.get_buckets()
threads = []
for bucket in buckets:
t = Thread(target=load_buckets,
args=(server, bucket.name, get, no_threads, moxi))
t.start()
threads.append(t)
for t in threads:
t.join()
except Exception as ex:
def backup_reset_clusters(self, servers):
BucketOperationHelper.delete_all_buckets_or_assert(servers, self)
ClusterOperationHelper.cleanup_cluster(servers, master=servers[0])
ClusterOperationHelper.wait_for_ns_servers_or_assert(servers, self)
def verify_loaded_data(self, master, bucket, inserted_keys):
keys_exist = BucketOperationHelper.keys_exist_or_assert_in_parallel(
inserted_keys, master, bucket, self, concurrency=4)
self.assertTrue(keys_exist, msg="unable to verify keys after restore")
def setUp(self):
self.log = logger.Logger.get_logger()
self.master = TestInputSingleton.input.servers[0]
ClusterOperationHelper.cleanup_cluster([self.master])
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
self._bucket_name = 'default'
serverInfo = self.master
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
self._bucket_port = info.moxi
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.memoryQuota * 2 // 3
# Add built-in user
testuser = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'password': '******'
}]
RbacBase().create_user_source(testuser, 'builtin', self.master)
# Assign user to role
role_list = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'roles': 'admin'
}]
RbacBase().add_user_role(role_list, RestConnection(self.master),
'builtin')
self.log.info("-->create_bucket: {},{},{}".format(
self._bucket_name, bucket_ram, info.memcached))
rest.create_bucket(bucket=self._bucket_name,
ramQuotaMB=bucket_ram,
proxyPort=info.memcached)
msg = 'create_bucket succeeded but bucket "default" does not exist'
if (testconstants.TESTRUNNER_CLIENT in list(os.environ.keys())
) and os.environ[
testconstants.TESTRUNNER_CLIENT] == testconstants.PYTHON_SDK:
self.client = SDKSmartClient(
serverInfo,
self._bucket_name,
compression=TestInputSingleton.input.param(
"sdk_compression", True))
else:
self.client = MemcachedClientHelper.direct_client(
serverInfo, self._bucket_name)
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(
self._bucket_name, rest),
msg=msg)
ready = BucketOperationHelper.wait_for_memcached(
serverInfo, self._bucket_name)
self.assertTrue(ready, "wait_for_memcached failed")
self._log_start()
def _test_cluster_topology_change_body(self):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master,
test_case=self)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
self.add_nodes_and_rebalance()
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_keys, rejected_keys = MemcachedClientHelper.load_bucket_and_return_the_keys(
servers=[self.master],
ram_load_ratio=1,
value_size_distribution=distribution,
moxi=True,
write_only=True,
number_of_threads=2)
self.log.info("Sleep after data load")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
#let's create a unique folder in the remote location
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node,
self.remote_tmp_folder)
shell.disconnect()
ClusterOperationHelper.cleanup_cluster(self.servers)
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, self)
servers = []
for i in range(0, len(self.servers) - 1):
servers.append(self.servers[i])
self.add_node_and_rebalance(servers[0], servers)
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket,
self)
BucketOperationHelper.create_bucket(serverInfo=self.master,
test_case=self)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder)
time.sleep(10)
BucketOperationHelper.verify_data(self.master, inserted_keys, False,
False, 11210, self)
def repetitive_create_delete(self):
self.repetitions = self.input.param("repetition_count", 1)
self.bufferspace = self.input.param("bufferspace", 6000000)
# the first front end load
self._load_all_buckets(self.master,
self.gen_create,
"create",
0,
batch_size=10000,
pause_secs=5,
timeout_secs=100)
self._wait_for_stats_all_buckets(self.servers)
rest = RestConnection(self.servers[0])
max_data_sizes = {}
initial_memory_usage = {}
self.sleep(30)
for bucket in self.buckets:
max_data_sizes[bucket.name] = rest.fetch_bucket_stats(
bucket=bucket.name)["op"]["samples"]["ep_max_size"][-1]
self.log.info("Initial max_data_size of bucket '{0}': {1}".format(
bucket.name, max_data_sizes[bucket.name]))
initial_memory_usage[bucket.name] = \
rest.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["mem_used"][-1]
self.log.info(
"initial memory consumption of bucket '{0}' with load: {1}".
format(bucket.name, initial_memory_usage[bucket.name]))
mem_usage = {}
self.sleep(10)
# the repetitions
for i in range(0, self.repetitions):
BucketOperationHelper.delete_all_buckets_or_assert(
self.servers, self)
del self.buckets[:]
self.log.info('About to create the buckets')
self._bucket_creation()
self.log.info('Done bucket creation, about to load them')
self._load_all_buckets(self.master,
self.gen_create,
"create",
0,
batch_size=10000,
pause_secs=5,
timeout_secs=100)
self.log.info('Buckets are loaded, waiting for stats')
self._wait_for_stats_all_buckets(self.servers)
self.log.info('Have the stats, sleeping for 30 seconds')
self.sleep(60)
for bucket in self.buckets:
mem_usage[bucket.name] = rest.fetch_bucket_stats(
bucket.name)["op"]["samples"]["mem_used"][-1]
self.log.info("Memory used after attempt {0} = {1}, Difference from initial snapshot: {2}" \
.format(i + 1, mem_usage[bucket.name],
(mem_usage[bucket.name] - initial_memory_usage[bucket.name])))
self.sleep(10)
if (self.repetitions > 0):
self.log.info(
"After {0} repetitive deletion-creation-load of the buckets, the memory consumption difference is .." \
.format(self.repetitions))
for bucket in self.buckets:
self.log.info("{0} :: Initial: {1} :: Now: {2} :: Difference: {3}" \
.format(bucket.name, initial_memory_usage[bucket.name], mem_usage[bucket.name],
(mem_usage[bucket.name] - initial_memory_usage[bucket.name])))
msg = "Memory used now, much greater than initial usage!"
assert mem_usage[bucket.name] <= initial_memory_usage[
bucket.name] + self.bufferspace, msg
else:
self.log.info(
"Verification skipped, as there weren't any repetitions..")
def _test_delete_key_and_backup_and_restore_body(self):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master,
name=bucket,
test_case=self)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(
len(keys), expiry))
client.delete(keys[0])
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
#let's create a unique folder in the remote location
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node,
self.remote_tmp_folder)
shell.disconnect()
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder)
time.sleep(10)
self.log.info('verifying that all those keys...')
missing_keys = []
verify_keys = []
for key in keys:
vBucketId = crc32.crc32_hash(key) & 1023 # or & 0x3FF
client.vbucketId = vBucketId
if key == keys[0]:
missing_keys.append(key)
else:
verify_keys.append(key)
self.assertTrue(
BucketOperationHelper.keys_dont_exist(self.master, missing_keys,
self), "Keys are not empty")
self.assertTrue(
BucketOperationHelper.verify_data(self.master, verify_keys, False,
False, 11210, self),
"Missing keys")
def tearDown(self):
BucketOperationHelper.delete_all_buckets_or_assert(
servers=self.servers, test_case=self)
self._log_finish()
def test_backup_upgrade_restore_default(self):
if len(self.servers) < 2:
self.log.error("At least 2 servers required for this test ..")
return
original_set = copy.copy(self.servers)
worker = self.servers[len(self.servers) - 1]
self.servers = self.servers[:len(self.servers) - 1]
shell = RemoteMachineShellConnection(self.master)
o, r = shell.execute_command("cat /opt/couchbase/VERSION.txt")
fin = o[0]
shell.disconnect()
initial_version = self.input.param("initial_version", fin)
final_version = self.input.param("final_version", fin)
if initial_version == final_version:
self.log.error("Same initial and final versions ..")
return
if not final_version.startswith('2.0'):
self.log.error("Upgrade test not set to run from 1.8.1 -> 2.0 ..")
return
builds, changes = BuildQuery().get_all_builds()
product = 'couchbase-server-enterprise'
#CASE where the worker isn't a 2.0+
worker_flag = 0
shell = RemoteMachineShellConnection(worker)
o, r = shell.execute_command("cat /opt/couchbase/VERSION.txt")
temp = o[0]
if not temp.startswith('2.0'):
worker_flag = 1
if worker_flag == 1:
self.log.info(
"Loading version {0} on worker.. ".format(final_version))
remote = RemoteMachineShellConnection(worker)
info = remote.extract_remote_info()
older_build = BuildQuery().find_build(builds, product,
info.deliverable_type,
info.architecture_type,
final_version)
remote.stop_couchbase()
remote.couchbase_uninstall()
remote.download_build(older_build)
remote.install_server(older_build)
remote.disconnect()
remote_tmp = "{1}/{0}".format("backup", "/root")
perm_comm = "mkdir -p {0}".format(remote_tmp)
if not initial_version == fin:
for server in self.servers:
remote = RemoteMachineShellConnection(server)
info = remote.extract_remote_info()
self.log.info(
"Loading version .. {0}".format(initial_version))
older_build = BuildQuery().find_build(builds, product,
info.deliverable_type,
info.architecture_type,
initial_version)
remote.stop_couchbase()
remote.couchbase_uninstall()
remote.download_build(older_build)
remote.install_server(older_build)
rest = RestConnection(server)
RestHelper(rest).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
rest.init_cluster(server.rest_username, server.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
remote.disconnect()
self.common_setUp()
bucket = "default"
if len(self.servers) > 1:
self.add_nodes_and_rebalance()
rest = RestConnection(self.master)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
rest.create_bucket(bucket, ramQuotaMB=size)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached_failed")
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_keys, rejected_keys = MemcachedClientHelper.load_bucket_and_return_the_keys(
servers=[self.master],
name=bucket,
ram_load_ratio=0.5,
value_size_distribution=distribution,
moxi=True,
write_only=True,
delete_ratio=0.1,
number_of_threads=2)
if len(self.servers) > 1:
rest = RestConnection(self.master)
self.assertTrue(RebalanceHelper.wait_for_replication(
rest.get_nodes(), timeout=180),
msg="replication did not complete")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket,
'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
node = RestConnection(self.master).get_nodes_self()
shell = RemoteMachineShellConnection(worker)
o, r = shell.execute_command(perm_comm)
shell.log_command_output(o, r)
shell.disconnect()
#Backup
#BackupHelper(self.master, self).backup(bucket, node, remote_tmp)
shell = RemoteMachineShellConnection(worker)
shell.execute_command(
"/opt/couchbase/bin/cbbackup http://{0}:{1} {2}".format(
self.master.ip, self.master.port, remote_tmp))
shell.disconnect()
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket,
self)
time.sleep(30)
#Upgrade
for server in self.servers:
self.log.info(
"Upgrading to current version {0}".format(final_version))
remote = RemoteMachineShellConnection(server)
info = remote.extract_remote_info()
new_build = BuildQuery().find_build(builds, product,
info.deliverable_type,
info.architecture_type,
final_version)
remote.stop_couchbase()
remote.couchbase_uninstall()
remote.download_build(new_build)
remote.install_server(new_build)
rest = RestConnection(server)
RestHelper(rest).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
rest.init_cluster(server.rest_username, server.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
remote.disconnect()
time.sleep(30)
#Restore
rest = RestConnection(self.master)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
rest.create_bucket(bucket, ramQuotaMB=size)
ready = BucketOperationHelper.wait_for_memcached(server, bucket)
self.assertTrue(ready, "wait_for_memcached_failed")
#BackupHelper(self.master, self).restore(backup_location=remote_tmp, moxi_port=info.moxi)
shell = RemoteMachineShellConnection(worker)
shell.execute_command(
"/opt/couchbase/bin/cbrestore {2} http://{0}:{1} -b {3}".format(
self.master.ip, self.master.port, remote_tmp, bucket))
shell.disconnect()
time.sleep(60)
keys_exist = BucketOperationHelper.keys_exist_or_assert_in_parallel(
inserted_keys, self.master, bucket, self, concurrency=4)
self.assertTrue(keys_exist, msg="unable to verify keys after restore")
time.sleep(30)
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket,
self)
rest = RestConnection(self.master)
helper = RestHelper(rest)
nodes = rest.node_statuses()
master_id = rest.get_nodes_self().id
if len(self.servers) > 1:
removed = helper.remove_nodes(
knownNodes=[node.id for node in nodes],
ejectedNodes=[
node.id for node in nodes if node.id != master_id
],
wait_for_rebalance=True)
shell = RemoteMachineShellConnection(worker)
shell.remove_directory(remote_tmp)
shell.disconnect()
self.servers = copy.copy(original_set)
if initial_version == fin:
builds, changes = BuildQuery().get_all_builds()
for server in self.servers:
remote = RemoteMachineShellConnection(server)
info = remote.extract_remote_info()
self.log.info(
"Loading version .. {0}".format(initial_version))
older_build = BuildQuery().find_build(builds, product,
info.deliverable_type,
info.architecture_type,
initial_version)
remote.stop_couchbase()
remote.couchbase_uninstall()
remote.download_build(older_build)
remote.install_server(older_build)
rest = RestConnection(server)
RestHelper(rest).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
rest.init_cluster(server.rest_username, server.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
remote.disconnect()
def _install_and_upgrade(self,
initial_version='1.6.5.3',
create_buckets=False,
insert_data=False,
start_upgraded_first=True,
load_ratio=-1,
roll_upgrade=False,
upgrade_path=[],
do_new_rest=False):
node_upgrade_path = []
node_upgrade_path.extend(upgrade_path)
#then start them in whatever order you want
inserted_keys = []
log = logger.Logger.get_logger()
if roll_upgrade:
log.info("performing an online upgrade")
input = TestInputSingleton.input
rest_settings = input.membase_settings
servers = input.servers
save_upgrade_config = False
is_amazon = False
if input.test_params.get('amazon', False):
is_amazon = True
if initial_version.startswith("1.6") or initial_version.startswith(
"1.7"):
product = 'membase-server-enterprise'
else:
product = 'couchbase-server-enterprise'
# install older build on all nodes
for server in servers:
remote = RemoteMachineShellConnection(server)
rest = RestConnection(server)
info = remote.extract_remote_info()
# check to see if we are installing from latestbuilds or releases
# note: for newer releases (1.8.0) even release versions can have the
# form 1.8.0r-55
if re.search('r', initial_version):
builds, changes = BuildQuery().get_all_builds()
older_build = BuildQuery().find_membase_build(
builds,
deliverable_type=info.deliverable_type,
os_architecture=info.architecture_type,
build_version=initial_version,
product=product,
is_amazon=is_amazon)
else:
older_build = BuildQuery().find_membase_release_build(
deliverable_type=info.deliverable_type,
os_architecture=info.architecture_type,
build_version=initial_version,
product=product,
is_amazon=is_amazon)
remote.membase_uninstall()
remote.couchbase_uninstall()
remote.stop_membase()
remote.stop_couchbase()
remote.download_build(older_build)
#now let's install ?
remote.membase_install(older_build)
RestHelper(rest).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
rest.init_cluster_port(rest_settings.rest_username,
rest_settings.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
remote.disconnect()
bucket_data = {}
master = servers[0]
if create_buckets:
#let's create buckets
#wait for the bucket
#bucket port should also be configurable , pass it as the
#parameter to this test ? later
self._create_default_bucket(master)
inserted_keys = self._load_data(master, load_ratio)
_create_load_multiple_bucket(self, master, bucket_data, howmany=2)
# cluster all the nodes together
ClusterOperationHelper.add_all_nodes_or_assert(master, servers,
rest_settings, self)
rest = RestConnection(master)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(
rebalanceStarted,
"unable to start rebalance on master node {0}".format(master.ip))
log.info('started rebalance operation on master node {0}'.format(
master.ip))
rebalanceSucceeded = rest.monitorRebalance()
self.assertTrue(
rebalanceSucceeded,
"rebalance operation for nodes: {0} was not successful".format(
otpNodeIds))
if initial_version == "1.7.0" or initial_version == "1.7.1":
self._save_config(rest_settings, master)
input_version = input.test_params['version']
node_upgrade_path.append(input_version)
current_version = initial_version
previous_version = current_version
#if we dont want to do roll_upgrade ?
log.info("Upgrade path: {0} -> {1}".format(initial_version,
node_upgrade_path))
log.info("List of servers {0}".format(servers))
if not roll_upgrade:
for version in node_upgrade_path:
previous_version = current_version
current_version = version
if version != initial_version:
log.info("Upgrading to version {0}".format(version))
self._stop_membase_servers(servers)
if previous_version.startswith(
"1.7") and current_version.startswith("1.8"):
save_upgrade_config = True
# No need to save the upgrade config from 180 to 181
if previous_version.startswith(
"1.8.0") and current_version.startswith("1.8.1"):
save_upgrade_config = False
appropriate_build = _get_build(servers[0],
version,
is_amazon=is_amazon)
self.assertTrue(
appropriate_build.url,
msg="unable to find build {0}".format(version))
for server in servers:
remote = RemoteMachineShellConnection(server)
remote.download_build(appropriate_build)
remote.membase_upgrade(
appropriate_build,
save_upgrade_config=save_upgrade_config)
RestHelper(
RestConnection(server)).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
#verify admin_creds still set
pools_info = RestConnection(server).get_pools_info()
self.assertTrue(pools_info['implementationVersion'],
appropriate_build.product_version)
if start_upgraded_first:
log.info("Starting server {0} post upgrade".format(
server))
remote.start_membase()
else:
remote.stop_membase()
remote.disconnect()
if not start_upgraded_first:
log.info("Starting all servers together")
self._start_membase_servers(servers)
time.sleep(TIMEOUT_SECS)
if version == "1.7.0" or version == "1.7.1":
self._save_config(rest_settings, master)
if create_buckets:
self.assertTrue(
BucketOperationHelper.wait_for_bucket_creation(
'default', RestConnection(master)),
msg="bucket 'default' does not exist..")
if insert_data:
self._verify_data(master, rest, inserted_keys)
# rolling upgrade
else:
version = input.test_params['version']
appropriate_build = _get_build(servers[0],
version,
is_amazon=is_amazon)
self.assertTrue(appropriate_build.url,
msg="unable to find build {0}".format(version))
# rebalance node out
# remove membase from node
# install destination version onto node
# rebalance it back into the cluster
for server_index in range(len(servers)):
server = servers[server_index]
master = servers[server_index - 1]
log.info("current master is {0}, rolling node is {1}".format(
master, server))
rest = RestConnection(master)
nodes = rest.node_statuses()
allNodes = []
toBeEjectedNodes = []
for node in nodes:
allNodes.append(node.id)
if "{0}:{1}".format(node.ip,
node.port) == "{0}:{1}".format(
server.ip, server.port):
toBeEjectedNodes.append(node.id)
helper = RestHelper(rest)
removed = helper.remove_nodes(knownNodes=allNodes,
ejectedNodes=toBeEjectedNodes)
self.assertTrue(
removed,
msg="Unable to remove nodes {0}".format(toBeEjectedNodes))
remote = RemoteMachineShellConnection(server)
remote.download_build(appropriate_build)
# if initial version is 180
# Don't uninstall the server
if not initial_version.startswith('1.8.0'):
remote.membase_uninstall()
remote.couchbase_uninstall()
remote.membase_install(appropriate_build)
else:
remote.membase_upgrade(appropriate_build)
RestHelper(rest).is_ns_server_running(
testconstants.NS_SERVER_TIMEOUT)
log.info(
"sleep for 10 seconds to wait for membase-server to start..."
)
time.sleep(TIMEOUT_SECS)
rest.init_cluster_port(rest_settings.rest_username,
rest_settings.rest_password)
rest.init_cluster_memoryQuota(
memoryQuota=rest.get_nodes_self().mcdMemoryReserved)
remote.disconnect()
#readd this to the cluster
ClusterOperationHelper.add_all_nodes_or_assert(
master, [server], rest_settings, self)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
# Issue rest call to the newly added node
# MB-5108
if do_new_rest:
master = server
rest = RestConnection(master)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(
rebalanceStarted,
"unable to start rebalance on master node {0}".format(
master.ip))
log.info(
'started rebalance operation on master node {0}'.format(
master.ip))
rebalanceSucceeded = rest.monitorRebalance()
self.assertTrue(
rebalanceSucceeded,
"rebalance operation for nodes: {0} was not successful".
format(otpNodeIds))
#TODO: how can i verify that the cluster init config is preserved
# verify data on upgraded nodes
if create_buckets:
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(
'default', RestConnection(master)),
msg="bucket 'default' does not exist..")
if insert_data:
self._verify_data(master, rest, inserted_keys)
rest = RestConnection(master)
buckets = rest.get_buckets()
for bucket in buckets:
BucketOperationHelper.keys_exist_or_assert(
bucket_data[bucket.name]["inserted_keys"], master,
bucket.name, self)
def _do_cleanup(self):
for key in self._clusters_keys_olst:
nodes = self._clusters_dic[key]
BucketOperationHelper.delete_all_buckets_or_assert(nodes, self)
ClusterOperationHelper.cleanup_cluster(nodes)
ClusterOperationHelper.wait_for_ns_servers_or_assert(nodes, self)
def tearDown(self):
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
self._log_finish()
