def setUp(self):
super(AutoFailoverAbortsRebalance, self).setUp()
self.master = self.servers[0]
self._get_params()
self.rest = RestConnection(self.orchestrator)
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
self.num_buckets = self.num_buckets - 1 # this is done as default is created by base class
if self.num_buckets:
BucketOperationHelper.create_multiple_buckets(self.master, self.num_replicas, node_ram_ratio * (2.0 / 3.0),
howmany=self.num_buckets, bucket_storage=self.bucket_storage)
self.buckets = self.rest.get_buckets()
for bucket in self.buckets:
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket.name)
self.assertTrue(ready, "wait_for_memcached failed")
self.initial_load_gen = BlobGenerator('auto-failover',
'auto-failover-',
self.value_size,
end=self.num_items)
self.update_load_gen = BlobGenerator('auto-failover',
'auto-failover-',
self.value_size,
end=self.update_items)
self.delete_load_gen = BlobGenerator('auto-failover',
'auto-failover-',
self.value_size,
start=self.update_items,
end=self.delete_items)
self._load_all_buckets(self.servers[0], self.initial_load_gen,
"create", 0)
self._async_load_all_buckets(self.orchestrator,
self.update_load_gen, "update", 0)
self._async_load_all_buckets(self.orchestrator,
self.delete_load_gen, "delete", 0)
def common_setup(input, testcase, bucket_ram_ratio=(2.8 / 3.0), replica=0):
log = logger.Logger.get_logger()
servers = input.servers
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
serverInfo = servers[0]
log.info('picking server : {0} as the master'.format(serverInfo))
#if all nodes are on the same machine let's have the bucket_ram_ratio as bucket_ram_ratio * 1/len(servers)
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(servers)
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
if "ascii" in TestInputSingleton.input.test_params\
and TestInputSingleton.input.test_params["ascii"].lower() == "true":
BucketOperationHelper.create_multiple_buckets(serverInfo, replica, node_ram_ratio * bucket_ram_ratio,
howmany=1, sasl=False)
else:
BucketOperationHelper.create_multiple_buckets(serverInfo, replica, node_ram_ratio * bucket_ram_ratio,
howmany=1, sasl=True)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(serverInfo, bucket.name)
testcase.assertTrue(ready, "wait_for_memcached failed")
def create_buckets(servers, testcase, howmany=1, replica=1, bucket_ram_ratio=(2.0 / 3.0)):
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(servers)
master = servers[0]
BucketOperationHelper.create_multiple_buckets(master, replica, node_ram_ratio * bucket_ram_ratio, howmany=howmany)
rest = RestConnection(master)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
testcase.assertTrue(ready, "wait_for_memcached failed")
def common_setup(self):
self.log = logger.Logger.get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
master = self.servers[0]
rest = RestConnection(master)
# Cleanup previous state
self.task_manager = None
rest.stop_rebalance()
RebalanceBaseTest.reset(self)
# Initialize test params
self.replica = self.input.param("replica", 1)
# By default we use keys-count for LoadTask
# Use keys-count=-1 to use load-ratio
self.keys_count = self.input.param("keys-count", 30000)
self.load_ratio = self.input.param("load-ratio", 6)
self.expiry_ratio = self.input.param("expiry-ratio", 0.1)
self.delete_ratio = self.input.param("delete-ratio", 0.1)
self.access_ratio = self.input.param("access-ratio", 0.8)
self.num_buckets = self.input.param("num-buckets", 1)
self.num_rebalance = self.input.param("num-rebalance", 1)
self.do_ascii = self.input.param("ascii", False)
self.do_verify = self.input.param("do-verify", True)
self.repeat = self.input.param("repeat", 1)
self.max_ops_per_second = self.input.param("max_ops_per_second", 500)
self.min_item_size = self.input.param("min_item_size", 128)
self.do_stop = self.input.param("do-stop", False)
self.skip_cleanup = self.input.param("skip-cleanup", False)
self.checkResidentRatio = self.input.param("checkResidentRatio", False)
self.activeRatio = self.input.param("activeRatio", 50)
self.replicaRatio = self.input.param("replicaRatio", 50)
self.case_number = self.input.param("case_number", 0)
self.log.info('picking server : {0} as the master'.format(master))
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(self.servers)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved * node_ram_ratio))
BucketOperationHelper.create_multiple_buckets(master, self.replica, node_ram_ratio * (2.0 / 3.0),
howmany=self.num_buckets, sasl=not self.do_ascii)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
self.assertTrue(ready, "wait_for_memcached failed")
# Initialize and start the taskManager
self.task_manager = taskmanager.TaskManager()
self.task_manager.start()
def _cluster_setup(self):
replicas = self.input.param("replicas", 1)
keys_count = self.input.param("keys-count", 0)
num_buckets = self.input.param("num-buckets", 1)
bucket_name = "default"
master = self.servers[0]
credentials = self.input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.reset_autofailover()
ClusterOperationHelper.add_and_rebalance(self.servers, True)
if num_buckets == 1:
bucket_ram = info.memoryQuota * 2 / 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replicas,
proxyPort=info.moxi)
else:
created = BucketOperationHelper.create_multiple_buckets(self.master, replicas, howmany=num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
for bucket in buckets:
inserted_keys_cnt = self.load_data(self.master, bucket.name, keys_count)
log.info('inserted {0} keys'.format(inserted_keys_cnt))
def setUp(self):
self.log = logger.Logger().get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.master = self.servers[0]
self.ip = self.master.ip
self.finished = False
self.keys = []
self.keycount = 0
self.failure_string = ""
self.cleanup()
rest = RestConnection(self.master)
info = rest.get_nodes_self()
self.port = info.moxi + 1
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
created = BucketOperationHelper.create_multiple_buckets(
self.master,
replica=1,
bucket_ram_ratio=(2.0 / 3.0),
howmany=10,
sasl=False)
self.assertTrue(created, "bucket creation failed")
ready = BucketOperationHelper.wait_for_memcached(
self.master, "bucket-0")
self.assertTrue(ready, "wait_for_memcached failed")
def setUp(self):
self.log = logger.Logger().get_logger()
self.input = TestInputSingleton.input
self.servers = self.input.servers
self.master = self.servers[0]
self.ip = self.master.ip
self.finished = False
self.keys = []
self.keycount = 0
self.failure_string = ""
self.cleanup()
rest = RestConnection(self.master)
info = rest.get_nodes_self()
self.port = info.moxi+1
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
created = BucketOperationHelper.create_multiple_buckets(self.master,
replica=1,
bucket_ram_ratio=(2.0 / 3.0),
howmany=10,
sasl=False)
self.assertTrue(created, "bucket creation failed")
ready = BucketOperationHelper.wait_for_memcached(self.master, "bucket-0")
self.assertTrue(ready, "wait_for_memcached failed")
def common_setUp(self, with_buckets):
ClusterOperationHelper.cleanup_cluster(self.servers)
server = self.servers[0]
if with_buckets:
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, test_case=self)
ok = BucketOperationHelper.create_multiple_buckets(server, 1)
if not ok:
self.fail("unable to create multiple buckets on this node : {0}".format(server))
def common_setUp(self, with_buckets):
ClusterOperationHelper.cleanup_cluster(self.servers)
server = self.servers[0]
if with_buckets:
BucketOperationHelper.delete_all_buckets_or_assert(self.servers, test_case=self)
ok = BucketOperationHelper.create_multiple_buckets(server, 1)
if not ok:
self.fail("unable to create multiple buckets on this node : {0}".format(server))
def _cluster_setup(self):
keys_count = self.input.param("keys-count", 0)
num_buckets = self.input.param("num-buckets", 1)
bucketType = self.input.param("bucketType", "ephemeral")
evictionPolicy = self.input.param("evictionPolicy",
"noEviction") # fullEviction
self.bucket_storage = self.input.param("bucket_storage", 'couchstore')
# master = self.servers[0]
# credentials = self.input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
memory = min(info.mcdMemoryReserved,
self.input.param("kv_memory", 1000))
rest.init_cluster_memoryQuota(memoryQuota=memory)
rest.reset_autoreprovision()
self._add_and_rebalance(self.servers, True)
if num_buckets == 1:
bucket_name = "default"
bucket_ram = info.memoryQuota * 2 // 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=self.replicas,
proxyPort=info.moxi,
bucketType=bucketType,
evictionPolicy=evictionPolicy,
storageBackend=self.bucket_storage)
else:
created = BucketOperationHelper.create_multiple_buckets(
self.master,
self.replicas,
howmany=num_buckets,
bucketType=bucketType,
evictionPolicy=evictionPolicy,
storageBackend=self.bucket_storage)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
for bucket in buckets:
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_count, rejected_count = self.load_bucket_and_return_the_keys(
servers=[self.master],
name=bucket.name,
# ram_load_ratio=0.02,
value_size_distribution=distribution,
write_only=True,
moxi=True,
number_of_threads=2,
number_of_items=keys_count)
self.loaded_items[bucket.name] = inserted_count
def _create_multiple_buckets(self, replica=1):
master = self.servers[0]
bucket_type = self.input.param('bucket-type', None)
if bucket_type == 'sasl':
created = BucketOperationHelper.create_multiple_buckets(master, replica,
howmany=self.num_buckets,
saslPassword="******")
else:
created = BucketOperationHelper.create_multiple_buckets(master, replica,
howmany=self.num_buckets,
saslPassword="")
self.assertTrue(created, "unable to create multiple buckets")
rest = RestConnection(master)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
def _create_multiple_buckets(self, replica=1):
master = self.servers[0]
created = BucketOperationHelper.create_multiple_buckets(master, replica, howmany=self.num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
rest = RestConnection(master)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
def _create_multiple_buckets(self, replica=1):
master = self.servers[0]
created = BucketOperationHelper.create_multiple_buckets(master, replica, howmany=self.num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
rest = RestConnection(master)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
def _cluster_setup(self):
log = logger.Logger.get_logger()
replicas = self._input.param("replicas", 1)
keys_count = self._input.param("keys-count", 0)
num_buckets = self._input.param("num-buckets", 1)
bucket_name = "default"
master = self._servers[0]
credentials = self._input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.reset_autofailover()
ClusterOperationHelper.add_all_nodes_or_assert(self.master,
self._servers,
credentials, self)
bucket_ram = info.memoryQuota * 2 / 3
if num_buckets == 1:
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replicas,
proxyPort=info.moxi)
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket_name)
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes],
ejectedNodes=[])
buckets = rest.get_buckets()
else:
created = BucketOperationHelper.create_multiple_buckets(
self.master, replicas, howmany=num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes],
ejectedNodes=[])
for bucket in buckets:
inserted_keys_cnt = self.load_data(self.master, bucket.name,
keys_count)
log.info('inserted {0} keys'.format(inserted_keys_cnt))
msg = "rebalance failed after adding these nodes {0}".format(nodes)
self.assertTrue(rest.monitorRebalance(), msg=msg)
self.assertTrue(ready, "wait_for_memcached failed")
def common_setup(input, testcase, bucket_ram_ratio=(2.8 / 3.0), replica=0):
log = logger.Logger.get_logger()
servers = input.servers
BucketOperationHelper.delete_all_buckets_or_assert(servers, testcase)
ClusterOperationHelper.cleanup_cluster(servers)
ClusterHelper.wait_for_ns_servers_or_assert(servers, testcase)
serverInfo = servers[0]
log.info('picking server : {0} as the master'.format(serverInfo))
#if all nodes are on the same machine let's have the bucket_ram_ratio as bucket_ram_ratio * 1/len(servers)
node_ram_ratio = BucketOperationHelper.base_bucket_ratio(servers)
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=int(info.mcdMemoryReserved *
node_ram_ratio))
if "ascii" in TestInputSingleton.input.test_params\
and TestInputSingleton.input.test_params["ascii"].lower() == "true":
BucketOperationHelper.create_multiple_buckets(serverInfo,
replica,
node_ram_ratio *
bucket_ram_ratio,
howmany=1,
sasl=False)
else:
BucketOperationHelper.create_multiple_buckets(serverInfo,
replica,
node_ram_ratio *
bucket_ram_ratio,
howmany=1,
sasl=True)
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(
serverInfo, bucket.name)
testcase.assertTrue(ready, "wait_for_memcached failed")
def _cluster_setup(self):
keys_count = self.input.param("keys-count", 0)
num_buckets = self.input.param("num-buckets", 1)
bucketType = self.input.param("bucketType", "ephemeral")
evictionPolicy = self.input.param("evictionPolicy", "noEviction") # fullEviction
# master = self.servers[0]
# credentials = self.input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
memory = min(info.mcdMemoryReserved, self.input.param("kv_memory", 1000))
rest.init_cluster_memoryQuota(memoryQuota=memory)
rest.reset_autoreprovision()
self._add_and_rebalance(self.servers, True)
if num_buckets == 1:
bucket_name = "default"
bucket_ram = info.memoryQuota * 2 / 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=self.replicas,
proxyPort=info.moxi,
bucketType=bucketType,
evictionPolicy=evictionPolicy)
else:
created = BucketOperationHelper.create_multiple_buckets(
self.master, self.replicas, howmany=num_buckets,
bucketType=bucketType, evictionPolicy=evictionPolicy)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
for bucket in buckets:
distribution = {10: 0.2, 20: 0.5, 30: 0.25, 40: 0.05}
inserted_keys, rejected_keys = self.load_bucket_and_return_the_keys(servers=[self.master],
name=bucket.name,
# ram_load_ratio=0.02,
value_size_distribution=distribution,
write_only=True,
moxi=True,
number_of_threads=2,
number_of_items=keys_count)
self.loaded_items[bucket.name] = inserted_keys
def _cluster_setup(self):
log = logger.Logger.get_logger()
replicas = self._input.param("replicas", 1)
keys_count = self._input.param("keys-count", 0)
num_buckets = self._input.param("num-buckets", 1)
bucket_name = "default"
master = self._servers[0]
credentials = self._input.membase_settings
rest = RestConnection(master)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.reset_autofailover()
ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
bucket_ram = info.memoryQuota * 2 / 3
if num_buckets == 1:
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replicas,
proxyPort=info.moxi)
ready = BucketOperationHelper.wait_for_memcached(master, bucket_name)
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
buckets = rest.get_buckets()
else:
created = BucketOperationHelper.create_multiple_buckets(master, replicas, howmany=num_buckets)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
# self.load_data(master, bucket_name, keys_count)
for bucket in buckets:
inserted_keys_cnt = self.load_data(master, bucket.name, keys_count)
log.info('inserted {0} keys'.format(inserted_keys_cnt))
msg = "rebalance failed after adding these nodes {0}".format(nodes)
self.assertTrue(rest.monitorRebalance(), msg=msg)
self.assertTrue(ready, "wait_for_memcached failed")
def _cluster_setup(self):
replicas = self.input.param("replicas", 1)
keys_count = self.input.param("keys-count", 0)
num_buckets = self.input.param("num-buckets", 1)
bucket_storage = self.input.param("bucket_storage", 'couchstore')
bucket_name = "default"
master = self.servers[0]
credentials = self.input.membase_settings
rest = RestConnection(self.master)
info = rest.get_nodes_self()
rest.init_cluster(username=self.master.rest_username,
password=self.master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.reset_autofailover()
ClusterOperationHelper.add_and_rebalance(self.servers, True)
if num_buckets == 1:
bucket_ram = info.memoryQuota * 2 // 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replicas,
proxyPort=info.moxi,
storageBackend=bucket_storage)
else:
created = BucketOperationHelper.create_multiple_buckets(
self.master,
replicas,
howmany=num_buckets,
bucket_ram_ratio=(1.0 / 4.0),
bucket_storage=bucket_storage)
self.assertTrue(created, "unable to create multiple buckets")
buckets = rest.get_buckets()
for bucket in buckets:
ready = BucketOperationHelper.wait_for_memcached(
self.master, bucket.name)
self.assertTrue(ready, msg="wait_for_memcached failed")
for bucket in buckets:
inserted_keys_cnt = self.load_data(self.master, bucket.name,
keys_count)
log.info('inserted {0} keys'.format(inserted_keys_cnt))
def _create_load_multiple_bucket(self, server, bucket_data, howmany=2):
created = BucketOperationHelper.create_multiple_buckets(server, 1, howmany=howmany)
self.assertTrue(created, "unable to create multiple buckets")
rest = RestConnection(server)
buckets = rest.get_buckets()
for bucket in buckets:
bucket_data[bucket.name] = {}
ready = BucketOperationHelper.wait_for_memcached(server, bucket.name)
self.assertTrue(ready, "wait_for_memcached failed")
#let's insert some data
distribution = {2 * 1024: 0.5, 20: 0.5}
bucket_data[bucket.name]["inserted_keys"], bucket_data[bucket.name]["reject_keys"] =\
MemcachedClientHelper.load_bucket_and_return_the_keys(servers=[server], name=bucket.name,
ram_load_ratio=2.0,
number_of_threads=2,
value_size_distribution=distribution,
write_only=True,
moxi=True)
RebalanceHelper.wait_for_stats(server, bucket.name, 'ep_queue_size', 0)
RebalanceHelper.wait_for_stats(server, bucket.name, 'ep_flusher_todo', 0)
def _create_load_multiple_bucket(self, server, bucket_data, howmany=2):
created = BucketOperationHelper.create_multiple_buckets(server,
1, howmany=howmany)
self.assertTrue(created, "unable to create multiple buckets")
rest = RestConnection(server)
buckets = rest.get_buckets()
for bucket in buckets:
bucket_data[bucket.name] = {}
ready = BucketOperationHelper.wait_for_memcached(server, bucket.name)
self.assertTrue(ready, "wait_for_memcached failed")
#let's insert some data
distribution = {2 * 1024: 0.5, 20: 0.5}
bucket_data[bucket.name]["inserted_keys"], \
bucket_data[bucket.name]["reject_keys"] = \
MemcachedClientHelper.load_bucket_and_return_the_keys(servers=[server],
name=bucket.name, ram_load_ratio=2.0, number_of_threads=2,
value_size_distribution=distribution, write_only=True,
moxi=True)
RebalanceHelper.wait_for_stats(server, bucket.name, 'ep_queue_size', 0)
RebalanceHelper.wait_for_stats(server, bucket.name, 'ep_flusher_todo',
0)
