def load_data(self, master, bucket, keys_count):
log = logger.Logger.get_logger()
inserted_keys_cnt = 0
while inserted_keys_cnt < keys_count:
keys_cnt, rejected_keys_cnt = MemcachedClientHelper.load_bucket(
servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True
)
inserted_keys_cnt += keys_cnt
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_queue_size", 0)
RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_flusher_todo", 0)
return inserted_keys_cnt
def load_data_for_buckets(rest, load_ratio, distribution, rebalanced_servers, bucket_data, test):
buckets = rest.get_buckets()
for bucket in buckets:
inserted_count, rejected_count = \
MemcachedClientHelper.load_bucket(name=bucket.name,
servers=rebalanced_servers,
ram_load_ratio=load_ratio,
value_size_distribution=distribution,
number_of_threads=1,
write_only=True,
moxi=True)
test.log.info('inserted {0} keys'.format(inserted_count))
bucket_data[bucket.name]["items_inserted_count"] += inserted_count
def load_data(self, master, bucket, keys_count):
log = logger.Logger.get_logger()
# gen_create = BlobGenerator("loadONE", "loadONE-", 256, start=0, end=keys_count)
# BaseTestCase._load_all_buckets(master, gen_create, "create", 0)
inserted_keys_cnt = 0
while inserted_keys_cnt < keys_count:
keys_cnt, rejected_keys_cnt = MemcachedClientHelper.load_bucket(
servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True
)
inserted_keys_cnt += keys_cnt
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_queue_size", 0)
RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_flusher_todo", 0)
return inserted_keys_cnt
def load_data(self, master, bucket, keys_count):
log = logger.Logger.get_logger()
inserted_keys_cnt = 0
while inserted_keys_cnt < keys_count:
keys_cnt, rejected_keys_cnt = \
MemcachedClientHelper.load_bucket(servers=[master],
name=bucket,
number_of_items=keys_count,
number_of_threads=5,
write_only=True)
inserted_keys_cnt += keys_cnt
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_queue_size', 0)
RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_flusher_todo', 0)
return inserted_keys_cnt
def load_data(self, master, bucket, keys_count):
log = logger.Logger.get_logger()
# gen_create = BlobGenerator("loadONE", "loadONE-", 256, start=0, end=keys_count)
# BaseTestCase._load_all_buckets(master, gen_create, "create", 0)
inserted_keys_cnt = 0
while inserted_keys_cnt < keys_count:
keys_cnt, rejected_keys_cnt =\
MemcachedClientHelper.load_bucket(servers=[master],
name=bucket,
number_of_items=keys_count,
number_of_threads=5,
write_only=True)
inserted_keys_cnt += keys_cnt
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_queue_size',
0)
RebalanceHelper.wait_for_stats_on_all(master, bucket,
'ep_flusher_todo', 0)
return inserted_keys_cnt
def load_data(self, master, bucket, keys_count):
inserted_keys_cnt = 0
repeat_count = 0
while inserted_keys_cnt < keys_count and repeat_count < 5:
keys_cnt, rejected_keys_cnt = \
MemcachedClientHelper.load_bucket(servers=[master],
name=bucket,
number_of_items=keys_count,
number_of_threads=5,
write_only=True)
inserted_keys_cnt += keys_cnt
if keys_cnt == 0:
repeat_count += 1
else:
repeat_count = 0
if repeat_count == 5:
log.exception("impossible to load data")
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_persistence(master, bucket)
return inserted_keys_cnt
def load_data(self, master, bucket, keys_count):
inserted_keys_cnt = 0
repeat_count = 0
while inserted_keys_cnt < keys_count and repeat_count < 5:
keys_cnt, rejected_keys_cnt = \
MemcachedClientHelper.load_bucket(servers=[master],
name=bucket,
number_of_items=keys_count,
number_of_threads=5,
write_only=True)
inserted_keys_cnt += keys_cnt
if keys_cnt == 0:
repeat_count += 1
else:
repeat_count = 0
if repeat_count == 5:
log.exception("impossible to load data")
log.info("wait until data is completely persisted on the disk")
RebalanceHelper.wait_for_persistence(master, bucket)
return inserted_keys_cnt
def load_buckets(server, name, get, threads, moxi):
distro = {500: 0.5, 1024: 0.5}
MemcachedClientHelper.load_bucket([server], name, -1, 10000000, distro,
threads, -1, get, moxi)
def load_buckets(server, name, get, threads, moxi):
distro = {500: 0.5, 1024: 0.5}
MemcachedClientHelper.load_bucket([server], name, -1, 10000000, distro, threads, -1, get, moxi)
def common_test_body(self, replica, failover_reason, load_ratio, age, max_nodes):
log = logger.Logger.get_logger()
bucket_name = "default"
log.info("replica : {0}".format(replica))
log.info("failover_reason : {0}".format(failover_reason))
log.info("load_ratio : {0}".format(load_ratio))
log.info("age : {0}".format(age))
log.info("max_nodes : {0}".format(max_nodes))
master = self._servers[0]
log.info('picking server : {0} as the master'.format(master))
rest = RestConnection(master)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
rest.update_autofailover_settings(True, age, max_nodes)
rest.reset_autofailover()
bucket_ram = info.memoryQuota * 2 / 3
rest.create_bucket(bucket=bucket_name,
ramQuotaMB=bucket_ram,
replicaNumber=replica,
proxyPort=info.moxi)
ready = BucketOperationHelper.wait_for_memcached(master, bucket_name)
self.assertTrue(ready, "wait_for_memcached failed")
credentials = self._input.membase_settings
log.info("inserting some items in the master before adding any nodes")
distribution = {512: 0.4, 1 * 1024: 0.59, 5 * 1024: 0.01}
if load_ratio > 10:
distribution = {5 * 1024: 0.4, 10 * 1024: 0.5, 20 * 1024: 0.1}
ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self)
nodes = rest.node_statuses()
rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[])
msg = "rebalance failed after adding these nodes {0}".format(nodes)
self.assertTrue(rest.monitorRebalance(), msg=msg)
inserted_count, rejected_count =\
MemcachedClientHelper.load_bucket(servers=self._servers,
ram_load_ratio=load_ratio,
value_size_distribution=distribution,
number_of_threads=1)
log.info('inserted {0} keys'.format(inserted_count))
nodes = rest.node_statuses()
# why are we in this while loop?
while (len(nodes) - replica) >= 1:
final_replication_state = RestHelper(rest).wait_for_replication(900)
msg = "replication state after waiting for up to 15 minutes : {0}"
self.log.info(msg.format(final_replication_state))
chosen = AutoFailoverBaseTest.choose_nodes(master, nodes, replica)
for node in chosen:
#let's do op
if failover_reason == 'stop_membase':
self.stop_membase(node)
log.info("10 seconds delay to wait for membase-server to shutdown")
#wait for 5 minutes until node is down
self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
msg="node status is not unhealthy even after waiting for 5 minutes")
elif failover_reason == "firewall":
self.enable_firewall(node)
self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300),
msg="node status is not unhealthy even after waiting for 5 minutes")
# list pre-autofailover stats
stats = rest.get_bucket_stats()
self.log.info("pre-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
AutoFailoverBaseTest.wait_for_failover_or_assert(master, replica, age, self)
# manually fail over any unhealthy:active nodes left, max that we should need to manually failover is replica-max_nodes
manual_failover_count = replica - max_nodes
for node in chosen:
self.log.info("checking {0}".format(node.ip))
if node.status.lower() == "unhealthy" and node.clusterMembership == "active":
msg = "node {0} not failed over and we are over out manual failover limit of {1}"
self.assertTrue(manual_failover_count > 0, msg.format(node.ip, (replica - max_nodes)))
self.log.info("manual failover {0}".format(node.ip))
rest.fail_over(node.id)
manual_failover_count -= 1
stats = rest.get_bucket_stats()
self.log.info("post-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
self.assertTrue(stats["curr_items"] == inserted_count, "failover completed but curr_items ({0}) does not match inserted items ({1})".format(stats["curr_items"], inserted_count))
log.info("10 seconds sleep after autofailover before invoking rebalance...")
time.sleep(10)
rest.rebalance(otpNodes=[node.id for node in nodes],
ejectedNodes=[node.id for node in chosen])
msg="rebalance failed while removing failover nodes {0}".format(chosen)
self.assertTrue(rest.monitorRebalance(), msg=msg)
nodes = rest.node_statuses()
if len(nodes) / (1 + replica) >= 1:
final_replication_state = RestHelper(rest).wait_for_replication(900)
msg = "replication state after waiting for up to 15 minutes : {0}"
self.log.info(msg.format(final_replication_state))
self.assertTrue(RebalanceHelper.wait_till_total_numbers_match(master,bucket_name,600),
msg="replication was completed but sum(curr_items) dont match the curr_items_total")
start_time = time.time()
stats = rest.get_bucket_stats()
while time.time() < (start_time + 120) and stats["curr_items"] != inserted_count:
self.log.info("curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
time.sleep(5)
stats = rest.get_bucket_stats()
RebalanceHelper.print_taps_from_all_nodes(rest, bucket_name)
self.log.info("curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count))
stats = rest.get_bucket_stats()
msg = "curr_items : {0} is not equal to actual # of keys inserted : {1}"
self.assertEquals(stats["curr_items"], inserted_count,
msg=msg.format(stats["curr_items"], inserted_count))
nodes = rest.node_statuses()
