def load_data(self, master, bucket, keys_count): log = logger.Logger.get_logger() inserted_keys_cnt = 0 while inserted_keys_cnt < keys_count: keys_cnt, rejected_keys_cnt = MemcachedClientHelper.load_bucket( servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True ) inserted_keys_cnt += keys_cnt log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_queue_size", 0) RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_flusher_todo", 0) return inserted_keys_cnt
def load_data_for_buckets(rest, load_ratio, distribution, rebalanced_servers, bucket_data, test): buckets = rest.get_buckets() for bucket in buckets: inserted_count, rejected_count = \ MemcachedClientHelper.load_bucket(name=bucket.name, servers=rebalanced_servers, ram_load_ratio=load_ratio, value_size_distribution=distribution, number_of_threads=1, write_only=True, moxi=True) test.log.info('inserted {0} keys'.format(inserted_count)) bucket_data[bucket.name]["items_inserted_count"] += inserted_count
def load_data(self, master, bucket, keys_count): log = logger.Logger.get_logger() # gen_create = BlobGenerator("loadONE", "loadONE-", 256, start=0, end=keys_count) # BaseTestCase._load_all_buckets(master, gen_create, "create", 0) inserted_keys_cnt = 0 while inserted_keys_cnt < keys_count: keys_cnt, rejected_keys_cnt = MemcachedClientHelper.load_bucket( servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True ) inserted_keys_cnt += keys_cnt log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_queue_size", 0) RebalanceHelper.wait_for_stats_on_all(master, bucket, "ep_flusher_todo", 0) return inserted_keys_cnt
def load_data(self, master, bucket, keys_count): log = logger.Logger.get_logger() inserted_keys_cnt = 0 while inserted_keys_cnt < keys_count: keys_cnt, rejected_keys_cnt = \ MemcachedClientHelper.load_bucket(servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True) inserted_keys_cnt += keys_cnt log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_queue_size', 0) RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_flusher_todo', 0) return inserted_keys_cnt
def load_data(self, master, bucket, keys_count): log = logger.Logger.get_logger() # gen_create = BlobGenerator("loadONE", "loadONE-", 256, start=0, end=keys_count) # BaseTestCase._load_all_buckets(master, gen_create, "create", 0) inserted_keys_cnt = 0 while inserted_keys_cnt < keys_count: keys_cnt, rejected_keys_cnt =\ MemcachedClientHelper.load_bucket(servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True) inserted_keys_cnt += keys_cnt log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_queue_size', 0) RebalanceHelper.wait_for_stats_on_all(master, bucket, 'ep_flusher_todo', 0) return inserted_keys_cnt
def load_data(self, master, bucket, keys_count): inserted_keys_cnt = 0 repeat_count = 0 while inserted_keys_cnt < keys_count and repeat_count < 5: keys_cnt, rejected_keys_cnt = \ MemcachedClientHelper.load_bucket(servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True) inserted_keys_cnt += keys_cnt if keys_cnt == 0: repeat_count += 1 else: repeat_count = 0 if repeat_count == 5: log.exception("impossible to load data") log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_persistence(master, bucket) return inserted_keys_cnt
def load_data(self, master, bucket, keys_count): inserted_keys_cnt = 0 repeat_count = 0 while inserted_keys_cnt < keys_count and repeat_count < 5: keys_cnt, rejected_keys_cnt = \ MemcachedClientHelper.load_bucket(servers=[master], name=bucket, number_of_items=keys_count, number_of_threads=5, write_only=True) inserted_keys_cnt += keys_cnt if keys_cnt == 0: repeat_count += 1 else: repeat_count = 0 if repeat_count == 5: log.exception("impossible to load data") log.info("wait until data is completely persisted on the disk") RebalanceHelper.wait_for_persistence(master, bucket) return inserted_keys_cnt
def load_buckets(server, name, get, threads, moxi): distro = {500: 0.5, 1024: 0.5} MemcachedClientHelper.load_bucket([server], name, -1, 10000000, distro, threads, -1, get, moxi)
def load_buckets(server, name, get, threads, moxi): distro = {500: 0.5, 1024: 0.5} MemcachedClientHelper.load_bucket([server], name, -1, 10000000, distro, threads, -1, get, moxi)
def common_test_body(self, replica, failover_reason, load_ratio, age, max_nodes): log = logger.Logger.get_logger() bucket_name = "default" log.info("replica : {0}".format(replica)) log.info("failover_reason : {0}".format(failover_reason)) log.info("load_ratio : {0}".format(load_ratio)) log.info("age : {0}".format(age)) log.info("max_nodes : {0}".format(max_nodes)) master = self._servers[0] log.info('picking server : {0} as the master'.format(master)) rest = RestConnection(master) info = rest.get_nodes_self() rest.init_cluster(username=master.rest_username, password=master.rest_password) rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved) rest.update_autofailover_settings(True, age, max_nodes) rest.reset_autofailover() bucket_ram = info.memoryQuota * 2 / 3 rest.create_bucket(bucket=bucket_name, ramQuotaMB=bucket_ram, replicaNumber=replica, proxyPort=info.moxi) ready = BucketOperationHelper.wait_for_memcached(master, bucket_name) self.assertTrue(ready, "wait_for_memcached failed") credentials = self._input.membase_settings log.info("inserting some items in the master before adding any nodes") distribution = {512: 0.4, 1 * 1024: 0.59, 5 * 1024: 0.01} if load_ratio > 10: distribution = {5 * 1024: 0.4, 10 * 1024: 0.5, 20 * 1024: 0.1} ClusterOperationHelper.add_all_nodes_or_assert(master, self._servers, credentials, self) nodes = rest.node_statuses() rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[]) msg = "rebalance failed after adding these nodes {0}".format(nodes) self.assertTrue(rest.monitorRebalance(), msg=msg) inserted_count, rejected_count =\ MemcachedClientHelper.load_bucket(servers=self._servers, ram_load_ratio=load_ratio, value_size_distribution=distribution, number_of_threads=1) log.info('inserted {0} keys'.format(inserted_count)) nodes = rest.node_statuses() # why are we in this while loop? while (len(nodes) - replica) >= 1: final_replication_state = RestHelper(rest).wait_for_replication(900) msg = "replication state after waiting for up to 15 minutes : {0}" self.log.info(msg.format(final_replication_state)) chosen = AutoFailoverBaseTest.choose_nodes(master, nodes, replica) for node in chosen: #let's do op if failover_reason == 'stop_membase': self.stop_membase(node) log.info("10 seconds delay to wait for membase-server to shutdown") #wait for 5 minutes until node is down self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300), msg="node status is not unhealthy even after waiting for 5 minutes") elif failover_reason == "firewall": self.enable_firewall(node) self.assertTrue(RestHelper(rest).wait_for_node_status(node, "unhealthy", 300), msg="node status is not unhealthy even after waiting for 5 minutes") # list pre-autofailover stats stats = rest.get_bucket_stats() self.log.info("pre-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count)) AutoFailoverBaseTest.wait_for_failover_or_assert(master, replica, age, self) # manually fail over any unhealthy:active nodes left, max that we should need to manually failover is replica-max_nodes manual_failover_count = replica - max_nodes for node in chosen: self.log.info("checking {0}".format(node.ip)) if node.status.lower() == "unhealthy" and node.clusterMembership == "active": msg = "node {0} not failed over and we are over out manual failover limit of {1}" self.assertTrue(manual_failover_count > 0, msg.format(node.ip, (replica - max_nodes))) self.log.info("manual failover {0}".format(node.ip)) rest.fail_over(node.id) manual_failover_count -= 1 stats = rest.get_bucket_stats() self.log.info("post-autofail - curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count)) self.assertTrue(stats["curr_items"] == inserted_count, "failover completed but curr_items ({0}) does not match inserted items ({1})".format(stats["curr_items"], inserted_count)) log.info("10 seconds sleep after autofailover before invoking rebalance...") time.sleep(10) rest.rebalance(otpNodes=[node.id for node in nodes], ejectedNodes=[node.id for node in chosen]) msg="rebalance failed while removing failover nodes {0}".format(chosen) self.assertTrue(rest.monitorRebalance(), msg=msg) nodes = rest.node_statuses() if len(nodes) / (1 + replica) >= 1: final_replication_state = RestHelper(rest).wait_for_replication(900) msg = "replication state after waiting for up to 15 minutes : {0}" self.log.info(msg.format(final_replication_state)) self.assertTrue(RebalanceHelper.wait_till_total_numbers_match(master,bucket_name,600), msg="replication was completed but sum(curr_items) dont match the curr_items_total") start_time = time.time() stats = rest.get_bucket_stats() while time.time() < (start_time + 120) and stats["curr_items"] != inserted_count: self.log.info("curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count)) time.sleep(5) stats = rest.get_bucket_stats() RebalanceHelper.print_taps_from_all_nodes(rest, bucket_name) self.log.info("curr_items : {0} versus {1}".format(stats["curr_items"], inserted_count)) stats = rest.get_bucket_stats() msg = "curr_items : {0} is not equal to actual # of keys inserted : {1}" self.assertEquals(stats["curr_items"], inserted_count, msg=msg.format(stats["curr_items"], inserted_count)) nodes = rest.node_statuses()