def verify_persistence(servers,
test,
keys_count=400000,
timeout_in_seconds=300):
log = logger.Logger.get_logger()
master = servers[0]
rest = RestConnection(master)
log.info("Verifying Persistence")
buckets = rest.get_buckets()
for bucket in buckets:
# Load some data
l_threads = MemcachedClientHelper.create_threads([master],
bucket.name, -1,
keys_count, {
1024: 0.50,
512: 0.50
}, 2, -1, True,
True)
[t.start() for t in l_threads]
# Do persistence verification
ready = ClusterOperationHelper.persistence_verification(
servers, bucket.name, timeout_in_seconds)
log.info("Persistence Verification returned ? {0}".format(ready))
log.info("waiting for persistence threads to finish...")
for t in l_threads:
t.aborted = True
for t in l_threads:
t.join()
log.info("persistence thread has finished...")
test.assertTrue(ready, msg="Cannot verify persistence")
def common_test_body(self, replica, load_ratio, timeout=10):
log = logger.Logger.get_logger()
start_time = time.time()
log.info("replica : {0}".format(replica))
log.info("load_ratio : {0}".format(load_ratio))
master = self._servers[0]
log.info('picking server : {0} as the master'.format(master))
rest = RestConnection(master)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.mcdMemoryReserved * 2 / 3
rest.create_bucket(bucket='default',
ramQuotaMB=bucket_ram,
replicaNumber=replica,
proxyPort=11211)
json_bucket = {'name': 'default', 'port': 11211, 'password': ''}
BucketOperationHelper.wait_for_memcached(master, json_bucket)
log.info("inserting some items in the master before adding any nodes")
distribution = {1024: 0.4, 2 * 1024: 0.5, 512: 0.1}
threads = MemcachedClientHelper.create_threads(
servers=[master],
value_size_distribution=distribution,
number_of_threads=len(self._servers),
number_of_items=400000000,
moxi=False,
write_only=True,
async_write=True)
for thread in threads:
thread.terminate_in_minutes = 24 * 60
thread.start()
while time.time() < (start_time + 60 * timeout):
#rebalance out step nodes
#let's add some items ?
nodes = rest.node_statuses()
delta = len(self._servers) - len(nodes)
if delta > 0:
if delta > 1:
how_many_add = Random().randint(1, delta)
else:
how_many_add = 1
self.log.info("going to add {0} nodes".format(how_many_add))
self.rebalance_in(how_many=how_many_add)
else:
self.log.info("all nodes already joined the clustr")
time.sleep(240)
RestHelper(rest).wait_for_replication(600)
#dont rebalance out if there are not too many nodes
if len(nodes) >= (3.0 / 4.0 * len(self._servers)):
nodes = rest.node_statuses()
how_many_out = Random().randint(1, len(nodes) - 1)
self.log.info("going to remove {0} nodes".format(how_many_out))
self.rebalance_out(how_many=how_many_out)
for t in threads:
t.aborted = True
t.join()
def common_test_body(self, replica, load_ratio, timeout=10):
log = logger.Logger.get_logger()
start_time = time.time()
log.info("replica : {0}".format(replica))
log.info("load_ratio : {0}".format(load_ratio))
master = self._servers[0]
log.info('picking server : {0} as the master'.format(master))
rest = RestConnection(master)
info = rest.get_nodes_self()
rest.init_cluster(username=master.rest_username,
password=master.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.mcdMemoryReserved * 2 / 3
rest.create_bucket(bucket='default',
ramQuotaMB=bucket_ram,
replicaNumber=replica,
proxyPort=11211)
json_bucket = {'name': 'default', 'port': 11211, 'password': ''}
BucketOperationHelper.wait_for_memcached(master, json_bucket)
log.info("inserting some items in the master before adding any nodes")
distribution = {1024: 0.4, 2 * 1024: 0.5, 512: 0.1}
threads = MemcachedClientHelper.create_threads(servers=[master],
value_size_distribution=distribution,
number_of_threads=len(self._servers),
number_of_items=400000000,
moxi=False,
write_only=True,
async_write=True)
for thread in threads:
thread.terminate_in_minutes = 24 * 60
thread.start()
while time.time() < ( start_time + 60 * timeout):
#rebalance out step nodes
#let's add some items ?
nodes = rest.node_statuses()
delta = len(self._servers) - len(nodes)
if delta > 0:
if delta > 1:
how_many_add = Random().randint(1, delta)
else:
how_many_add = 1
self.log.info("going to add {0} nodes".format(how_many_add))
self.rebalance_in(how_many=how_many_add)
else:
self.log.info("all nodes already joined the clustr")
time.sleep(240)
RestHelper(rest).wait_for_replication(600)
#dont rebalance out if there are not too many nodes
if len(nodes) >= (3.0 / 4.0 * len(self._servers)):
nodes = rest.node_statuses()
how_many_out = Random().randint(1, len(nodes) - 1)
self.log.info("going to remove {0} nodes".format(how_many_out))
self.rebalance_out(how_many=how_many_out)
for t in threads:
t.aborted = True
t.join()
def load_bucket_and_return_the_keys(servers=None,
name='default',
ram_load_ratio=-1,
number_of_items=-1,
value_size_distribution=None,
number_of_threads=1,
override_vBucketId=-1,
write_only=False,
moxi=True,
delete_ratio=0,
expiry_ratio=0):
inserted_keys = []
rejected_keys = []
log = logger.Logger.get_logger()
threads = MemcachedClientHelper.create_threads(
servers,
name,
ram_load_ratio,
number_of_items,
value_size_distribution,
number_of_threads,
override_vBucketId,
write_only=write_only,
moxi=moxi,
delete_ratio=delete_ratio,
expiry_ratio=expiry_ratio)
# we can start them!
for thread in threads:
thread.start()
log.info("waiting for all worker thread to finish their work...")
[thread.join() for thread in threads]
log.info("worker threads are done...")
inserted_count = 0
rejected_count = 0
deleted_count = 0
expired_count = 0
for thread in threads:
t_inserted, t_rejected = thread.keys_set()
inserted_count += thread.inserted_keys_count()
rejected_count += thread.rejected_keys_count()
deleted_count += thread._delete_count
expired_count += thread._expiry_count
inserted_keys.extend(t_inserted)
rejected_keys.extend(t_rejected)
msg = "inserted keys count : {0} , rejected keys count : {1}"
log.info(msg.format(inserted_count, rejected_count))
msg = "deleted keys count : {0} , expired keys count : {1}"
log.info(msg.format(deleted_count, expired_count))
return inserted_keys, rejected_keys
def threads_for_buckets(rest, load_ratio, distribution, rebalanced_servers, bucket_data, delete_ratio=0,
expiry_ratio=0):
buckets = rest.get_buckets()
for bucket in buckets:
threads = MemcachedClientHelper.create_threads(servers=rebalanced_servers,
name=bucket.name,
ram_load_ratio=load_ratio,
value_size_distribution=distribution,
number_of_threads=4,
delete_ratio=delete_ratio,
expiry_ratio=expiry_ratio)
[t.start() for t in threads]
bucket_data[bucket.name]["threads"] = threads
return bucket_data
def load_bucket_and_return_the_keys(servers=None,
name='default',
ram_load_ratio=-1,
number_of_items=-1,
value_size_distribution=None,
number_of_threads=1,
override_vBucketId=-1,
write_only=False,
moxi=True,
delete_ratio=0,
expiry_ratio=0):
inserted_keys = []
rejected_keys = []
log = logger.Logger.get_logger()
threads = MemcachedClientHelper.create_threads(servers,
name,
ram_load_ratio,
number_of_items,
value_size_distribution,
number_of_threads,
override_vBucketId,
write_only=write_only,
moxi=moxi,
delete_ratio=delete_ratio,
expiry_ratio=expiry_ratio)
# we can start them!
for thread in threads:
thread.start()
log.info("waiting for all worker thread to finish their work...")
[thread.join() for thread in threads]
log.info("worker threads are done...")
inserted_count = 0
rejected_count = 0
deleted_count = 0
expired_count = 0
for thread in threads:
t_inserted, t_rejected = thread.keys_set()
inserted_count += thread.inserted_keys_count()
rejected_count += thread.rejected_keys_count()
deleted_count += thread._delete_count
expired_count += thread._expiry_count
inserted_keys.extend(t_inserted)
rejected_keys.extend(t_rejected)
msg = "inserted keys count : {0} , rejected keys count : {1}"
log.info(msg.format(inserted_count, rejected_count))
msg = "deleted keys count : {0} , expired keys count : {1}"
log.info(msg.format(deleted_count, expired_count))
return inserted_keys, rejected_keys
def verify_persistence(servers, test, keys_count=400000, timeout_in_seconds=300):
log = logger.Logger.get_logger()
master = servers[0]
rest = RestConnection(master)
log.info("Verifying Persistence")
buckets = rest.get_buckets()
for bucket in buckets:
#Load some data
l_threads = MemcachedClientHelper.create_threads([master], bucket.name,
- 1, keys_count, {1024: 0.50, 512: 0.50}, 2, -1,
True, True)
[t.start() for t in l_threads]
# Do persistence verification
ready = ClusterOperationHelper.persistence_verification(servers, bucket.name, timeout_in_seconds)
log.info("Persistence Verification returned ? {0}".format(ready))
log.info("waiting for persistence threads to finish...")
for t in l_threads:
t.aborted = True
for t in l_threads:
t.join()
log.info("persistence thread has finished...")
test.assertTrue(ready, msg="Cannot verify persistence")
