def _verify_data(self, version):
#verify all the keys
#let's use vbucketaware
rest = RestConnection(self.servers[0])
moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name)
index = 0
all_verified = True
keys_failed = []
for key in self.updated_keys:
try:
index += 1
flag, keyx, value = moxi.get(key=key)
self.assertTrue(value.endswith(version),
msg='values do not match . key value should endwith {0}'.format(version))
except MemcachedError as error:
self.log.error(error)
self.log.error(
"memcachedError : {0} - unable to get a pre-inserted key : {0}".format(error.status, key))
keys_failed.append(key)
all_verified = False
# except :
# self.log.error("unknown errors unable to get a pre-inserted key : {0}".format(key))
# keys_failed.append(key)
# all_verified = False
self.assertTrue(all_verified,
'unable to verify #{0} keys'.format(len(keys_failed)))
def _dockey_data_ops_with_moxi(self, dockey="dockey", data_op="create"):
expected_rows = self.num_items
for bucket in self.buckets:
try:
client = MemcachedClientHelper.proxy_client(
self.master, bucket.name)
except Exception as ex:
self.log.exception(
"unable to create memcached client due to {0}..".format(
ex))
try:
for itr in range(self.num_items):
key = dockey + str(itr)
value = str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
expected_rows = 0
except Exception as ex:
self.log.exception(
"Received exception {0} while performing data op - {1}".
format(ex, data_op))
self._wait_for_stats_all_buckets(self.servers[:self.nodes_init])
if self.bucket_type != 'ephemeral':
# views not supported for ephemeral buckets
self._verify_with_views(expected_rows)
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii):
for bucket in buckets:
try:
client = MemcachedClientHelper.proxy_client(
server, bucket.name, force_ascii=use_ascii)
except Exception as ex:
self.err = "unable to create memcached client due to {0}..".format(
ex)
try:
for itr in xrange(items):
key = 'bucketflush' + str(itr)
value = 'bucketflush-' + str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
except MemcachedError as exp:
if exp.status != 134:
self.log.info("Unexpected Exception Caught - {0}".format(exp))
self.err = "Expected Exception Caught - {0}".format(exp)
except Exception as exp:
self.log.info("Unexpected Exception Caught - {0}".format(exp))
self.err = "Unexpected exception caught- {0}".format(exp)
else:
self.err = "All buckets may not have been flushed"
def _insert_data(self, howmany):
self.onenodemc = MemcachedClientHelper.proxy_client(self.master, "default")
items = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)]
for item in items:
self.onenodemc.set(item, 0, 0, item)
self.log.info("inserted {0} items".format(howmany))
self.onenodemc.close()
def test_MB_14288(self):
mc = MemcachedClientHelper.proxy_client(self.master, "default")
blob = bytearray(1024 * 1024 * 10)
mc.set("MB-14288", 0, 0, blob)
flags_v, cas_v, retrieved = mc.get("MB-14288")
if not blob == retrieved:
self.fail("It should be possible to store and retrieve values > 1M")
def insert_key(serverInfo, bucket_name, count, size):
client = MemcachedClientHelper.proxy_client(serverInfo, bucket_name)
value = MemcachedClientHelper.create_value("*", size)
for i in range(count * 1000):
key = "key_" + str(i)
flag = random.randint(1, 999)
client.set(key, 0, flag, value)
def incr_test(self, key, exp, flags, value, incr_amt, decr_amt, incr_time):
global update_value
serverInfo = self.master
client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name)
# self.log.info('Waitting 15 seconds for memcached started')
# time.sleep(15)
if key != 'no_key':
client.set(key, exp, flags, value)
if exp:
self.log.info('Wait {0} seconds for the key expired' .format(exp + 2))
time.sleep(exp + 2)
if decr_amt:
c, d = client.decr(key, decr_amt)
self.log.info('decr amt {0}' .format(c))
try:
i = 0
while i < incr_time:
update_value, cas = client.incr(key, incr_amt)
i += 1
self.log.info('incr {0} times with value {1}'.format(incr_time, incr_amt))
return update_value
except mc_bin_client.MemcachedError as error:
self.log.info('memcachedError : {0}'.format(error.status))
self.test.fail("unable to increment value: {0}".format(incr_amt))
def _verify_data(self, version):
#verify all the keys
#let's use vbucketaware
moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name)
index = 0
all_verified = True
keys_failed = []
for key in self.updated_keys:
try:
index += 1
flag, keyx, value = moxi.get(key=key)
self.assertTrue(value.endswith(version),
msg='values do not match . key value should endwith {0}'.format(version))
except MemcachedError as error:
self.log.error(error)
self.log.error(
"memcachedError : {0} - unable to get a pre-inserted key : {0}".format(error.status, key))
keys_failed.append(key)
all_verified = False
# except :
# self.log.error("unknown errors unable to get a pre-inserted key : {0}".format(key))
# keys_failed.append(key)
# all_verified = False
self.assertTrue(all_verified,
'unable to verify #{0} keys'.format(len(keys_failed)))
def _insert_data(self, howmany):
self.onenodemc = MemcachedClientHelper.proxy_client(
self.master, "default")
items = [
"{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)
]
for item in items:
self.onenodemc.set(item, 0, 0, item)
self.log.info("inserted {0} items".format(howmany))
self.onenodemc.close()
def _insert_data(self, howmany):
self.onenodemc = MemcachedClientHelper.proxy_client(self.master, "default")
items = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)]
try:
for item in items:
self.onenodemc.set(item, 0, 0, item)
self.log.info("inserted {0} items".format(howmany))
self.onenodemc.close()
except Exception as e:
print e
self.fail("could not set item to bucket")
def touch_test(self):
timeout = 900 # 15 minutes
stats_all_buckets = {}
payload = MemcachedClientHelper.create_value('*', self.value_size)
mc = MemcachedClientHelper.proxy_client(self.servers[0], "default")
prefix = "test_"
self.num_items = self.input.param("items", 10000)
k = 0
while k < 100:
key = "{0}{1}".format(prefix, k)
mc.set(key, 0, 0, payload)
k += 1
active_resident_threshold = 30
threshold_reached = False
end_time = time.time() + float(timeout)
while not threshold_reached and time.time() < end_time:
if int(mc.stats()["vb_active_perc_mem_resident"]
) >= active_resident_threshold:
self.log.info(
"vb_active_perc_mem_resident_ratio reached at %s " %
(mc.stats()["vb_active_perc_mem_resident"]))
items = self.num_items
self.num_items += self.input.param("items", 40000)
random_key = self.key_generator()
generate_load = BlobGenerator(random_key,
'%s-' % random_key,
self.value_size,
end=self.num_items)
self._load_all_buckets(self.servers[0],
generate_load,
"create",
0,
True,
batch_size=40000,
pause_secs=3)
else:
threshold_reached = True
self.log.info("DGM {0} state achieved!!!!".format(
active_resident_threshold))
if time.time() > end_time and int(mc.stats(
)["vb_active_perc_mem_resident"]) >= active_resident_threshold:
raise Exception("failed to load items into bucket")
""" at active resident ratio above, the first 100 keys
insert into bucket will be non resident. Then do touch command to test it """
self.log.info(
"Run touch command to test items which are in non resident.")
k = 0
while k < 100:
key = "{0}{1}".format(prefix, k)
try:
mc.touch(key, 0)
k += 1
except Exception as ex:
raise Exception(ex)
def keys_exist_or_assert(keys,
server,
bucket_name,
test,
queue=None,
scope=None,
collection=None):
# we should try out at least three times
log = logger.Logger.get_logger()
# verify all the keys
client = MemcachedClientHelper.proxy_client(server, bucket_name)
# populate key
retry = 1
keys_left_to_verify = []
keys_left_to_verify.extend(copy.deepcopy(keys))
log_count = 0
while retry < 6 and len(keys_left_to_verify) > 0:
msg = "trying to verify {0} keys - attempt #{1} : {2} keys left to verify"
log.info(msg.format(len(keys), retry, len(keys_left_to_verify)))
keys_not_verified = []
for key in keys_left_to_verify:
try:
client.get(key=key, scope=scope, collection=collection)
except mc_bin_client.MemcachedError as error:
keys_not_verified.append(key)
if log_count < 100:
log.error("key {0} does not exist because {1}".format(
key, error))
log_count += 1
retry += 1
keys_left_to_verify = keys_not_verified
if len(keys_left_to_verify) > 0:
log_count = 0
for key in keys_left_to_verify:
log.error("key {0} not found".format(key))
log_count += 1
if log_count > 100:
break
msg = "unable to verify {0} keys".format(len(keys_left_to_verify))
log.error(msg)
if test:
queue.put(False)
test.fail(msg=msg)
if queue is None:
return False
else:
queue.put(False)
log.info("verified that {0} keys exist".format(len(keys)))
if queue is None:
return True
else:
queue.put(True)
def _insert_data_till_stopped(self, bucket, prefix, number_of_items):
moxi = MemcachedClientHelper.proxy_client(self.servers[0], bucket)
inserted_index = []
while not self.shutdown_load_data:
for i in range(0, number_of_items):
key = doc_name = "{0}-{1}".format(prefix, i)
value = {"name": doc_name, "age": random.randint(0, 5000)}
try:
moxi.set(key, 0, 0, json.dumps(value))
inserted_index.append(i)
except Exception as ex:
self.log.error("unable to set item , error {0}".format(ex))
self.log.info("inserted {0} json documents".format(inserted_index))
def _insert_data_till_stopped(self, bucket, prefix, number_of_items):
moxi = MemcachedClientHelper.proxy_client(self.servers[0], bucket)
inserted_index = []
while not self.shutdown_load_data:
for i in range(0, number_of_items):
key = doc_name = "{0}-{1}".format(prefix, i)
value = {"name": doc_name, "age": random.randint(0, 5000)}
try:
moxi.set(key, 0, 0, json.dumps(value))
inserted_index.append(i)
except Exception as ex:
self.log.error("unable to set item , error {0}".format(ex))
self.log.info("inserted {0} json documents".format(inserted_index))
def insert_docs(self,
num_of_docs,
prefix,
extra_values={},
wait_for_persistence=True,
return_docs=False):
moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {
"type":
"Point",
"coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]
}
value = {
"name": doc_name,
"age": random.randrange(1, 1000),
"geometry": geom
}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
moxi.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133
or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error(
"moxi not fully restarted, "
"waiting 5 seconds. error {0}".format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
if wait_for_persistence:
self.wait_for_persistence()
self.log.info("inserted {0} json documents".format(num_of_docs))
return doc_names
def _insert_data(self, howmany):
self.onenodemc = MemcachedClientHelper.proxy_client(
self.master, "default")
items = [
"{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, howmany)
]
try:
for item in items:
self.onenodemc.set(item, 0, 0, item)
self.log.info("inserted {0} items".format(howmany))
self.onenodemc.close()
except Exception as e:
print(e)
self.fail("could not set item to bucket")
def _eject_items(self, item_count, prefix):
client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name)
time_start = time.time()
for i in range(item_count):
timeout_end = time.time() + 10
passed = False
while time.time() < timeout_end and not passed:
try:
client.evict_key(prefix + "_key_" + str(i))
passed = True
except Exception as e:
self.log.error("failed to eject key {0}, error: {1}".format(prefix + "_key_" + str(i), e))
time.sleep(2)
self.assertTrue(passed, "exit due to eject errors after {0} seconds".format(time.time() - time_start))
self.log.info("ejected {0} items in {1} seconds".format(item_count, time.time() - time_start))
def setUp(self):
self.log = logger.Logger.get_logger()
self.params = TestInputSingleton.input.test_params
self.master = TestInputSingleton.input.servers[0]
rest = RestConnection(self.master)
rest.init_cluster(self.master.rest_username, self.master.rest_password)
info = rest.get_nodes_self()
rest.init_cluster_memoryQuota(self.master.rest_username, self.master.rest_password,
memoryQuota=info.mcdMemoryReserved)
ClusterOperationHelper.cleanup_cluster([self.master])
ClusterOperationHelper.wait_for_ns_servers_or_assert([self.master], self)
self._create_default_bucket()
self.keys_cleanup = []
self.onenodemc = MemcachedClientHelper.direct_client(self.master, "default", timeout=600)
self.onenodemoxi = MemcachedClientHelper.proxy_client(self.master, "default", timeout=600)
def _load_items(self, item_count, expiration, prefix, vprefix=""):
flags = 0
client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name)
time_start = time.time()
for i in range(item_count):
timeout_end = time.time() + 10
passed = False
while time.time() < timeout_end and not passed:
try:
client.set(prefix + "_key_" + str(i), expiration, flags, vprefix + "_value_" + str(i))
passed = True
except Exception as e:
self.log.error("failed to set key {0}, error: {1}".format(prefix + "_key_" + str(i), e))
time.sleep(2)
self.assertTrue(passed, "exit due to set errors after {0} seconds".format(time.time() - time_start))
self.log.info("loaded {0} items in {1} seconds".format(item_count, time.time() - time_start))
def set_test(self, key, exp, flags, values):
serverInfo = self.master
client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name,)
# self.log.info('Waitting 15 seconds for memcached started')
# time.sleep(15)
for v in values:
for f in flags:
client.set(key, exp, f, v)
flags_v, cas_v, get_v = client.get(key)
if get_v == v:
if flags_v == f:
self.log.info('Flags is set to {0}; and when run get {1}'.format(f, flags_v))
else:
self.test.fail('FAILED. Flags is set to {0}; and when run get {1}'.format(f, flags_v))
self.log.info('Value is set {0}; and when run get {1}'.format(v, get_v))
else:
self.test.fail('FAILED. Value is set to {0}; and when run get {1}'.format(v, get_v))
def keys_exist_or_assert(keys, server, bucket_name, test, queue=None):
# we should try out at least three times
log = logger.Logger.get_logger()
# verify all the keys
client = MemcachedClientHelper.proxy_client(server, bucket_name)
# populate key
retry = 1
keys_left_to_verify = []
keys_left_to_verify.extend(copy.deepcopy(keys))
log_count = 0
while retry < 6 and len(keys_left_to_verify) > 0:
msg = "trying to verify {0} keys - attempt #{1} : {2} keys left to verify"
log.info(msg.format(len(keys), retry, len(keys_left_to_verify)))
keys_not_verified = []
for key in keys_left_to_verify:
try:
client.get(key=key)
except mc_bin_client.MemcachedError as error:
keys_not_verified.append(key)
if log_count < 100:
log.error("key {0} does not exist because {1}".format(key, error))
log_count += 1
retry += 1
keys_left_to_verify = keys_not_verified
if len(keys_left_to_verify) > 0:
log_count = 0
for key in keys_left_to_verify:
log.error("key {0} not found".format(key))
log_count += 1
if log_count > 100:
break
msg = "unable to verify {0} keys".format(len(keys_left_to_verify))
log.error(msg)
if test:
queue.put(False)
test.fail(msg=msg)
if queue is None:
return False
else:
queue.put(False)
log.info("verified that {0} keys exist".format(len(keys)))
if queue is None:
return True
else:
queue.put(True)
def _update_keys(self, version):
rejected_keys = []
#quit after updating max 100,000 keys
self.updated_keys = []
moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name)
for key in self.keys:
if len(self.updated_keys) > 10000:
break
value = '{0}'.format(version)
try:
moxi.append(key, value)
self.updated_keys.append(key)
except MemcachedError:
# self.log.error(error)
# self.log.error("unable to update key : {0} to bucket : {1}".format(key, client.vbucketId))
rejected_keys.append(key)
if len(rejected_keys) > 0:
self.log.error("unable to update {0} keys".format(len(rejected_keys)))
def decr_test(self, key, exp, flags, value, incr_amt, decr_amt, decr_time):
global update_value
serverInfo = self.master
client = MemcachedClientHelper.proxy_client(serverInfo, self.bucket_name)
if key != 'no_key':
client.set(key, exp, flags, value)
if exp:
self.log.info('Wait {0} seconds for the key expired' .format(exp + 2))
time.sleep(exp + 2)
if incr_amt:
c, d = client.incr(key, incr_amt)
self.log.info('incr amt {0}' .format(c))
i = 0
while i < decr_time:
update_value, cas = client.decr(key, decr_amt)
i += 1
self.log.info('decr {0} times with value {1}'.format(decr_time, decr_amt))
return update_value
def _get_items(self, item_count, prefix, vprefix=""):
client = MemcachedClientHelper.proxy_client(self.master, self.default_bucket_name)
time_start = time.time()
get_count = 0
last_error = ""
error_count = 0
for i in range(item_count):
try:
value = client.get(prefix + "_key_" + str(i))[2]
assert(value == vprefix + "_value_" + str(i))
get_count += 1
except Exception as e:
last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e)
error_count += 1
if error_count > 0:
self.log.error("got {0} errors, last error: {1}".format(error_count, last_error))
self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start))
return get_count
def _update_keys(self, version):
rejected_keys = []
#quit after updating max 100,000 keys
self.updated_keys = []
moxi = MemcachedClientHelper.proxy_client(self.servers[0], self.bucket_name)
for key in self.keys:
if len(self.updated_keys) > 10000:
break
value = '{0}'.format(version)
try:
moxi.append(key, value)
self.updated_keys.append(key)
except MemcachedError:
# self.log.error(error)
# self.log.error("unable to update key : {0} to bucket : {1}".format(key, client.vbucketId))
rejected_keys.append(key)
if len(rejected_keys) > 0:
self.log.error("unable to update {0} keys".format(len(rejected_keys)))
def delete_docs(self, num_of_docs, prefix):
moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = "{0}-{1}".format(prefix, i)
try:
moxi.delete(key)
except MemcachedError as e:
if e.args[0] == "Memcached error #1: Not found":
continue
else:
raise
doc_names.append(key)
self.wait_for_persistence(180)
self.log.info("deleted {0} json documents".format(len(doc_names)))
return doc_names
def delete_docs(self, num_of_docs, prefix):
moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = "{0}-{1}".format(prefix, i)
try:
moxi.delete(key)
except MemcachedError as e:
# Don't care if we try to delete a document that doesn't exist
if e.status == memcacheConstants.ERR_NOT_FOUND:
continue
else:
raise
doc_names.append(key)
self.wait_for_persistence(180)
self.log.info("deleted {0} json documents".format(len(doc_names)))
return doc_names
def delete_docs(self, num_of_docs, prefix):
moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = "{0}-{1}".format(prefix, i)
try:
moxi.delete(key)
except MemcachedError as e:
# Don't care if we try to delete a document that doesn't exist
if e.status == memcacheConstants.ERR_NOT_FOUND:
continue
else:
raise
doc_names.append(key)
self.wait_for_persistence(180)
self.log.info("deleted {0} json documents".format(len(doc_names)))
return doc_names
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii):
for bucket in buckets:
try:
client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii)
except Exception as ex:
self.log.error("unable to create memcached client due to {0}..".format(ex))
with self.assertRaises(MemcachedError) as exp:
for itr in xrange(items):
key = 'bucketflush' + str(itr)
value = 'bucketflush-' + str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
memcached_exception = exp.exception
self.assertEqual(memcached_exception.status, 134, msg="Unexpected Exception - {0}".format(memcached_exception))
self.log.info("Expected Exception Caught - {0}".format(memcached_exception))
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii):
for bucket in buckets:
try:
client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii)
except Exception as ex:
self.log.error("unable to create memcached client due to {0}..".format(ex))
with self.assertRaises(MemcachedError) as exp:
for itr in xrange(items):
key = "bucketflush" + str(itr)
value = "bucketflush-" + str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
memcached_exception = exp.exception
self.assertEqual(memcached_exception.status, 134, msg="Unexpected Exception - {0}".format(memcached_exception))
self.log.info("Expected Exception Caught - {0}".format(memcached_exception))
def _reader_thread(self, inserted_keys, bucket_data, moxi=False):
errors = []
rest = RestConnection(self._servers[0])
smartclient = None
for name in bucket_data:
for key in inserted_keys:
if moxi:
moxi = MemcachedClientHelper.proxy_client(self._servers[0], name)
else:
smartclient = VBucketAwareMemcached(rest, name)
try:
if moxi:
moxi.get(key)
else:
smartclient.memcached(key).get(key)
except Exception as ex:
errors.append({"error": ex, "key": key})
self.log.info(ex)
if not moxi:
smartclient.done()
smartclient = VBucketAwareMemcached(rest, name)
def _dockey_data_ops_with_moxi(self, dockey="dockey", data_op="create"):
expected_rows = self.num_items
for bucket in self.buckets:
try:
client = MemcachedClientHelper.proxy_client(self.master, bucket.name)
except Exception as ex:
self.log.exception("unable to create memcached client due to {0}..".format(ex))
try:
for itr in xrange(self.num_items):
key = dockey + str(itr)
value = str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
expected_rows = 0
except Exception as ex:
self.log.exception("Received exception {0} while performing data op - {1}".format(ex, data_op))
self._wait_for_stats_all_buckets(self.servers[:self.nodes_init])
self._verify_with_views(expected_rows)
def touch_test(self):
timeout = 900 # 15 minutes
stats_all_buckets = {}
payload = MemcachedClientHelper.create_value('*', self.value_size)
mc = MemcachedClientHelper.proxy_client(self.servers[0], "default")
prefix = "test_"
self.num_items = self.input.param("items", 10000)
k = 0
while k < 100:
key = "{0}{1}".format(prefix, k)
mc.set(key, 0, 0, payload)
k += 1
active_resident_threshold = 30
threshold_reached = False
end_time = time.time() + float(timeout)
while not threshold_reached and time.time() < end_time:
if int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
self.log.info("vb_active_perc_mem_resident_ratio reached at %s " % (mc.stats()["vb_active_perc_mem_resident"]))
items = self.num_items
self.num_items += self.input.param("items", 40000)
random_key = self.key_generator()
generate_load = BlobGenerator(random_key, '%s-' % random_key, self.value_size, end=self.num_items)
self._load_all_buckets(self.servers[0], generate_load, "create", 0, True, batch_size=40000, pause_secs=3)
else:
threshold_reached = True
self.log.info("DGM {0} state achieved!!!!".format(active_resident_threshold))
if time.time() > end_time and int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
raise Exception("failed to load items into bucket")
""" at active resident ratio above, the first 100 keys
insert into bucket will be non resident. Then do touch command to test it """
self.log.info("Run touch command to test items which are in non resident.")
k = 0
while k < 100:
key = "{0}{1}".format(prefix, k)
try:
mc.touch(key, 0)
k += 1
except Exception as ex:
raise Exception(ex)
def insert_docs(self, num_of_docs, prefix, extra_values={},
wait_for_persistence=True, return_docs=False):
moxi = MemcachedClientHelper.proxy_client(self.master, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {"type": "Point", "coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]}
value = {"name": doc_name, "age": 1000, "geometry": geom}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
moxi.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("moxi not fully restarted, "
"waiting 5 seconds. error {0}"
.format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
if wait_for_persistence:
self.wait_for_persistence()
self.log.info("inserted {0} json documents".format(num_of_docs))
return doc_names
def data_ops_with_moxi(self, server, data_op, buckets, items, use_ascii):
for bucket in buckets:
try:
client = MemcachedClientHelper.proxy_client(server, bucket.name, force_ascii=use_ascii)
except Exception as ex:
self.log.error("unable to create memcached client due to {0}..".format(ex))
try:
for itr in xrange(items):
key = 'bucketflush' + str(itr)
value = 'bucketflush-' + str(itr)
if data_op in ["create", "update"]:
client.set(key, 0, 0, value)
elif data_op == "delete":
client.delete(key)
except MemcachedError as exp:
self.assertEqual(exp.status, 134, msg="Unexpected Exception - {0}".format(exp))
self.log.info("Expected Exception Caught - {0}".format(exp))
except Exception as exp:
self.log.info("Unxpected Exception Caught - {0}".format(exp))
self.fail("Unexpected exception caught- {0}".format(exp))
else:
self.fail("All buckets may not have been flushed")
expiry = 0
if "count" in params:
count = int(params["count"])
print count
if "size" in params:
size = int(params["size"])
if "prefix" in params:
prefix = params["prefix"]
#if "expiry" in params:
# expiry = int(params["expiry"])
payload = MemcachedClientHelper.create_value('*', size)
rest = RestConnection(server)
buckets = rest.get_buckets()
for bucket in buckets:
smart = VBucketAwareMemcached(rest, bucket.name)
mc = MemcachedClientHelper.proxy_client(server, bucket.name)
i = 0
counter_10 = 0
all_set = False
while i < count:
try:
key = "{0}-{1}".format(prefix, i)
#mc = smart.memcached(key)
#do an expiry every 10 times
if counter_10 >= 7:
if all_set == True:
mc.delete(key)
mc.set(key, 0, 0, payload)
if counter_10 == 10:
counter_10 = 0
else:
rest = RestConnection(server)
nodes = rest.node_statuses()
vm = VBucketAwareMemcached(rest,{"name":"bucket-0","password":""})
key = str(uuid.uuid4())
vm.memcached(key).set(key, 0, 0, "hi")
vm.memcached(key).get(key)
RebalanceHelper.print_taps_from_all_nodes(rest,bucket="bucket-0",password="")
RebalanceHelper.verify_items_count(server,"bucket-0")
RebalanceHelper.verify_items_count(server,"bucket-1")
RebalanceHelper.verify_items_count(server,"bucket-2")
RebalanceHelper.wait_till_total_numbers_match(server,"bucket-0",120,"")
cm = MemcachedClientHelper.proxy_client(server, "bucket-0", "")
key = str(uuid.uuid4())
cm.set(key, 0, 0, "hi")
cm.get(key)
cm1 = MemcachedClientHelper.direct_client(server, "default", "")
key = str(uuid.uuid4())
cm1.set(key, 0, 0, "hi")
cm1.get(key)
c1 = MemcachedClientHelper.proxy_client(server, "hello", "")
key = str(uuid.uuid4())
c1.set(key, 0, 0, "hi")
expiry = 0
if "count" in params:
count = int(params["count"])
print(count)
if "size" in params:
size = int(params["size"])
if "prefix" in params:
prefix = params["prefix"]
#if "expiry" in params:
# expiry = int(params["expiry"])
payload = MemcachedClientHelper.create_value('*', size)
rest = RestConnection(server)
buckets = rest.get_buckets()
for bucket in buckets:
smart = VBucketAwareMemcached(rest, bucket.name)
mc = MemcachedClientHelper.proxy_client(server, bucket.name)
i = 0
counter_10 = 0
all_set = False
while i < count:
try:
key = "{0}-{1}".format(prefix, i)
#mc = smart.memcached(key)
#do an expiry every 10 times
if counter_10 >= 7:
if all_set == True:
mc.delete(key)
mc.set(key, 0, 0, payload)
if counter_10 == 10:
counter_10 = 0
else:
server.port = 9000
rest = RestConnection(server)
nodes = rest.node_statuses()
vm = VBucketAwareMemcached(rest, {"name": "bucket-0", "password": ""})
key = str(uuid.uuid4())
vm.memcached(key).set(key, 0, 0, "hi")
vm.memcached(key).get(key)
RebalanceHelper.print_taps_from_all_nodes(rest, bucket="bucket-0", password="")
RebalanceHelper.verify_items_count(server, "bucket-0")
RebalanceHelper.verify_items_count(server, "bucket-1")
RebalanceHelper.verify_items_count(server, "bucket-2")
RebalanceHelper.wait_till_total_numbers_match(server, "bucket-0", 120, "")
cm = MemcachedClientHelper.proxy_client(server, "bucket-0", "")
key = str(uuid.uuid4())
cm.set(key, 0, 0, "hi")
cm.get(key)
cm1 = MemcachedClientHelper.direct_client(server, "default", "")
key = str(uuid.uuid4())
cm1.set(key, 0, 0, "hi")
cm1.get(key)
c1 = MemcachedClientHelper.proxy_client(server, "hello", "")
key = str(uuid.uuid4())
c1.set(key, 0, 0, "hi")
c1.get(key)
c2 = MemcachedClientHelper.direct_client(server, "hello", "")
