def insert_docs(self, num_of_docs, prefix, extra_values={}, wait_for_persistence=True, return_docs=False):
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {"type": "Point", "coordinates": [random.randrange(-180, 180), random.randrange(-90, 90)]}
value = {"name": doc_name, "age": random.randrange(1, 1000), "geometry": geom}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("waiting 5 seconds. error {0}".format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
if wait_for_persistence:
self.wait_for_persistence()
self.log.info("inserted {0} json documents".format(num_of_docs))
return doc_names
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND,
'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(
err == ERR_NOT_FOUND,
'was able to replace cas on removed key {0}'.format(KEY_NAME))
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def test_CASnotzero(self):
# MB-31149
# observe.observeseqnotests.ObserveSeqNoTests.test_CASnotzero
# set value, append and check CAS value
self.log.info('Starting test_CASnotzero')
# without hello(mutationseqencenumber)
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
KEY_NAME = "test1key"
client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
client.generic_request(
client.memcached(KEY_NAME).append, 'test1key', 'appended data')
get_meta_resp = client.generic_request(
client.memcached(KEY_NAME).getMeta, 'test1key')
self.log.info(
'the CAS value without hello(mutationseqencenumber): {} '.format(
get_meta_resp[4]))
self.assertNotEqual(get_meta_resp[4], 0)
# with hello(mutationseqencenumber)
KEY_NAME = "test2key"
client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value1'}))
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
client.generic_request(
client.memcached(KEY_NAME).append, 'test2key', 'appended data456')
get_meta_resp = client.generic_request(
client.memcached(KEY_NAME).getMeta, 'test2key')
self.log.info(
'the CAS value with hello(mutationseqencenumber): {} '.format(
get_meta_resp[4]))
self.assertNotEqual(get_meta_resp[4], 0)
def insert_docs(self,
num_of_docs,
prefix='doc',
extra_values={},
return_docs=False,
scope=None,
collection=None):
random.seed(12345)
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {
"type":
"Point",
"coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]
}
value = {
"name": doc_name,
"age": random.randrange(1, 1000),
"geometry": geom,
"height": random.randrange(1, 13000),
"bloom": random.randrange(1, 6),
"shed_leaves": random.randrange(6, 13)
}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key,
0,
0,
json.dumps(value),
scope=scope,
collection=collection)
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132
or e.status == 134) and fail_count < 60:
if i == 0:
self.log.error(
"waiting 5 seconds. error {0}".format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names
def insert_nested_docs(self,
num_of_docs,
prefix='doc',
levels=16,
size=512,
return_docs=False,
long_path=False,
scope=None,
collection=None):
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
dict = {'doc': {}, 'levels': levels}
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
if long_path:
self._createNestedJson_longPath(key, dict)
else:
self._createNestedJson(key, dict)
value = dict['doc']
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key,
0,
0,
json.dumps(value),
scope=scope,
collection=collection)
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133
or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error(
"waiting 5 seconds. error {0}".format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names
def insert_docs(self,
num_of_docs,
prefix,
extra_values={},
wait_for_persistence=True,
return_docs=False):
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {
"type":
"Point",
"coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]
}
value = {
"name": doc_name,
"age": random.randrange(1, 1000),
"geometry": geom
}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133
or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error(
"waiting 5 seconds. error {0}".format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
if wait_for_persistence:
self.wait_for_persistence()
self.log.info("inserted {0} json documents".format(num_of_docs))
return doc_names
def _test_view_on_multiple_docs(self, num_docs, params={"stale":"update_after"}, delay=10):
self.log.info("description : create a view on {0} documents".format(num_docs))
master = self.servers[0]
rest = RestConnection(master)
bucket = "default"
view_name = "dev_test_view_on_{1}_docs-{0}".format(str(uuid.uuid4())[:7], self.num_docs)
map_fn = "function (doc) {if(doc.name.indexOf(\"" + view_name + "\") != -1) { emit(doc.name, doc);}}"
rest.create_view(view_name, bucket, [View(view_name, map_fn, dev_view=False)])
self.created_views[view_name] = bucket
rest = RestConnection(self.servers[0])
smart = VBucketAwareMemcached(rest, bucket)
doc_names = []
prefix = str(uuid.uuid4())[:7]
total_time = 0
self.log.info("inserting {0} json objects".format(num_docs))
for i in range(0, num_docs):
key = doc_name = "{0}-{1}-{2}".format(view_name, prefix, i)
doc_names.append(doc_name)
value = {"name": doc_name, "age": 1000}
smart.set(key, 0, 0, json.dumps(value))
self.log.info("inserted {0} json documents".format(len(doc_names)))
time.sleep(10)
results = ViewBaseTests._get_view_results(self, rest, bucket, view_name, len(doc_names), extra_params=params)
view_time = results['view_time']
keys = ViewBaseTests._get_keys(self, results)
RebalanceHelper.wait_for_persistence(master, bucket, 0)
total_time = view_time
# Keep trying this for maximum 5 minutes
start_time = time.time()
# increase timeout to 600 seconds for windows testing
while (len(keys) != len(doc_names)) and (time.time() - start_time < 900):
msg = "view returned {0} items , expected to return {1} items"
self.log.info(msg.format(len(keys), len(doc_names)))
self.log.info("trying again in {0} seconds".format(delay))
time.sleep(delay)
results = ViewBaseTests._get_view_results(self, rest, bucket, view_name, len(doc_names), extra_params=params)
view_time = results['view_time']
total_time += view_time
keys = ViewBaseTests._get_keys(self, results)
self.log.info("View time: {0} secs".format(total_time))
# Only if the lengths are not equal, look for missing keys
if len(keys) != len(doc_names):
not_found = list(set(doc_names) - set(keys))
ViewBaseTests._print_keys_not_found(self, not_found, 10)
self.fail("map function did not return docs for {0} keys".format(len(not_found)))
def load_docs(self, node, num_docs, bucket = 'default', password = '',
exp = 0, flags = 0):
client = VBucketAwareMemcached(RestConnection(node), bucket)
for i in range(num_docs):
key = "key%s"%i
rc = client.set(key, 0, 0, "value")
def _load_docs(self, num_docs, prefix, verify=True, bucket='default', expire=0, flag=0):
master = self.servers[0]
rest = RestConnection(master)
smart = VBucketAwareMemcached(rest, bucket)
doc_names = []
for i in range(0, num_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
doc_names.append(doc_name)
value = {"name": doc_name, "age": i}
smart.set(key, expire, flag, json.dumps(value))
# loop till value is set
RebalanceHelper.wait_for_persistence(master, bucket)
self.log.info("inserted {0} json documents".format(num_docs))
if verify:
ViewBaseTests._verify_keys(self, doc_names, prefix)
return doc_names
def insert_docs(self, num_of_docs, prefix='doc', extra_values={},
return_docs=False):
random.seed(12345)
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {"type": "Point", "coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]}
value = {
"name": doc_name,
"age": random.randrange(1, 1000),
"geometry": geom,
"array" :[0,1,2,3,4,5,6,7,8,9,20],
"isDict" : True,
"dict_value" : {"name":"abc", "age":1},
"height": random.randrange(1, 13000),
"bloom": random.randrange(1, 6),
"shed_leaves": random.randrange(6, 13)}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("waiting 5 seconds. error {0}"
.format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names
def load_docs(self,
node,
num_docs,
bucket='default',
password='',
exp=0,
flags=0):
client = VBucketAwareMemcached(RestConnection(node), bucket)
for i in range(num_docs):
key = "key%s" % i
rc = client.set(key, 0, 0, "value")
def load(self, path, bucket, prefix='test'):
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
for file in os.listdir(path):
f = open(path + '/' + file, 'r')
rq_s = f.read()
f.close()
rq_json = json.loads(rq_s)
key = str(file)
try:
o, c, d = client.set(key, 0, 0, json.dumps(rq_json))
except Exception, ex:
print 'WARN======================='
print ex
def insert_nested_docs(self, num_of_docs, prefix='doc', levels=16, size=512, return_docs=False, long_path=False):
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
dict = {'doc' : {}, 'levels' : levels }
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
if long_path:
self._createNestedJson_longPath(key, dict)
else:
self._createNestedJson(key, dict)
value = dict['doc']
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("waiting 5 seconds. error {0}"
.format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names
def test_new_response_fields(self):
self.log.info('\n\nStarting test_new_response_fields')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
set_resp = self.extract_vbucket_uuid_and_seqno(
client.set('test1key', 0, 0, '123456789'))
# test the inplace operations
test = client.generic_request(
client.memcached('test1key').set, 'test1key', 0, 0,
'totally new value')
replace_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').replace, 'test1key', 0, 0,
'totally new value'))
self.verify_vbucket_and_seqno(set_resp, replace_resp, 'replace')
append_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').append, 'test1key',
'appended data'))
self.verify_vbucket_and_seqno(replace_resp, append_resp, 'append')
prepend_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').prepend, 'test1key',
'prepended data'))
self.verify_vbucket_and_seqno(append_resp, prepend_resp, 'prepend')
# and finally do the delete
delete_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').delete, 'test1key'))
self.verify_vbucket_and_seqno(set_resp, delete_resp, 'delete')
#meta commands under construction
# test the 'meta' commands
TEST_SEQNO = 123
TEST_CAS = 456
set_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').set_with_meta,
'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, '123456789')
set_meta_vbucket_uuid, set_meta_seqno = struct.unpack(
'>QQ', set_with_meta_resp[2])
set_with_meta_dict = {
'vbucket_uuid': set_meta_vbucket_uuid,
'seqno': set_meta_seqno
}
get_meta_resp = client.generic_request(
client.memcached('test1keyformeta').getMeta, 'test1keyformeta')
self.assertTrue(TEST_SEQNO == get_meta_resp[3], \
msg='get meta seqno does not match as set. Expected {0}, actual {1}'.format(TEST_SEQNO, get_meta_resp[3]) )
self.assertTrue(TEST_CAS == get_meta_resp[4], \
msg='get meta cas does not match as set. Expected {0}, actual {1}'.format(TEST_CAS, get_meta_resp[4]) )
# def del_with_meta(self, key, exp, flags, seqno, old_cas, new_cas, vbucket= -1):
del_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').del_with_meta,
'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, TEST_CAS + 1)
vbucket_uuid, seqno = struct.unpack('>QQ', del_with_meta_resp[2])
del_with_meta_dict = {'vbucket_uuid': vbucket_uuid, 'seqno': seqno}
self.verify_vbucket_and_seqno(set_with_meta_dict, del_with_meta_dict,
'set/del with meta')
# do some integer operations
set_resp = self.extract_vbucket_uuid_and_seqno(
client.set('key-for-integer-value', 0, 0, '123'))
incr_resp = client.generic_request(
client.memcached('key-for-integer-value').incr,
'key-for-integer-value')
incr_resp_dict = {'vbucket_uuid': incr_resp[2], 'seqno': incr_resp[3]}
self.verify_vbucket_and_seqno(set_resp, incr_resp_dict, 'incr')
decr_resp = client.generic_request(
client.memcached('key-for-integer-value').decr,
'key-for-integer-value')
decr_resp_dict = {'vbucket_uuid': decr_resp[2], 'seqno': decr_resp[3]}
self.verify_vbucket_and_seqno(incr_resp_dict, decr_resp_dict, 'decr')
add_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('totally new key').add, 'totally new key', 0,
0, 'totally new value'))
self.assertTrue(add_resp['vbucket_uuid'] > 0,
msg='Add request vbucket uuid is zero')
self.log.info('\n\nComplete test_new_response_fields\n\n')
def _run_observe(self):
tasks = []
query_set = "true"
persisted = 0
mutated = False
count = 0
for bucket in self.buckets:
self.cluster.create_view(self.master, self.default_design_doc,
self.default_view, bucket,
self.wait_timeout * 2)
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
self.max_time = timedelta(microseconds=0)
if self.mutate_by == "multi_set":
key_val = self._create_multi_set_batch()
client.setMulti(0, 0, key_val)
keys = ["observe%s" % (i) for i in range(self.num_items)]
for key in keys:
mutated = False
while not mutated and count < 60:
try:
if self.mutate_by == "set":
# client.memcached(key).set(key, 0, 0, "set")
client.set(key, 0, 0, "setvalue")
elif self.mutate_by == "append":
client.memcached(key).append(key, "append")
elif self.mutate_by == "prepend":
client.memcached(key).prepend(key, "prepend")
elif self.mutate_by == "incr":
client.memcached(key).incr(key, 1)
elif self.mutate_by == "decr":
client.memcached(key).decr(key)
mutated = True
t_start = datetime.now()
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error(
"Memcached error 134, wait for 5 seconds and then try again"
)
count += 1
time.sleep(5)
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(
key)
t_end = datetime.now()
#self.log.info("##########key:-%s################" % (key))
#self.log.info("Persisted:- %s" % (persisted))
#self.log.info("Persist_Time:- %s" % (rep_time))
#self.log.info("Time2:- %s" % (t_end - t_start))
if self.max_time <= (t_end - t_start):
self.max_time = (t_end - t_start)
self.log.info("Max Time taken for observe is :- %s" %
self.max_time)
self.log.info("Cas Value:- %s" % (cas))
query = {
"stale": "false",
"full_set": "true",
"connection_timeout": 600000
}
self.cluster.query_view(self.master,
"dev_Doc1",
self.default_view.name,
query,
self.num_items,
bucket,
timeout=self.wait_timeout)
self.log.info(
"Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
% (self.default_view, bucket))
# check whether observe has to run with delete and delete parallel with observe or not
if len(self.observe_with) > 0:
if self.observe_with == "delete":
self.log.info("Deleting 0- %s number of items" %
(self.num_items // 2))
self._load_doc_data_all_buckets('delete', 0,
self.num_items // 2)
query_set = "true"
elif self.observe_with == "delete_parallel":
self.log.info("Deleting Parallel 0- %s number of items" %
(self.num_items // 2))
tasks = self._async_load_doc_data_all_buckets(
'delete', 0, self.num_items // 2)
query_set = "false"
for key in keys:
opaque, rep_time, persist_time, persisted, cas = client.memcached(
key).observe(key)
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
if self.observe_with == "delete_parallel":
for task in tasks:
task.result()
query = {
"stale": "false",
"full_set": query_set,
"connection_timeout": 600000
}
self.cluster.query_view(self.master,
"dev_Doc1",
self.default_view.name,
query,
self.num_items // 2,
bucket,
timeout=self.wait_timeout)
self.log.info(
"Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
% (self.default_view, self.default_bucket_name))
"""test_observe_basic_data_load_delete will test observer basic scenario
def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
self.log.info(
'\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('failoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached('failoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be failed over'.format(
server_with_key))
RebalanceHelper.wait_for_persistence(self.master,
self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']))
# verify: no (hard) failover, everything else as before
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
time.sleep(30)
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('hardfailoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached('hardfailoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be hard failed over'.
format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(10)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
time.sleep(10)
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey', op_data['vbucket_uuid']))
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
def test_basic_operations(self):
self.log.info('\n\nStarting test_basic_operations')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
all_clients = []
for s in self.servers:
all_clients.append(
MemcachedClientHelper.direct_client(s,
self.default_bucket_name))
# basic case
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('test1key', 0, 0, 'test1value'))
op_data['format_type'] = 'no_failover'
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
o = client.observe_seqno('test1key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict(o)
self.check_results(op_data, results)
# 2. Disable persistence, Set a key, observe seqno, should not be persisted,
# enable and wait for persistence, observe seqno and check everything
self.log.info(
'\n\nVerify responses are correct when keys are not persisted')
for i in all_clients:
i.stop_persistence()
mc = MemcachedClientHelper.direct_client(self.master, "default")
self.log.info('setting the kv')
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('test2key', 0, 0, 'test2value'))
op_data['format_type'] = 'no_failover'
self.log.info('calling observe seq no')
o = client.observe_seqno('test2key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict(o)
# can't use check results because persisted is
self.assertTrue(
op_data['vbucket_uuid'] == results['new_vbucket_uuid'],
msg='Observe Vbucket uuid does not match. Expected: {0}. Actual {1}'
.format(hex(op_data['vbucket_uuid']),
hex(results['new_vbucket_uuid'])))
self.assertTrue(
op_data['seqno'] == results['current_seqno'],
msg='Observe seqno does not match. Expected: {0}. Actual {1}'.
format(op_data['seqno'], results['current_seqno']))
self.assertTrue(
op_data['seqno'] > results['last_persisted_seqno'],
msg='Persisted seqno is too big. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['last_persisted_seqno']))
self.log.info('starting persistence')
for s in all_clients:
s.start_persistence()
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
results = self.observe_seqno_response_to_dict(
client.observe_seqno('test2key', op_data['vbucket_uuid']))
self.check_results(op_data, results)
# error case - broken
"""
mc.set('badbuckettestkey', 0, 0, 'testvalue',1)
try:
o = client.observe_seqno('badbuckettestkey', 2)
self.fail('bucket is incorrect, should have returned an error')
except AssertionError, ex: # this is to catch the above fail, should it ever happen
raise
except Exception, ex:
traceback.print_exc()
if ex.status != memcacheConstants.ERR_NOT_FOUND:
self.log.info('Observe seqno incorrect error code for invalid bucket. Expected: {0}. Actual {1}'.format(
memcacheConstants.ERR_NOT_FOUND, ex.status))
raise Exception(ex)
"""
self.log.info('\n\nComplete test_basic_operations')
def _run_observe(self):
tasks = []
query_set = "true"
persisted = 0
mutated = False
count = 0
for bucket in self.buckets:
self.cluster.create_view(self.master, self.default_design_doc,
self.default_view, bucket , self.wait_timeout * 2)
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
self.max_time = timedelta(microseconds=0)
if self.mutate_by == "multi_set":
key_val = self._create_multi_set_batch()
client.setMulti(0, 0, key_val)
keys = ["observe%s" % (i) for i in xrange(self.num_items)]
for key in keys:
mutated = False
while not mutated and count < 60:
try:
if self.mutate_by == "set":
# client.memcached(key).set(key, 0, 0, "set")
client.set(key, 0, 0, "setvalue")
elif self.mutate_by == "append":
client.memcached(key).append(key, "append")
elif self.mutate_by == "prepend" :
client.memcached(key).prepend(key, "prepend")
elif self.mutate_by == "incr":
client.memcached(key).incr(key, 1)
elif self.mutate_by == "decr":
client.memcached(key).decr(key)
mutated = True
t_start = datetime.now()
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error("Memcached error 134, wait for 5 seconds and then try again")
count += 1
time.sleep(5)
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(key)
t_end = datetime.now()
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
self.log.info("Persist_Time:- %s" % (rep_time))
self.log.info("Time2:- %s" % (t_end - t_start))
if self.max_time <= (t_end - t_start):
self.max_time = (t_end - t_start)
self.log.info("Max Time taken for observe is :- %s" % self.max_time)
self.log.info("Cas Value:- %s" % (cas))
query = {"stale" : "false", "full_set" : "true", "connection_timeout" : 60000}
self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items, bucket, timeout=self.wait_timeout)
self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, bucket))
# check whether observe has to run with delete and delete parallel with observe or not
if len (self.observe_with) > 0 :
if self.observe_with == "delete" :
self.log.info("Deleting 0- %s number of items" % (self.num_items / 2))
self._load_doc_data_all_buckets('delete', 0, self.num_items / 2)
query_set = "true"
elif self.observe_with == "delete_parallel":
self.log.info("Deleting Parallel 0- %s number of items" % (self.num_items / 2))
tasks = self._async_load_doc_data_all_buckets('delete', 0, self.num_items / 2)
query_set = "false"
for key in keys:
opaque, rep_time, persist_time, persisted, cas = client.memcached(key).observe(key)
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
if self.observe_with == "delete_parallel":
for task in tasks:
task.result()
query = {"stale" : "false", "full_set" : query_set, "connection_timeout" : 60000}
self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items / 2, bucket, timeout=self.wait_timeout)
self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, self.default_bucket_name))
"""test_observe_basic_data_load_delete will test observer basic scenario
def insert_nested_specific_docs(self, num_of_docs, prefix='doc', extra_values={},
return_docs=False,collection=None):
random.seed(12345)
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {"type": "Point", "coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]}
value = {
"padding": None,
"d1" :{
"padding": None,
"d2" :{
"int_array" : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
"str_array" :["john", "doe", "john", "block", "jim", "john"],
"mix_array" : [1, 2, True, False, 'bird', 5.0, 6.0, `123`],
"d3" : {
"d4_01" : 1,
"d4_02" : [21, 22, 23, 24, 25 ],
"d4_03" : False,
"d4_04" : "San Francisco",
"d4_05" : {
"d5_01" : random.randrange(6, 13),
"d5_02" : [ random.randrange(5, 10), random.randrange(6, 13)],
"d5_03" : "abcdefghi",
"d5_04" : {
"d6_01" : random.randrange(6, 13),
"d6_02" : [1, 2, True, False, 'bird', 5.0, 6.0, `123`]
}
}
},
"d2_02" : {"d2_02_01":"name"},
"d2_03" :geom
},
"d1_02" :[1, 2, True, False, 'bird', 5.0, 6.0, `123`],
"d1_03" : False
},
"age": random.randrange(1, 1000),
"geometry": geom,
"array" :[0,1,2,3,4,5,6,7,8,9,20],
"isDict" : True,
"dict_value" : {"name":"abc", "age":1},
"height": random.randrange(1, 13000),
"bloom": random.randrange(1, 6),
"shed_leaves": random.randrange(6, 13)}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value),collection=collection)
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("waiting 5 seconds. error {0}"
.format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names
class GenericLoadingTask(Thread, Task):
def __init__(self, server, bucket, kv_store):
Thread.__init__(self)
Task.__init__(self, "load_gen_task")
self.kv_store = kv_store
self.client = VBucketAwareMemcached(RestConnection(server), bucket)
def execute(self, task_manager):
self.start()
self.state = EXECUTING
def check(self, task_manager):
pass
def run(self):
while self.has_next() and not self.done():
self.next()
self.state = FINISHED
self.set_result(True)
def has_next(self):
raise NotImplementedError
def next(self):
raise NotImplementedError
def _unlocked_create(self, partition, key, value):
try:
value_json = json.loads(value)
value_json['mutated'] = 0
value = json.dumps(value_json)
except ValueError:
index = random.choice(range(len(value)))
value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:]
try:
self.client.set(key, self.exp, 0, value)
partition.set(key, value, self.exp)
except MemcachedError as error:
self.state = FINISHED
self.set_exception(error)
def _unlocked_read(self, partition, key):
try:
o, c, d = self.client.get(key)
except MemcachedError as error:
if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None:
pass
else:
self.state = FINISHED
self.set_exception(error)
def _unlocked_update(self, partition, key):
value = partition.get_valid(key)
if value is None:
return
try:
value_json = json.loads(value)
value_json['mutated'] += 1
value = json.dumps(value_json)
except ValueError:
index = random.choice(range(len(value)))
value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:]
try:
self.client.set(key, self.exp, 0, value)
partition.set(key, value, self.exp)
except MemcachedError as error:
self.state = FINISHED
self.set_exception(error)
def _unlocked_delete(self, partition, key):
try:
self.client.delete(key)
partition.delete(key)
except MemcachedError as error:
if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None:
pass
else:
self.state = FINISHED
self.set_exception(error)
def test_basic_operations(self):
self.log.info('\n\nStarting test_basic_operations')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
all_clients = []
for s in self.servers:
all_clients.append( MemcachedClientHelper.direct_client(s, self.default_bucket_name))
# basic case
op_data = self.extract_vbucket_uuid_and_seqno( client.set('test1key', 0, 0, 'test1value') )
op_data['format_type'] = 'no_failover'
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
o = client.observe_seqno('test1key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict( o )
self.check_results( op_data, results)
# 2. Disable persistence, Set a key, observe seqno, should not be persisted,
# enable and wait for persistence, observe seqno and check everything
self.log.info('\n\nVerify responses are correct when keys are not persisted')
for i in all_clients:
i.stop_persistence()
mc = MemcachedClientHelper.direct_client(self.master, "default")
self.log.info('setting the kv')
op_data = self.extract_vbucket_uuid_and_seqno( client.set('test2key', 0, 0, 'test2value') )
op_data['format_type'] = 'no_failover'
self.log.info('calling observe seq no')
o = client.observe_seqno('test2key', op_data['vbucket_uuid'])
results = self.observe_seqno_response_to_dict( o )
# can't use check results because persisted is
self.assertTrue(op_data['vbucket_uuid'] == results['new_vbucket_uuid'],
msg='Observe Vbucket uuid does not match. Expected: {0}. Actual {1}'.format(
hex(op_data['vbucket_uuid']), hex(results['new_vbucket_uuid'])) )
self.assertTrue(op_data['seqno'] == results['current_seqno'],
msg='Observe seqno does not match. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['current_seqno']) )
self.assertTrue(op_data['seqno'] > results['last_persisted_seqno'],
msg='Persisted seqno is too big. Expected: {0}. Actual {1}'.format(
op_data['seqno'], results['last_persisted_seqno']) )
self.log.info('starting persistence')
for s in all_clients:
s.start_persistence()
for s in self.servers:
RebalanceHelper.wait_for_persistence(s, self.default_bucket_name)
results = self.observe_seqno_response_to_dict( client.observe_seqno('test2key', op_data['vbucket_uuid']) )
self.check_results( op_data, results)
# error case
mc.set('badbuckettestkey', 0, 0, 'testvalue',1)
try:
o = client.observe_seqno('badbuckettestkey', 2)
self.fail('bucket is incorrect, should have returned an error')
except AssertionError, ex: # this is to catch the above fail, should it ever happen
raise
def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
self.log.info('\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno( client.set('failoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nserver {0} has the key and it will be failed over'.format(server_with_key))
RebalanceHelper.wait_for_persistence(self.master, self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']) )
# verify: no (hard) failover, everything else as before
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
op_data = self.extract_vbucket_uuid_and_seqno( client.set('hardfailoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nserver {0} has the key and it will be hard failed over'.format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey',op_data['vbucket_uuid']) )
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
def test_new_response_fields(self):
self.log.info('\n\nStarting test_new_response_fields')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
set_resp = self.extract_vbucket_uuid_and_seqno( client.set('test1key', 0, 0, '123456789') )
# test the inplace operations
test = client.generic_request(client.memcached('test1key').set, 'test1key', 0, 0,'totally new value')
replace_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').replace, 'test1key', 0, 0,'totally new value') )
self.verify_vbucket_and_seqno( set_resp, replace_resp, 'replace')
append_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').append, 'test1key', 'appended data') )
self.verify_vbucket_and_seqno(replace_resp, append_resp, 'append')
prepend_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').prepend, 'test1key', 'prepended data') )
self.verify_vbucket_and_seqno(append_resp, prepend_resp, 'prepend')
# and finally do the delete
delete_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').delete,'test1key') )
self.verify_vbucket_and_seqno( set_resp, delete_resp, 'delete')
#meta commands under construction
# test the 'meta' commands
TEST_SEQNO = 123
TEST_CAS = 456
set_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').set_with_meta, 'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, '123456789')
set_meta_vbucket_uuid, set_meta_seqno = struct.unpack('>QQ', set_with_meta_resp[2])
set_with_meta_dict = {'vbucket_uuid':set_meta_vbucket_uuid, 'seqno': set_meta_seqno}
get_meta_resp = client.generic_request(client.memcached( 'test1keyformeta').getMeta, 'test1keyformeta')
self.assertTrue(TEST_SEQNO == get_meta_resp[3], \
msg='get meta seqno does not match as set. Expected {0}, actual {1}'.format(TEST_SEQNO,get_meta_resp[3]) )
self.assertTrue(TEST_CAS == get_meta_resp[4], \
msg='get meta cas does not match as set. Expected {0}, actual {1}'.format(TEST_CAS,get_meta_resp[4]) )
# def del_with_meta(self, key, exp, flags, seqno, old_cas, new_cas, vbucket= -1):
del_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').del_with_meta,'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, TEST_CAS+1)
vbucket_uuid, seqno = struct.unpack('>QQ', del_with_meta_resp[2])
del_with_meta_dict = {'vbucket_uuid':vbucket_uuid, 'seqno': seqno}
self.verify_vbucket_and_seqno( set_with_meta_dict, del_with_meta_dict, 'set/del with meta')
# do some integer operations
set_resp = self.extract_vbucket_uuid_and_seqno( client.set('key-for-integer-value', 0, 0, '123') )
incr_resp = client.generic_request(client.memcached('key-for-integer-value').incr, 'key-for-integer-value')
incr_resp_dict = {'vbucket_uuid':incr_resp[2], 'seqno':incr_resp[3]}
self.verify_vbucket_and_seqno(set_resp, incr_resp_dict, 'incr')
decr_resp = client.generic_request(client.memcached('key-for-integer-value').decr,'key-for-integer-value')
decr_resp_dict = {'vbucket_uuid':decr_resp[2], 'seqno':decr_resp[3]}
self.verify_vbucket_and_seqno(incr_resp_dict, decr_resp_dict, 'decr')
add_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('totally new key').add, 'totally new key', 0, 0,'totally new value') )
self.assertTrue( add_resp['vbucket_uuid'] > 0, msg='Add request vbucket uuid is zero')
self.log.info('\n\nComplete test_new_response_fields\n\n')
def insert_nested_specific_docs(self, num_of_docs, prefix='doc', extra_values={},
return_docs=False):
random.seed(12345)
rest = RestConnection(self.master)
smart = VBucketAwareMemcached(rest, self.bucket)
doc_names = []
for i in range(0, num_of_docs):
key = doc_name = "{0}-{1}".format(prefix, i)
geom = {"type": "Point", "coordinates":
[random.randrange(-180, 180),
random.randrange(-90, 90)]}
value = {
"padding": None,
"d1" :{
"padding": None,
"d2" :{
"int_array" : [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
"str_array" :["john", "doe", "john", "block", "jim", "john"],
"mix_array" : [1, 2, True, False, 'bird', 5.0, 6.0, `123`],
"d3" : {
"d4_01" : 1,
"d4_02" : [21, 22, 23, 24, 25 ],
"d4_03" : False,
"d4_04" : "San Francisco",
"d4_05" : {
"d5_01" : random.randrange(6, 13),
"d5_02" : [ random.randrange(5, 10), random.randrange(6, 13)],
"d5_03" : "abcdefghi",
"d5_04" : {
"d6_01" : random.randrange(6, 13),
"d6_02" : [1, 2, True, False, 'bird', 5.0, 6.0, `123`]
}
}
},
"d2_02" : {"d2_02_01":"name"},
"d2_03" :geom
},
"d1_02" :[1, 2, True, False, 'bird', 5.0, 6.0, `123`],
"d1_03" : False
},
"age": random.randrange(1, 1000),
"geometry": geom,
"array" :[0,1,2,3,4,5,6,7,8,9,20],
"isDict" : True,
"dict_value" : {"name":"abc", "age":1},
"height": random.randrange(1, 13000),
"bloom": random.randrange(1, 6),
"shed_leaves": random.randrange(6, 13)}
value.update(extra_values)
if not return_docs:
doc_names.append(doc_name)
else:
doc_names.append(value)
# loop till value is set
fail_count = 0
while True:
try:
smart.set(key, 0, 0, json.dumps(value))
break
except MemcachedError as e:
fail_count += 1
if (e.status == 133 or e.status == 132) and fail_count < 60:
if i == 0:
self.log.error("waiting 5 seconds. error {0}"
.format(e))
time.sleep(5)
else:
self.log.error(e)
time.sleep(1)
else:
raise e
self.log.info("Inserted {0} json documents".format(num_of_docs))
return doc_names