def corrupt_cas_is_healed_on_reboot(self):
self.log.info('Start corrupt_cas_is_healed_on_reboot')
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
# set a key
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':'value1'}))
# client.memcached(KEY_NAME).set('k2', 0, 0,json.dumps({'value':'value2'}))
# figure out which node it is on
mc_active = client.memcached(KEY_NAME)
# set the CAS to -2 and then mutate to increment to -1 and then it should stop there
self._corrupt_max_cas(mc_active,KEY_NAME)
# print 'max cas k2', mc_active.getMeta('k2')[4]
# CAS should be 0 now, do some gets and sets to verify that nothing bad happens
# self._restart_memcache('default')
remote = RemoteMachineShellConnection(self.master)
remote.stop_server()
time.sleep(30)
remote.start_server()
time.sleep(30)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
mc_active = client.memcached(KEY_NAME)
maxCas = mc_active.getMeta(KEY_NAME)[4]
self.assertTrue(maxCas == 0, 'max cas after reboot is not 0 it is {0}'.format(maxCas))
def corrupt_cas_is_healed_on_reboot(self):
self.log.info('Start corrupt_cas_is_healed_on_reboot')
KEY_NAME = 'key1'
rest = RestConnection(self.master)
# set a key
client = VBucketAwareMemcached(rest, 'default')
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,
json.dumps({'value': 'value1'}))
# client.memcached(KEY_NAME).set('k2', 0, 0,json.dumps({'value':'value2'}))
# figure out which node it is on
mc_active = client.memcached(KEY_NAME)
# set the CAS to -2 and then mutate to increment to -1 and then it should stop there
self._corrupt_max_cas(mc_active, KEY_NAME)
corrupt_cas = mc_active.getMeta(KEY_NAME)[4]
# self._restart_memcache('default')
remote = RemoteMachineShellConnection(self.master)
remote.stop_server()
time.sleep(30)
remote.start_server()
time.sleep(30)
client = VBucketAwareMemcached(rest, 'default')
mc_active = client.memcached(KEY_NAME)
curr_cas = mc_active.getMeta(KEY_NAME)[4]
self.assertTrue(
curr_cas == corrupt_cas,
'Corrupted cas (%s) != curr_cas (%s)' % (corrupt_cas, curr_cas))
def do_verification(kv_store, rest, bucket):
keys = kv_store.keys()
smart = VBucketAwareMemcached(rest, bucket)
validation_failures = {}
for k in keys:
expected = kv_store.read(k)
if expected:
if expected["status"] == "deleted":
# it should not exist there ?
try:
smart.memcached(k).get(k)
validation_failures[k] = ["deleted key"]
except MemcachedError:
pass
elif expected["status"] == "expired":
try:
smart.memcached(k).get(k)
validation_failures[k] = ["expired key"]
except MemcachedError:
pass
else:
try:
x, y, value = smart.memcached(k).get(k)
actualmd5 = hashlib.md5(value).digest()
if actualmd5 != expected["value"]:
validation_failures[k] = ["value mismatch"]
except MemcachedError:
validation_failures[k] = ["key not found"]
return validation_failures
def do_verification(kv_store, rest, bucket):
keys = kv_store.keys()
smart = VBucketAwareMemcached(rest, bucket)
validation_failures = {}
for k in keys:
expected = kv_store.read(k)
if expected:
if expected["status"] == "deleted":
# it should not exist there ?
try:
smart.memcached(k).get(k)
validation_failures[k] = ["deleted key"]
except MemcachedError:
pass
elif expected["status"] == "expired":
try:
smart.memcached(k).get(k)
validation_failures[k] = ["expired key"]
except MemcachedError:
pass
else:
try:
x, y, value = smart.memcached(k).get(k)
actualmd5 = hashlib.md5(value).digest()
if actualmd5 != expected["value"]:
validation_failures[k] = ["value mismatch"]
except MemcachedError:
validation_failures[k] = ["key not found"]
return validation_failures
def test_CASnotzero(self):
# MB-31149
# observe.observeseqnotests.ObserveSeqNoTests.test_CASnotzero
# set value, append and check CAS value
self.log.info('Starting test_CASnotzero')
# without hello(mutationseqencenumber)
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
KEY_NAME = "test1key"
client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
client.generic_request(
client.memcached(KEY_NAME).append, 'test1key', 'appended data')
get_meta_resp = client.generic_request(
client.memcached(KEY_NAME).getMeta, 'test1key')
self.log.info(
'the CAS value without hello(mutationseqencenumber): {} '.format(
get_meta_resp[4]))
self.assertNotEqual(get_meta_resp[4], 0)
# with hello(mutationseqencenumber)
KEY_NAME = "test2key"
client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value1'}))
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
client.generic_request(
client.memcached(KEY_NAME).append, 'test2key', 'appended data456')
get_meta_resp = client.generic_request(
client.memcached(KEY_NAME).getMeta, 'test2key')
self.log.info(
'the CAS value with hello(mutationseqencenumber): {} '.format(
get_meta_resp[4]))
self.assertNotEqual(get_meta_resp[4], 0)
def do_get_random_key(self):
# MB-31548, get_Random key gets hung sometimes.
self.log.info("Creating few docs in the bucket")
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
key = "test_docs-"
for index in range(1000):
doc_key = key + str(index)
client.memcached(doc_key).set(doc_key, 0, 0,
json.dumps({'value': 'value1'}))
self.log.info("Performing random_gets")
mc = MemcachedClient(self.master.ip, 11210)
mc.sasl_auth_plain(self.master.rest_username,
self.master.rest_password)
mc.bucket_select('default')
count = 0
while (count < 1000000):
count = count + 1
try:
mc.get_random_key()
except MemcachedError as error:
self.fail("<MemcachedError #%d ``%s''>" %
(error.status, error.message))
if count % 1000 == 0:
self.log.info('The number of iteration is {}'.format(count))
def insert_key(serverInfo, bucket_name, count, size):
rest = RestConnection(serverInfo)
smart = VBucketAwareMemcached(rest, bucket_name)
for i in xrange(count * 1000):
key = "key_" + str(i)
flag = random.randint(1, 999)
value = {"value": MemcachedClientHelper.create_value("*", size)}
smart.memcached(key).set(key, 0, 0, json.dumps(value))
def insert_key(serverInfo, bucket_name, count, size):
rest = RestConnection(serverInfo)
smart = VBucketAwareMemcached(rest, bucket_name)
for i in xrange(count * 1000):
key = "key_" + str(i)
flag = random.randint(1, 999)
value = {"value": MemcachedClientHelper.create_value("*", size)}
smart.memcached(key).set(key, 0, 0, json.dumps(value))
def test_meta_rebalance_out(self):
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,
json.dumps({'value': value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket(vbucket_id)
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int(
mc_active.stats('vbucket-details')[
'vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'])
self.assertTrue(
cas_active == max_cas,
'[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [],
[self.master])
rebalance.result()
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
get_meta_resp = mc_replica.getMeta(KEY_NAME,
request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:],
[self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(
replica_CAS == active_CAS,
'cas mismatch active: {0} replica {1}'.format(
active_CAS, replica_CAS))
def test_MB_32114(self):
try:
from sdk_client import SDKClient
except:
from sdk_client3 import SDKClient
import couchbase.subdocument as SD
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
if self.maxttl:
self._expiry_pager(self.master)
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.master.ip],
bucket='default')
KEY_NAME = 'key1'
for i in range(1000):
mcd = client.memcached(KEY_NAME + str(i))
rc = mcd.set(KEY_NAME + str(i), 0, 0,
json.dumps({'value': 'value2'}))
sdk_client.mutate_in(
KEY_NAME + str(i),
SD.upsert("subdoc_key",
"subdoc_val",
xattr=True,
create_parents=True))
# wait for it to persist
persisted = 0
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(
KEY_NAME + str(i))
start_time = time.time()
self._load_doc_data_all_buckets(batch_size=1000)
end_time = time.time()
for i in range(1000):
try:
mcd = client.memcached(KEY_NAME + str(i))
_, flags, exp, seqno, cas = client.memcached(
KEY_NAME + str(i)).getMeta(KEY_NAME + str(i))
rc = mcd.del_with_meta(KEY_NAME + str(i), 0, 0, 2, cas + 1)
except MemcachedError as exp:
self.fail("Exception with del_with meta - {0}".format(exp))
self.cluster.compact_bucket(self.master, "default")
if self.maxttl:
time_to_sleep = (self.maxttl - (end_time - start_time)) + 20
self.sleep(int(time_to_sleep))
else:
self.sleep(60)
active_bucket_items = rest.get_active_key_count("default")
replica_bucket_items = rest.get_replica_key_count("default")
print('active_bucket_items ', active_bucket_items)
print('replica_bucket_items ', replica_bucket_items)
if active_bucket_items * self.num_replicas != replica_bucket_items:
self.fail("Mismatch in data !!!")
def test_meta_failover(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
time.sleep(60)
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
#print 'replica CAS {0}'.format(replica_CAS)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def corrupt_cas_is_healed_on_rebalance_out_in(self):
self.log.info('Start corrupt_cas_is_healed_on_rebalance_out_in')
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
# set a key
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':'value1'}))
# figure out which node it is on
mc_active = client.memcached(KEY_NAME)
mc_replica = client.memcached( KEY_NAME, replica_index=0 )
# set the CAS to -2 and then mutate to increment to -1 and then it should stop there
self._corrupt_max_cas(mc_active,KEY_NAME)
# CAS should be 0 now, do some gets and sets to verify that nothing bad happens
resp = mc_active.get(KEY_NAME)
self.log.info( 'get for {0} is {1}'.format(KEY_NAME, resp))
# remove that node
self.log.info('Remove the node with -1 max cas')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master])
#rebalance = self.cluster.async_rebalance([self.master], [], self.servers[-1:])
rebalance.result()
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
#rebalance = self.cluster.async_rebalance([self.master], self.servers[-1:],[])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, 'default')
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
def corrupt_cas_is_healed_on_rebalance_out_in(self):
self.log.info('Start corrupt_cas_is_healed_on_rebalance_out_in')
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
# set a key
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':'value1'}))
# figure out which node it is on
mc_active = client.memcached(KEY_NAME)
mc_replica = client.memcached( KEY_NAME, replica_index=0 )
# set the CAS to -2 and then mutate to increment to -1 and then it should stop there
self._corrupt_max_cas(mc_active,KEY_NAME)
# CAS should be 0 now, do some gets and sets to verify that nothing bad happens
resp = mc_active.get(KEY_NAME)
self.log.info( 'get for {0} is {1}'.format(KEY_NAME, resp))
# remove that node
self.log.info('Remove the node with -1 max cas')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master])
#rebalance = self.cluster.async_rebalance([self.master], [], self.servers[-1:])
rebalance.result()
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
#rebalance = self.cluster.async_rebalance([self.master], self.servers[-1:],[])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, 'default')
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
def test_full_eviction_changed_to_value_eviction(self):
KEY_NAME = 'key1'
gen_create = BlobGenerator('eviction', 'eviction-', self.value_size, end=self.num_items)
gen_create2 = BlobGenerator('eviction2', 'eviction2-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, gen_create, "create", 0)
self._wait_for_stats_all_buckets(self.servers[:self.nodes_init])
self._verify_stats_all_buckets(self.servers[:self.nodes_init])
remote = RemoteMachineShellConnection(self.master)
for bucket in self.buckets:
output, _ = remote.execute_couchbase_cli(cli_command='bucket-edit',
cluster_host="localhost:8091",
user=self.master.rest_username,
password=self.master.rest_password,
options='--bucket=%s --bucket-eviction-policy=valueOnly' % bucket.name)
self.assertTrue(' '.join(output).find('SUCCESS') != -1, 'Eviction policy wasn\'t changed')
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers[:self.nodes_init], self,
wait_time=self.wait_timeout, wait_if_warmup=True)
self.sleep(10, 'Wait some time before next load')
# self._load_all_buckets(self.master, gen_create2, "create", 0)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
mcd = client.memcached(KEY_NAME)
try:
rc = mcd.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
self.fail('Bucket is incorrectly functional')
except MemcachedError as e:
pass # this is the exception we are hoping for
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def test_full_eviction_changed_to_value_eviction(self):
KEY_NAME = 'key1'
gen_create = BlobGenerator('eviction', 'eviction-', self.value_size, end=self.num_items)
gen_create2 = BlobGenerator('eviction2', 'eviction2-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, gen_create, "create", 0)
self._wait_for_stats_all_buckets(self.servers[:self.nodes_init])
self._verify_stats_all_buckets(self.servers[:self.nodes_init])
remote = RemoteMachineShellConnection(self.master)
for bucket in self.buckets:
output, _ = remote.execute_couchbase_cli(cli_command='bucket-edit',
cluster_host="localhost",
user=self.master.rest_username,
password=self.master.rest_password,
options='--bucket=%s --bucket-eviction-policy=valueOnly' % bucket.name)
self.assertTrue(' '.join(output).find('SUCCESS') != -1, 'Eviction policy wasn\'t changed')
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers[:self.nodes_init], self,
wait_time=self.wait_timeout, wait_if_warmup=True)
self.sleep(10, 'Wait some time before next load')
#self._load_all_buckets(self.master, gen_create2, "create", 0)
#import pdb;pdb.set_trace()
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
mcd = client.memcached(KEY_NAME)
try:
rc = mcd.set(KEY_NAME, 0,0, json.dumps({'value':'value2'}))
self.fail('Bucket is incorrectly functional')
except MemcachedError, e:
pass # this is the exception we are hoping for
def _getr_items(self, item_count, replica_count, prefix, vprefix=""):
time_start = time.time()
get_count = 0
last_error = ""
error_count = 0
awareness = VBucketAwareMemcached(self.rest, self.default_bucket_name)
for r in range(replica_count):
for i in range(item_count):
retry = True
key = prefix + "_key_" + str(i)
while retry:
client = awareness.memcached(key, r)
try:
value = client.getr(prefix + "_key_" + str(i))[2]
assert(value == vprefix + "_value_" + str(i))
get_count += 1
retry = False
except mc_bin_client.MemcachedError as e:
last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e)
error_count += 1
if e.status == 7:
self.log.info("getting new vbucket map {0}")
awareness.reset(self.rest)
else:
retry = False
except Exception as e:
last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e)
error_count += 1
retry = False
if error_count > 0:
self.log.error("got {0} errors, last error: {1}".format(error_count, last_error))
self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start))
awareness.done()
return get_count
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND,
'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(
err == ERR_NOT_FOUND,
'was able to replace cas on removed key {0}'.format(KEY_NAME))
def get_rev_info(rest_conn, bucket, keys):
vbmc = VBucketAwareMemcached(rest_conn,bucket)
ris = []
for k in keys:
mc = vbmc.memcached(k)
ri = mc.getRev(k)
ris.append(ri)
return ris
def test_meta_hard_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,
json.dumps({'value': value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket(vbucket_id)
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int(
mc_active.stats('vbucket-details')[
'vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'])
self.assertTrue(cas_pre == max_cas,
'[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# reboot nodes
self._reboot_server()
time.sleep(60)
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,
request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(
cas_pre == cas_post,
'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
def test_expiry_after_append(self):
# create a doc and set expiry for the doc
# append to the doc and check if the expiry is not changed
self.key = "expiry_after_append"
array = {
"name": "Douglas Reynholm",
"place": "India",
}
jsonDump = json.dumps(array)
self.client.set(self.key, 60, 0, jsonDump)
client1 = VBucketAwareMemcached(RestConnection(self.master), 'default')
get_meta_resp_before = client1.generic_request(client1.memcached(self.key).getMeta, self.key)
self.log.info("Sleeping for 5 sec")
time.sleep(5)
client1.generic_request(client1.memcached(self.key).append, self.key, 'appended data')
get_meta_resp_after = client1.generic_request(client1.memcached(self.key).getMeta, self.key)
self.assertEquals(get_meta_resp_before[2], get_meta_resp_after[2]) # 3rd value is expiry value
def test_MB_36087(self):
try:
from sdk_client import SDKClient
except:
from sdk_client3 import SDKClient
import couchbase.subdocument as SD
g_key = "test_doc"
bucket_name = "default"
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.master.ip],
bucket=bucket_name)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, bucket_name)
for i in range(self.num_items):
key = g_key + str(i)
mcd = client.memcached(key)
rc = mcd.set(key, 0, 0, json.dumps({'value': 'value2'}))
sdk_client.mutate_in(
key,
SD.upsert("subdoc_key",
"subdoc_val",
xattr=True,
create_parents=True))
# Wait for key to persist
persisted = 0
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = \
client.observe(key)
time.sleep(10)
# Evict the key
try:
rc = mcd.evict_key(key)
except MemcachedError as exp:
self.fail("Exception with evict meta - %s" % exp)
# Perform del_with_meta
try:
mcd = client.memcached(key)
_, flags, exp, seqno, cas = client.memcached(key).getMeta(key)
rc = mcd.del_with_meta(key, 0, 0, 2, cas + 1)
except MemcachedError as exp:
self.fail("Exception with del_with meta - {0}".format(exp))
def _verify_es_results(self, bucket='default'):
es_docs = self.esrest_conn.all_docs()
self.log.info("Retrieved ES Docs")
memcached_conn = VBucketAwareMemcached(self.rest, bucket)
self.log.info("Comparing CB and ES data")
for doc in es_docs:
es_data = doc['doc']
mc_active = memcached_conn.memcached(str(es_data['_id']))
cb_flags, cb_cas, cb_data = mc_active.get(str(es_data['_id']))
self.assertDictEqual(es_data, json.loads(cb_data), "Data mismatch found - es data: {0} cb data: {1}".
format(str(es_data), str(cb_data)))
self.log.info("Data verified")
def _reader_thread(self, inserted_keys, bucket_data, moxi=False):
errors = []
rest = RestConnection(self._servers[0])
smartclient = None
for name in bucket_data:
for key in inserted_keys:
if moxi:
moxi = MemcachedClientHelper.proxy_client(self._servers[0], name)
else:
smartclient = VBucketAwareMemcached(rest, name)
try:
if moxi:
moxi.get(key)
else:
smartclient.memcached(key).get(key)
except Exception as ex:
errors.append({"error": ex, "key": key})
self.log.info(ex)
if not moxi:
smartclient.done()
smartclient = VBucketAwareMemcached(rest, name)
def test_meta_hard_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# reboot nodes
self._reboot_server()
time.sleep(60)
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=True)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def _verify_es_results(self, bucket='default'):
esrest_conn = EsRestConnection(self.dest_master)
es_docs = esrest_conn.all_docs()
self.log.info("Retrieved ES Docs")
rest_conn = RestConnection(self.src_master)
memcached_conn = VBucketAwareMemcached(rest_conn, bucket)
self.log.info("Comparing CB and ES data")
for doc in es_docs:
es_data = doc['doc']
mc_active = memcached_conn.memcached(str(es_data['_id']))
cb_flags, cb_cas, cb_data = mc_active.get(str(es_data['_id']))
self.assertDictEqual(es_data, json.loads(cb_data), "Data mismatch found - es data: {0} cb data: {1}".
format(str(es_data), str(cb_data)))
self.log.info("Data verified")
def do_basic_ops(self):
KEY_NAME = 'key1'
KEY_NAME2 = 'key2'
CAS = 1234
self.log.info('Starting basic ops')
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
mcd = client.memcached(KEY_NAME)
# MB-17231 - incr with full eviction
rc = mcd.incr(KEY_NAME, 1)
print 'rc for incr', rc
# MB-17289 del with meta
rc = mcd.set(KEY_NAME, 0,0, json.dumps({'value':'value2'}))
print 'set is', rc
cas = rc[1]
# wait for it to persist
persisted = 0
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(KEY_NAME)
try:
rc = mcd.evict_key(KEY_NAME)
except MemcachedError as exp:
self.fail("Exception with evict meta - {0}".format(exp) )
CAS = 0xabcd
# key, value, exp, flags, seqno, remote_cas
try:
#key, exp, flags, seqno, cas
rc = mcd.del_with_meta(KEY_NAME2, 0, 0, 2, CAS)
except MemcachedError as exp:
self.fail("Exception with del_with meta - {0}".format(exp) )
def do_basic_ops(self):
KEY_NAME = 'key1'
KEY_NAME2 = 'key2'
CAS = 1234
self.log.info('Starting basic ops')
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, 'default')
mcd = client.memcached(KEY_NAME)
# MB-17231 - incr with full eviction
rc = mcd.incr(KEY_NAME, 1)
print('rc for incr', rc)
# MB-17289 del with meta
rc = mcd.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
print('set is', rc)
cas = rc[1]
# wait for it to persist
persisted = 0
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(
KEY_NAME)
time.sleep(10)
try:
rc = mcd.evict_key(KEY_NAME)
except MemcachedError as exp:
self.fail("Exception with evict meta - {0}".format(exp))
CAS = 0xabcd
# key, value, exp, flags, seqno, remote_cas
try:
#key, exp, flags, seqno, cas
rc = mcd.del_with_meta(KEY_NAME2, 0, 0, 2, CAS)
except MemcachedError as exp:
self.fail("Exception with del_with meta - {0}".format(exp))
def test_full_eviction_changed_to_value_eviction(self):
KEY_NAME = 'key1'
gen_create = BlobGenerator('superlongnameofkey1234567890123456789012345678902', 'eviction-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, gen_create, "create", 0)
self._wait_for_stats_all_buckets(self.servers[:self.nodes_init])
self._verify_stats_all_buckets(self.servers[:self.nodes_init])
total_keys_full_eviction = 0
for bucket in self.buckets:
total_keys_full_eviction += len(bucket.kvs[1].key_set()[0])
remote = RemoteMachineShellConnection(self.master)
for bucket in self.buckets:
output, _ = remote.execute_couchbase_cli(cli_command='bucket-edit',
cluster_host="localhost:8091",
user=self.master.rest_username,
password=self.master.rest_password,
options='--bucket=%s --bucket-eviction-policy=valueOnly' % bucket.name)
self.assertTrue(' '.join(output).find('SUCCESS') != -1, 'Eviction policy wasn\'t changed')
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers[:self.nodes_init], self,
wait_time=self.wait_timeout, wait_if_warmup=True)
total_keys_value_eviction = 0
for bucket in self.buckets:
total_keys_value_eviction += len(bucket.kvs[1].key_set()[0])
self.assertTrue(total_keys_full_eviction == total_keys_value_eviction, msg="Keys before and after eviction policy change from fullEviction to valueOnly differ")
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, "default")
mcd = client.memcached(KEY_NAME)
try:
rc = mcd.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'}))
# self.fail(rc) # Bucket is incorrectly functioning
except MemcachedError as e:
pass # this is the exception we are hoping for
def test_appendprepend(self):
# MB-32078, Append with CAS=0 can return ENGINE_KEY_EEXISTS
# observe.observeseqnotests.ObserveSeqNoTests.test_appendprepend
self.log.info('Starting test_appendprepend')
TEST_SEQNO = 123
TEST_CAS = 456
KEY_NAME = 'test_appendprepend'
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
# create a value with CAS
client.generic_request(
client.memcached(KEY_NAME).set_with_meta, KEY_NAME, 0, 0,
TEST_SEQNO, TEST_CAS, '123456789')
get_resp = client.generic_request(
client.memcached(KEY_NAME).get, KEY_NAME)
self.assertEqual(get_resp[2], '123456789')
# check if the changing the value fails
try:
client.generic_request(
client.memcached(KEY_NAME).set_with_meta, KEY_NAME, 0, 0, 0, 0,
'value')
self.fail('Expected to fail but passed')
except MemcachedError as exp:
self.assertEqual(int(exp.status), 2)
# check if append works fine
client.generic_request(
client.memcached(KEY_NAME).append, KEY_NAME, 'appended data')
get_resp = client.generic_request(
client.memcached(KEY_NAME).get, KEY_NAME)
self.assertEqual(get_resp[2], '123456789appended data')
# check if prepend works fine and verify the data
client.generic_request(
client.memcached(KEY_NAME).prepend, KEY_NAME, 'prepended data')
get_resp = client.generic_request(
client.memcached(KEY_NAME).get, KEY_NAME)
self.assertEqual(get_resp[2], 'prepended data123456789appended data')
def _run_observe(self):
tasks = []
query_set = "true"
persisted = 0
mutated = False
count = 0
for bucket in self.buckets:
self.cluster.create_view(self.master, self.default_design_doc,
self.default_view, bucket , self.wait_timeout * 2)
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
self.max_time = timedelta(microseconds=0)
if self.mutate_by == "multi_set":
key_val = self._create_multi_set_batch()
client.setMulti(0, 0, key_val)
keys = ["observe%s" % (i) for i in xrange(self.num_items)]
for key in keys:
mutated = False
while not mutated and count < 60:
try:
if self.mutate_by == "set":
# client.memcached(key).set(key, 0, 0, "set")
client.set(key, 0, 0, "setvalue")
elif self.mutate_by == "append":
client.memcached(key).append(key, "append")
elif self.mutate_by == "prepend" :
client.memcached(key).prepend(key, "prepend")
elif self.mutate_by == "incr":
client.memcached(key).incr(key, 1)
elif self.mutate_by == "decr":
client.memcached(key).decr(key)
mutated = True
t_start = datetime.now()
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error("Memcached error 134, wait for 5 seconds and then try again")
count += 1
time.sleep(5)
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(key)
t_end = datetime.now()
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
self.log.info("Persist_Time:- %s" % (rep_time))
self.log.info("Time2:- %s" % (t_end - t_start))
if self.max_time <= (t_end - t_start):
self.max_time = (t_end - t_start)
self.log.info("Max Time taken for observe is :- %s" % self.max_time)
self.log.info("Cas Value:- %s" % (cas))
query = {"stale" : "false", "full_set" : "true", "connection_timeout" : 60000}
self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items, bucket, timeout=self.wait_timeout)
self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, bucket))
# check whether observe has to run with delete and delete parallel with observe or not
if len (self.observe_with) > 0 :
if self.observe_with == "delete" :
self.log.info("Deleting 0- %s number of items" % (self.num_items / 2))
self._load_doc_data_all_buckets('delete', 0, self.num_items / 2)
query_set = "true"
elif self.observe_with == "delete_parallel":
self.log.info("Deleting Parallel 0- %s number of items" % (self.num_items / 2))
tasks = self._async_load_doc_data_all_buckets('delete', 0, self.num_items / 2)
query_set = "false"
for key in keys:
opaque, rep_time, persist_time, persisted, cas = client.memcached(key).observe(key)
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
if self.observe_with == "delete_parallel":
for task in tasks:
task.result()
query = {"stale" : "false", "full_set" : query_set, "connection_timeout" : 60000}
self.cluster.query_view(self.master, "dev_Doc1", self.default_view.name, query, self.num_items / 2, bucket, timeout=self.wait_timeout)
self.log.info("Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s" % (self.default_view, self.default_bucket_name))
"""test_observe_basic_data_load_delete will test observer basic scenario
class SyncReplicationTest(unittest.TestCase):
awareness = None
def common_setup(self, replica):
self._input = TestInputSingleton.input
self._servers = self._input.servers
first = self._servers[0]
self.log = logger.Logger().get_logger()
self.log.info(self._input)
rest = RestConnection(first)
for server in self._servers:
RestHelper(RestConnection(server)).is_ns_server_running()
ClusterOperationHelper.cleanup_cluster(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
ClusterOperationHelper.add_all_nodes_or_assert(self._servers[0], self._servers, self._input.membase_settings, self)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
info = rest.get_nodes_self()
bucket_ram = info.mcdMemoryReserved * 3 / 4
rest.create_bucket(bucket="default",
ramQuotaMB=int(bucket_ram),
replicaNumber=replica,
proxyPort=rest.get_nodes_self().moxi)
msg = "wait_for_memcached fails"
ready = BucketOperationHelper.wait_for_memcached(first, "default"),
self.assertTrue(ready, msg)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(rebalanceStarted,
"unable to start rebalance on master node {0}".format(first.ip))
self.log.info('started rebalance operation on master node {0}'.format(first.ip))
rebalanceSucceeded = rest.monitorRebalance()
# without a bucket this seems to fail
self.assertTrue(rebalanceSucceeded,
"rebalance operation for nodes: {0} was not successful".format(otpNodeIds))
self.awareness = VBucketAwareMemcached(rest, "default")
def tearDown(self):
if self.awareness:
self.awareness.done()
ClusterOperationHelper.cleanup_cluster(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
def test_one_replica(self):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 1024)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
mc.sync_replication(1, [{"key": k, "vbucket": vBucket}])
for k in keys:
mc = self.awareness.memcached(k)
mc.get(k)
def test_one_replica_one_node(self):
pass
def test_one_replica_multiple_nodes(self):
pass
def test_one_replica_bucket_replica_one(self):
pass
def test_two_replica(self):
self._unsupported_replicas(2)
def test_three_replica(self):
self._unsupported_replicas(1)
def _unsupported_replicas(self, replica):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 102400)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
mc.get(k)
try:
mc.sync_replication(replica, [{"key": k, "vbucket": vBucket}])
msg = "server did not raise an error when running sync_replication with {0} replicas"
self.fail(msg.format(replica))
except MemcachedError as error:
self.log.info("error {0} {1} as expected".format(error.status, error.msg))
for k in keys:
mc = self.awareness.memcached(k)
mc.get(k)
def test_invalid_key(self):
pass
def test_not_your_vbucket(self):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 1024)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
not_your_vbucket_mc = self.awareness.not_my_vbucket_memcached(k)
try:
count = 0
expected_error = 0
while count < 100:
a, b, response = not_your_vbucket_mc.sync_replication(1,
[{"key": k, "vbucket": vBucket}])
count += 1
self.log.info("response : {0}".format(response))
if response and response[0]["event"] != "invalid key":
expected_error += 1
if expected_error is not 100:
self.fail(msg="server did not raise an error when running sync_replication with invalid vbucket")
except MemcachedError as error:
self.log.error(error)
def test_some_invalid_keys(self):
pass
def stest_ome_not_your_vbucket(self):
pass
def test_some_large_values(self):
pass
def test_too_many_keys(self):
pass
def test_singlenode(self):
pass
def test_new_response_fields(self):
self.log.info('\n\nStarting test_new_response_fields')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
set_resp = self.extract_vbucket_uuid_and_seqno( client.set('test1key', 0, 0, '123456789') )
# test the inplace operations
test = client.generic_request(client.memcached('test1key').set, 'test1key', 0, 0,'totally new value')
replace_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').replace, 'test1key', 0, 0,'totally new value') )
self.verify_vbucket_and_seqno( set_resp, replace_resp, 'replace')
append_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').append, 'test1key', 'appended data') )
self.verify_vbucket_and_seqno(replace_resp, append_resp, 'append')
prepend_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').prepend, 'test1key', 'prepended data') )
self.verify_vbucket_and_seqno(append_resp, prepend_resp, 'prepend')
# and finally do the delete
delete_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('test1key').delete,'test1key') )
self.verify_vbucket_and_seqno( set_resp, delete_resp, 'delete')
#meta commands under construction
# test the 'meta' commands
TEST_SEQNO = 123
TEST_CAS = 456
set_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').set_with_meta, 'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, '123456789')
set_meta_vbucket_uuid, set_meta_seqno = struct.unpack('>QQ', set_with_meta_resp[2])
set_with_meta_dict = {'vbucket_uuid':set_meta_vbucket_uuid, 'seqno': set_meta_seqno}
get_meta_resp = client.generic_request(client.memcached( 'test1keyformeta').getMeta, 'test1keyformeta')
self.assertTrue(TEST_SEQNO == get_meta_resp[3], \
msg='get meta seqno does not match as set. Expected {0}, actual {1}'.format(TEST_SEQNO,get_meta_resp[3]) )
self.assertTrue(TEST_CAS == get_meta_resp[4], \
msg='get meta cas does not match as set. Expected {0}, actual {1}'.format(TEST_CAS,get_meta_resp[4]) )
# def del_with_meta(self, key, exp, flags, seqno, old_cas, new_cas, vbucket= -1):
del_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').del_with_meta,'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, TEST_CAS+1)
vbucket_uuid, seqno = struct.unpack('>QQ', del_with_meta_resp[2])
del_with_meta_dict = {'vbucket_uuid':vbucket_uuid, 'seqno': seqno}
self.verify_vbucket_and_seqno( set_with_meta_dict, del_with_meta_dict, 'set/del with meta')
# do some integer operations
set_resp = self.extract_vbucket_uuid_and_seqno( client.set('key-for-integer-value', 0, 0, '123') )
incr_resp = client.generic_request(client.memcached('key-for-integer-value').incr, 'key-for-integer-value')
incr_resp_dict = {'vbucket_uuid':incr_resp[2], 'seqno':incr_resp[3]}
self.verify_vbucket_and_seqno(set_resp, incr_resp_dict, 'incr')
decr_resp = client.generic_request(client.memcached('key-for-integer-value').decr,'key-for-integer-value')
decr_resp_dict = {'vbucket_uuid':decr_resp[2], 'seqno':decr_resp[3]}
self.verify_vbucket_and_seqno(incr_resp_dict, decr_resp_dict, 'decr')
add_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request( client.memcached('totally new key').add, 'totally new key', 0, 0,'totally new value') )
self.assertTrue( add_resp['vbucket_uuid'] > 0, msg='Add request vbucket uuid is zero')
self.log.info('\n\nComplete test_new_response_fields\n\n')
def verify_cas(self, ops, generator):
"""Verify CAS value manipulation.
For update we use the latest CAS value return by set()
to do the mutation again to see if there is any exceptions.
We should be able to mutate that item with the latest CAS value.
For delete(), after it is called, we try to mutate that item with the
cas vavlue returned by delete(). We should see Memcached Error. Otherwise
the test should fail.
For expire, We want to verify using the latest CAS value of that item
can not mutate it because it is expired already."""
for bucket in self.buckets:
client = VBucketAwareMemcached(RestConnection(self.master),
bucket.name)
gen = generator
cas_error_collection = []
data_error_collection = []
while gen.has_next():
key, value = next(gen)
if ops in ["update", "touch"]:
for x in range(self.mutate_times):
o_old, cas_old, d_old = client.get(key)
if ops == 'update':
client.memcached(key).cas(
key, 0, 0, cas_old,
"{0}-{1}".format("mysql-new-value", x))
else:
client.memcached(key).touch(key, 10)
o_new, cas_new, d_new = client.memcached(key).get(key)
if cas_old == cas_new:
print('cas did not change')
cas_error_collection.append(cas_old)
if ops == 'update':
if d_new != "{0}-{1}".format("mysql-new-value", x):
data_error_collection.append(
(d_new,
"{0}-{1}".format("mysql-new-value", x)))
if cas_old != cas_new and d_new == "{0}-{1}".format(
"mysql-new-value", x):
self.log.info(
"Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} "
.format(cas_old, key, cas_new))
mc_active = client.memcached(key)
mc_replica = client.memcached(key, replica_index=0)
active_cas = int(
mc_active.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
self.assertTrue(
active_cas == cas_new,
'cbstats cas mismatch. Expected {0}, actual {1}'.
format(cas_new, active_cas))
replica_cas = int(
mc_replica.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
poll_count = 0
while replica_cas != active_cas and poll_count < 5:
time.sleep(1)
replica_cas = int(
mc_replica.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
poll_count = poll_count + 1
if poll_count > 0:
self.log.info(
'Getting the desired CAS was delayed {0} seconds'
.format(poll_count))
self.assertTrue(
active_cas == replica_cas,
'replica cas mismatch. Expected {0}, actual {1}'.
format(cas_new, replica_cas))
elif ops == "delete":
o, cas, d = client.memcached(key).delete(key)
time.sleep(10)
self.log.info(
"Delete operation set item cas with key {0} to {1}".
format(key, cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas,
value)
raise Exception(
"The item should already be deleted. We can't mutate it anymore"
)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" %
(error.status, error.msg))
pass
else:
raise Exception(error)
elif ops == "expire":
o, cas, d = client.memcached(key).set(
key, self.expire_time, 0, value)
time.sleep(self.expire_time + 1)
self.log.info(
"Try to mutate an expired item with its previous cas {0}"
.format(cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas,
value)
raise Exception(
"The item should already be expired. We can't mutate it anymore"
)
except MemcachedError as error:
# It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" %
(error.status, error.msg))
pass
else:
raise Exception(error)
if len(cas_error_collection) > 0:
for cas_value in cas_error_collection:
self.log.error(
"Set operation fails to modify the CAS {0}".format(
cas_value))
raise Exception("Set operation fails to modify the CAS value")
if len(data_error_collection) > 0:
for data_value in data_error_collection:
self.log.error(
"Set operation fails. item-value is {0}, expected is {1}"
.format(data_value[0], data_value[1]))
raise Exception("Set operation fails to change item value")
def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
self.log.info(
'\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('failoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached('failoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be failed over'.format(
server_with_key))
RebalanceHelper.wait_for_persistence(self.master,
self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']))
# verify: no (hard) failover, everything else as before
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
time.sleep(30)
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
op_data = self.extract_vbucket_uuid_and_seqno(
client.set('hardfailoverkey', 0, 0, 'failovervalue'))
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached('hardfailoverkey').host
self.log.info(
'\n\nserver {0} has the key and it will be hard failed over'.
format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode='ns_1@' +
server_with_key,
graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(10)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached('hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
time.sleep(10)
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey', op_data['vbucket_uuid']))
self.check_results(op_data, after_failover_results)
self.log.info('Test complete')
def verify_cas(self, ops, generator):
"""Verify CAS value manipulation.
For update we use the latest CAS value return by set()
to do the mutation again to see if there is any exceptions.
We should be able to mutate that item with the latest CAS value.
For delete(), after it is called, we try to mutate that item with the
cas vavlue returned by delete(). We should see Memcached Error. Otherwise
the test should fail.
For expire, We want to verify using the latest CAS value of that item
can not mutate it because it is expired already."""
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
for bucket in self.buckets:
gen = generator
cas_error_collection = []
data_error_collection = []
while gen.has_next():
key, value = gen.next()
if ops in ["update","touch"]:
for x in range(self.mutate_times):
o_old, cas_old, d_old = client.get(key)
if ops == 'update':
client.memcached(key).cas(key, 0, 0, cas_old, "{0}-{1}".format("mysql-new-value", x))
else:
rc = client.memcached(key).touch(key,10)
o_new, cas_new, d_new = client.memcached(key).get(key)
if cas_old == cas_new:
print 'cas did not change'
cas_error_collection.append(cas_old)
if ops == 'update':
if d_new != "{0}-{1}".format("mysql-new-value", x):
data_error_collection.append((d_new,"{0}-{1}".format("mysql-new-value", x)))
if cas_old != cas_new and d_new == "{0}-{1}".format("mysql-new-value", x):
self.log.info("Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} "
.format(cas_old, key, cas_new))
mc_active = client.memcached( key )
mc_replica = client.memcached( key, replica_index=0 )
active_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(active_cas == cas_new,
'cbstats cas mismatch. Expected {0}, actual {1}'.format( cas_new, active_cas))
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
poll_count = 0
while replica_cas != active_cas and poll_count < 5:
time.sleep(1)
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
poll_count = poll_count + 1
if poll_count > 0:
self.log.info('Getting the desired CAS was delayed {0} seconds'.format( poll_count) )
self.assertTrue(active_cas == replica_cas,
'replica cas mismatch. Expected {0}, actual {1}'.format( cas_new, replica_cas))
elif ops == "delete":
o, cas, d = client.memcached(key).delete(key)
time.sleep(10)
self.log.info("Delete operation set item cas with key {0} to {1}".format(key, cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas, value)
raise Exception("The item should already be deleted. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError because the key is deleted.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
elif ops == "expire":
o, cas, d = client.memcached(key).set(key, self.expire_time, 0, value)
time.sleep(self.expire_time+1)
self.log.info("Try to mutate an expired item with its previous cas {0}".format(cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas, value)
raise Exception("The item should already be expired. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
if len(cas_error_collection) > 0:
for cas_value in cas_error_collection:
self.log.error("Set operation fails to modify the CAS {0}".format(cas_value))
raise Exception("Set operation fails to modify the CAS value")
if len(data_error_collection) > 0:
for data_value in data_error_collection:
self.log.error("Set operation fails. item-value is {0}, expected is {1}".format(data_value[0], data_value[1]))
raise Exception("Set operation fails to change item value")
def test_failover_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
def test_failover_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [], [self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key, request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS, replica_CAS))
def test_restart_revid_conflict_resolution(self):
self.log.info(' Starting test_restart_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
k=0
key = "{0}{1}".format(self.prefix, k)
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
# set a key
value = 'value0'
client.memcached(key).set(key, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
try:
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789', vbucket_id)
except mc_bin_client.MemcachedError as e:
# this is expected
pass
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
#self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
#self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# Restart Nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
cas_restart = mc_active.getMeta(key)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post))
class OpsChangeCasTests(BucketConfig):
def setUp(self):
super(OpsChangeCasTests, self).setUp()
self.prefix = "test_"
self.expire_time = self.input.param("expire_time", 35)
self.item_flag = self.input.param("item_flag", 0)
self.value_size = self.input.param("value_size", 256)
self.items = self.input.param("items", 20)
self.rest = RestConnection(self.master)
self.client = VBucketAwareMemcached(self.rest, self.bucket)
def tearDown(self):
super(OpsChangeCasTests, self).tearDown()
def test_meta_rebalance_out(self):
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket(vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [], [self.master])
rebalance.result()
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
get_meta_resp = mc_replica.getMeta(KEY_NAME, request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS, replica_CAS))
# not supported in 4.6 self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_failover(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [], [self.master])
rebalance.result()
time.sleep(60)
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
#print 'replica CAS {0}'.format(replica_CAS)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
get_meta_resp = mc_active.getMeta(KEY_NAME, request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS, replica_CAS))
self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_soft_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# restart nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME, request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_hard_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# reboot nodes
self._reboot_server()
time.sleep(60)
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME, request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Test Incremental sets on cas and max cas values for keys
'''
def test_cas_set(self):
self.log.info(' Starting test-sets')
self._load_ops(ops='set', mutations=20)
time.sleep(60)
self._check_cas(check_conflict_resolution=False)
''' Test Incremental updates on cas and max cas values for keys
'''
def test_cas_updates(self):
self.log.info(' Starting test-updates')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
self._load_ops(ops='replace', mutations=20)
#self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
''' Test Incremental deletes on cas and max cas values for keys
'''
def test_cas_deletes(self):
self.log.info(' Starting test-deletes')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
self._load_ops(ops='replace', mutations=20)
self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
''' Test expiry on cas and max cas values for keys
'''
def test_cas_expiry(self):
self.log.info(' Starting test-expiry')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
#self._load_ops(ops='replace',mutations=20)
self._load_ops(ops='expiry')
self._check_cas(check_conflict_resolution=False)
self._check_expiry()
''' Test touch on cas and max cas values for keys
'''
def test_cas_touch(self):
self.log.info(' Starting test-touch')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
#self._load_ops(ops='replace',mutations=20)
self._load_ops(ops='touch')
self._check_cas(check_conflict_resolution=False)
''' Test getMeta on cas and max cas values for keys
'''
def test_cas_getMeta(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
#self._load_ops(ops='add')
self._load_ops(ops='replace', mutations=20)
self._check_cas(check_conflict_resolution=False)
self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
def test_cas_setMeta_lower(self):
self.log.info(' Starting test-getMeta')
# set some kv
self._load_ops(ops='set', mutations=1)
#self._check_cas(check_conflict_resolution=False)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
self.log.info('For key {0} the vbucket is {1}'.format( key, vbucket_id ))
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
#mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
TEST_CAS = k
rc = mc_active.getMeta(key)
cas = rc[4] + 1
self.log.info('Key {0} retrieved CAS is {1} and will set CAS to {2}'.format(key, rc[4], cas))
rev_seqno = rc[3]
# do a set meta based on the existing CAS
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 123, cas)
# check what get meta say
rc = mc_active.getMeta(key)
cas_post_meta = rc[4]
self.log.info('Getmeta CAS is {0}'.format(cas_post_meta))
self.assertTrue( cas_post_meta == cas, 'Meta expected {0} actual {1}'.format( cas, cas_post_meta))
# and what stats says
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Max CAS for key {0} vbucket is {1}'.format( key, max_cas))
self.assertTrue(cas_post_meta >= max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
# do another mutation and compare
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
# and then mix in a set with meta
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 225, max_cas+1)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
# and one more mutation for good measure
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
def test_cas_setMeta_higher(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
#mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
#print '-'*100
TEST_SEQNO = 123
TEST_CAS = 9966180844186042368
cas = mc_active.getMeta(key)[4]
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, TEST_CAS, '123456789',vbucket_id,
# add_extended_meta_data=True, conflict_resolution_mode=1)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, TEST_CAS)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not equal it is {0}'.format(cas_post_meta))
self.assertTrue(max_cas > cas, '[ERROR]Max cas is not higher than original cas {0}'.format(cas))
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, max_cas+1)
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 125, TEST_CAS+1, '223456789',vbucket_id,
# add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_3 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr3 {0}'.format(get_meta_3)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not lower it is higher than {0}'.format(cas_post_meta))
#self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
''' Test deleteMeta on cas and max cas values for keys
'''
def test_cas_deleteMeta(self):
self.log.info(' Starting test-deleteMeta')
# load 20 kvs and check the CAS
self._load_ops(ops='set', mutations=20)
time.sleep(60)
self._check_cas(check_conflict_resolution=False)
k=0
test_cas = 456
while k<1:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
test_cas = test_cas + 1
# get the meta data
cas = mc_active.getMeta(key)[4] + 1
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, cas)
cas_post_meta = mc_active.getMeta(key)[4]
# verify the observed CAS is as set
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
# what is test cas for? Commenting out for now
"""
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, test_cas)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta < max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
"""
# test the delete
mc_active.set(key, 0, 0, json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
#
self.log.info('Doing delete with meta, using a lower CAS value')
get_meta_pre = mc_active.getMeta(key)[4]
del_with_meta_resp = mc_active.del_with_meta(key, 0, 0, TEST_SEQNO, test_cas, test_cas+1)
get_meta_post = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas > test_cas+1, '[ERROR]Max cas {0} is not greater than delete cas {1}'.format(max_cas, test_cas))
''' Testing skipping conflict resolution, whereby the last write wins, and it does neither cas CR nor rev id CR
'''
def test_cas_skip_conflict_resolution(self):
self.log.info(' Starting test_cas_skip_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
low_seq=12
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to allow insertion of lower Seq Number')
lower_cas = int(cas)-1
#import pdb;pdb.set_trace()
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, low_seq, lower_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, low_seq, lower_cas, 3)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
#print 'cas meta data after set_meta_force {0}'.format(cas_post_meta)
#print 'all meta data after set_meta_force {0}'.format(all_post_meta)
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq > post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
'''
def test_revid_conflict_resolution(self):
self.log.info(' Starting test_cas_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, cas, '123456789', vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing conflict resolution, where timeSync is enabled and cas is lower but higher revid, expect Higher Cas to Win
'''
def test_cas_conflict_resolution(self):
self.log.info(' Starting test_cas_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
lower_cas = int(cas)-100
self.log.info('Forcing lower rev-id to win with higher CAS value, instead of higher rev-id with Lower Cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, lower_cas, '123456789',vbucket_id)
try:
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, lower_cas)
except mc_bin_client.MemcachedError as e:
# this is expected
pass
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect CAS conflict_resolution to occur, and the first mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
#self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a restart server'''
def test_restart_revid_conflict_resolution(self):
self.log.info(' Starting test_restart_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
k=0
key = "{0}{1}".format(self.prefix, k)
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
# set a key
value = 'value0'
client.memcached(key).set(key, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
try:
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789', vbucket_id)
except mc_bin_client.MemcachedError as e:
# this is expected
pass
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
#self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
#self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# Restart Nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
cas_restart = mc_active.getMeta(key)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a rebalance server'''
def test_rebalance_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [], [self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key, request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
print('active cas {0}'.format(active_CAS))
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS, replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a failover server'''
def test_failover_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0, json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
#set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key, request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [], [self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key, request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS, replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Test getMeta on cas and max cas values for empty vbucket
'''
def test_cas_getMeta_empty_vBucket(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
k=0
all_keys = []
while k<10:
k+=1
key = "{0}{1}".format(self.prefix, k)
all_keys.append(key)
vbucket_ids = self.client._get_vBucket_ids(all_keys)
print('bucket_ids')
for v in vbucket_ids:
print(v)
print('done')
i=1111
if i not in vbucket_ids and i <= 1023:
vb_non_existing=i
elif i>1023:
i +=1
else:
self.log.info('ERROR generating empty vbucket id')
vb_non_existing=vbucket_ids.pop()
print('nominated vb_nonexisting is {0}'.format(vb_non_existing))
mc_active = self.client.memcached(all_keys[0]) #Taking a temp connection to the mc.
#max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(vb_non_existing) + ':max_cas'] )
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(all_keys[0])) + ':max_cas'] )
self.assertTrue( max_cas != 0, msg='[ERROR] Max cas is non-zero')
''' Test addMeta on cas and max cas values for keys
'''
def test_meta_backup(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
'''Do the backup on the bucket '''
self.shell = RemoteMachineShellConnection(self.master)
self.buckets = RestConnection(self.master).get_buckets()
self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username,
self.input.membase_settings.rest_password)
self.backup_location = "/tmp/backup"
self.command_options = self.input.param("command_options", '')
try:
shell = RemoteMachineShellConnection(self.master)
self.shell.execute_cluster_backup(self.couchbase_login_info, self.backup_location, self.command_options)
time.sleep(5)
shell.restore_backupFile(self.couchbase_login_info, self.backup_location, [bucket.name for bucket in self.buckets])
print('Done with restore')
finally:
self._check_cas(check_conflict_resolution=False)
''' Common function to verify the expected values on cas
'''
def _check_cas(self, check_conflict_resolution=False, master=None, bucket=None, time_sync=None):
self.log.info(' Verifying cas and max cas for the keys')
#select_count = 20 #Verifying top 20 keys
if master:
self.rest = RestConnection(master)
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
while k < self.items:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
#print 'max_cas is {0}'.format(max_cas)
self.assertTrue(cas == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas))
if check_conflict_resolution:
get_meta_resp = mc_active.getMeta(key, request_extended_meta_data=False)
if time_sync == 'enabledWithoutDrift':
self.assertTrue( get_meta_resp[5] == 1, msg='[ERROR] Metadata indicate conflict resolution is not set')
elif time_sync == 'disabled':
self.assertTrue( get_meta_resp[5] == 0, msg='[ERROR] Metadata indicate conflict resolution is set')
''' Common function to add set delete etc operations on the bucket
'''
def _load_ops(self, ops=None, mutations=1, master=None, bucket=None):
if master:
self.rest = RestConnection(master)
if bucket:
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
payload = MemcachedClientHelper.create_value('*', self.value_size)
while k < self.items:
key = "{0}{1}".format(self.prefix, k)
k += 1
for i in range(mutations):
if ops=='set':
#print 'set'
self.client.memcached(key).set(key, 0, 0, payload)
elif ops=='add':
#print 'add'
self.client.memcached(key).add(key, 0, 0, payload)
elif ops=='replace':
self.client.memcached(key).replace(key, 0, 0, payload)
#print 'Replace'
elif ops=='delete':
#print 'delete'
self.client.memcached(key).delete(key)
elif ops=='expiry':
#print 'expiry'
self.client.memcached(key).set(key, self.expire_time, 0, payload)
elif ops=='touch':
#print 'touch'
self.client.memcached(key).touch(key, 10)
self.log.info("Done with specified {0} ops".format(ops))
'''Check if items are expired as expected'''
def _check_expiry(self):
time.sleep(self.expire_time+30)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
self.log.info("Try to mutate an expired item with its previous cas {0}".format(cas))
try:
all = mc_active.getMeta(key)
a=self.client.memcached(key).get(key)
self.client.memcached(key).cas(key, 0, self.item_flag, cas, 'new')
all = mc_active.getMeta(key)
raise Exception("The item should already be expired. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
def test_failover(self):
self.log.info('\n\nStarting test_failover')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
self.log.info('\n\nVerify responses are correct after graceful failover')
op_data = self.extract_vbucket_uuid_and_seqno( client.set('failoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'no_failover'
# don't really need to do this so it is commented
#pre_failover_results = self.observe_seqno_response_to_dict( client.observe_seqno('failoverkey', vbucket_uuid) )
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting graceful failover scenario')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nserver {0} has the key and it will be failed over'.format(server_with_key))
RebalanceHelper.wait_for_persistence(self.master, self.default_bucket_name)
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=True)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('server should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'failoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('failoverkey', op_data['vbucket_uuid']) )
# verify: no (hard) failover, everything else as before
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
# now do a hard failover
# which server did the key go to and gracefully fail that server
self.log.info('\n\nstarting hard failover scenario')
client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO );
op_data = self.extract_vbucket_uuid_and_seqno( client.set('hardfailoverkey', 0, 0, 'failovervalue') )
op_data['format_type'] = 'hard_failover'
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nserver {0} has the key and it will be hard failed over'.format(server_with_key))
# now failover
RestConnection(self.master).fail_over(otpNode = 'ns_1@' + server_with_key, graceful=False)
if server_with_key in self.servers:
self.servers.remove(server_with_key)
self.log.info('\n\nserver should be failed over now')
time.sleep(5)
# reinstantiate the client so we get the new view of the world
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
server_with_key = client.memcached( 'hardfailoverkey').host
self.log.info('\n\nkey is now on server {0}'.format(server_with_key))
after_failover_results = self.observe_seqno_response_to_dict(
client.observe_seqno('hardfailoverkey',op_data['vbucket_uuid']) )
self.check_results( op_data, after_failover_results)
self.log.info('Test complete')
def test_items_append(self):
self.desired_item_size = self.input.param("desired_item_size", 2048)
self.append_size = self.input.param("append_size", 1024)
self.fixed_append_size = self.input.param("fixed_append_size", True)
self.append_ratio = self.input.param("append_ratio", 0.5)
self._load_all_buckets(self.master, self.gen_create, "create", 0,
batch_size=10000, pause_secs=5, timeout_secs=100)
for bucket in self.buckets:
verify_dict = {}
vkeys, dkeys = bucket.kvs[1].key_set()
key_count = len(vkeys)
app_ratio = self.append_ratio * key_count
selected_keys = []
i = 0
for key in vkeys:
i += 1
if i >= app_ratio:
break
selected_keys.append(key)
awareness = VBucketAwareMemcached(RestConnection(self.master), bucket.name)
if self.kv_verify:
for key in selected_keys:
value = awareness.memcached(key).get(key)[2]
verify_dict[key] = value
self.log.info("Bucket: {0}".format(bucket.name))
self.log.info("Appending to have items whose initial size was "
+ "{0} to equal or cross a size of {1}".format(self.value_size, self.desired_item_size))
self.log.info("Item-appending of {0} items starting ..".format(len(selected_keys)+1))
index = 3
while self.value_size < self.desired_item_size:
str_len = self.append_size
if not self.fixed_append_size:
str_len = int(math.pow(2, index))
for key in selected_keys:
random_string = self.random_str_generator(str_len)
awareness.memcached(key).append(key, random_string)
if self.kv_verify:
verify_dict[key] = verify_dict[key] + random_string
self.value_size += str_len
index += 1
self.log.info("The appending of {0} items ended".format(len(selected_keys)+1))
msg = "Bucket:{0}".format(bucket.name)
self.log.info("VERIFICATION <" + msg + ">: Phase 0 - Check the gap between "
+ "mem_used by the bucket and total_allocated_bytes")
stats = StatsCommon()
mem_used_stats = stats.get_stats(self.servers, bucket, 'memory', 'mem_used')
total_allocated_bytes_stats = stats.get_stats(self.servers, bucket, 'memory', 'total_allocated_bytes')
total_fragmentation_bytes_stats = stats.get_stats(self.servers, bucket, 'memory', 'total_fragmentation_bytes')
for server in self.servers:
self.log.info("In {0} bucket {1}, total_fragmentation_bytes + the total_allocated_bytes = {2}"
.format(server.ip, bucket.name, (int(total_fragmentation_bytes_stats[server]) + int(total_allocated_bytes_stats[server]))))
self.log.info("In {0} bucket {1}, mem_used = {2}".format(server.ip, bucket.name, mem_used_stats[server]))
self.log.info("In {0} bucket {1}, the difference between acutal memory used by memcached and mem_used is {2} times"
.format(server.ip, bucket.name, float(int(total_fragmentation_bytes_stats[server]) + int(total_allocated_bytes_stats[server]))/float(mem_used_stats[server])))
self.log.info("VERIFICATION <" + msg + ">: Phase1 - Check if any of the "
+ "selected keys have value less than the desired value size")
for key in selected_keys:
value = awareness.memcached(key).get(key)[2]
if len(value) < self.desired_item_size:
self.fail("Failed to append enough to make value size surpass the "
+ "size, for key {0}".format(key))
if self.kv_verify:
self.log.info("VERIFICATION <" + msg + ">: Phase2 - Check if the content "
+ "after the appends match whats expected")
for k in verify_dict:
if awareness.memcached(key).get(k)[2] != verify_dict[k]:
self.fail("Content at key {0}: not what's expected.".format(k))
self.log.info("VERIFICATION <" + msg + ">: Successful")
def _run_observe(self):
tasks = []
query_set = "true"
persisted = 0
mutated = False
count = 0
for bucket in self.buckets:
self.cluster.create_view(self.master, self.default_design_doc,
self.default_view, bucket,
self.wait_timeout * 2)
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
self.max_time = timedelta(microseconds=0)
if self.mutate_by == "multi_set":
key_val = self._create_multi_set_batch()
client.setMulti(0, 0, key_val)
keys = ["observe%s" % (i) for i in range(self.num_items)]
for key in keys:
mutated = False
while not mutated and count < 60:
try:
if self.mutate_by == "set":
# client.memcached(key).set(key, 0, 0, "set")
client.set(key, 0, 0, "setvalue")
elif self.mutate_by == "append":
client.memcached(key).append(key, "append")
elif self.mutate_by == "prepend":
client.memcached(key).prepend(key, "prepend")
elif self.mutate_by == "incr":
client.memcached(key).incr(key, 1)
elif self.mutate_by == "decr":
client.memcached(key).decr(key)
mutated = True
t_start = datetime.now()
except MemcachedError as error:
if error.status == 134:
loaded = False
self.log.error(
"Memcached error 134, wait for 5 seconds and then try again"
)
count += 1
time.sleep(5)
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = client.observe(
key)
t_end = datetime.now()
#self.log.info("##########key:-%s################" % (key))
#self.log.info("Persisted:- %s" % (persisted))
#self.log.info("Persist_Time:- %s" % (rep_time))
#self.log.info("Time2:- %s" % (t_end - t_start))
if self.max_time <= (t_end - t_start):
self.max_time = (t_end - t_start)
self.log.info("Max Time taken for observe is :- %s" %
self.max_time)
self.log.info("Cas Value:- %s" % (cas))
query = {
"stale": "false",
"full_set": "true",
"connection_timeout": 600000
}
self.cluster.query_view(self.master,
"dev_Doc1",
self.default_view.name,
query,
self.num_items,
bucket,
timeout=self.wait_timeout)
self.log.info(
"Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
% (self.default_view, bucket))
# check whether observe has to run with delete and delete parallel with observe or not
if len(self.observe_with) > 0:
if self.observe_with == "delete":
self.log.info("Deleting 0- %s number of items" %
(self.num_items // 2))
self._load_doc_data_all_buckets('delete', 0,
self.num_items // 2)
query_set = "true"
elif self.observe_with == "delete_parallel":
self.log.info("Deleting Parallel 0- %s number of items" %
(self.num_items // 2))
tasks = self._async_load_doc_data_all_buckets(
'delete', 0, self.num_items // 2)
query_set = "false"
for key in keys:
opaque, rep_time, persist_time, persisted, cas = client.memcached(
key).observe(key)
self.log.info("##########key:-%s################" % (key))
self.log.info("Persisted:- %s" % (persisted))
if self.observe_with == "delete_parallel":
for task in tasks:
task.result()
query = {
"stale": "false",
"full_set": query_set,
"connection_timeout": 600000
}
self.cluster.query_view(self.master,
"dev_Doc1",
self.default_view.name,
query,
self.num_items // 2,
bucket,
timeout=self.wait_timeout)
self.log.info(
"Observe Validation:- view: %s in design doc dev_Doc1 and in bucket %s"
% (self.default_view, self.default_bucket_name))
"""test_observe_basic_data_load_delete will test observer basic scenario
class Upgrade_EpTests(UpgradeTests):
def setUp(self):
super(Upgrade_EpTests, self).setUp()
print self.master
self.rest = RestConnection(self.master)
self.bucket = 'default' # temp fix
self.client = VBucketAwareMemcached(self.rest, self.bucket)
self.time_synchronization ='disabled'
print 'checking for self.servers '.format(self.servers[1])
self.prefix = "test_"
self.expire_time = 5
self.item_flag = 0
self.value_size = 256
def tearDown(self):
#super(Upgrade_EpTests, self).tearDown()
self.testcase = '2'
if not "skip_cleanup" in TestInputSingleton.input.test_params:
BucketOperationHelper.delete_all_buckets_or_assert(
self.servers, self.testcase)
ClusterOperationHelper.cleanup_cluster(self.servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers, self.testcase)
def test_upgrade(self):
self.log.info('Starting upgrade tests...')
#o = OpsChangeCasTests()
self.log.info('Inserting few items pre upgrade')
self._load_ops(ops='set', mutations=20, master=self.master, bucket=self.bucket)
self.log.info('Upgrading ..')
try:
UpgradeTests.test_upgrade(self)
finally:
self.log.info(' Done with Upgrade ')
self.log.info('Testing the meta details on items post upgrade')
#self._check_config()
self._check_cas(check_conflict_resolution=True, master=self.servers[1], bucket=self.bucket)
def _check_config(self):
result = self.rest.get_bucket_json(self.bucket)["timeSynchronization"]
print result
self.assertEqual(result,self.time_synchronization, msg='ERROR, Mismatch on expected time synchronization values')
self.log.info("Verified results")
def _load_ops(self, ops=None, mutations=1, master=None, bucket=None):
if master:
self.rest = RestConnection(master)
if bucket:
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
payload = MemcachedClientHelper.create_value('*', self.value_size)
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
for i in range(mutations):
if ops=='set':
#print 'set'
self.client.memcached(key).set(key, 0, 0,payload)
elif ops=='add':
#print 'add'
self.client.memcached(key).add(key, 0, 0,payload)
elif ops=='replace':
self.client.memcached(key).replace(key, 0, 0,payload)
#print 'Replace'
elif ops=='delete':
#print 'delete'
self.client.memcached(key).delete(key)
elif ops=='expiry':
#print 'expiry'
self.client.memcached(key).set(key, self.expire_time ,0, payload)
elif ops=='touch':
#print 'touch'
self.client.memcached(key).touch(key, 10)
self.log.info("Done with specified {0} ops".format(ops))
''' Common function to verify the expected values on cas
'''
def _check_cas(self, check_conflict_resolution=False, master=None, bucket=None, time_sync=None):
self.log.info(' Verifying cas and max cas for the keys')
if master:
self.rest = RestConnection(master)
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
#print 'max_cas is {0}'.format(max_cas)
self.assertTrue(cas == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas))
if check_conflict_resolution:
get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=True)
if time_sync == 'enabledWithoutDrift':
self.assertTrue( get_meta_resp[5] == 1, msg='[ERROR] Metadata indicate conflict resolution is not set')
elif time_sync == 'disabled':
self.assertTrue( get_meta_resp[5] == 0, msg='[ERROR] Metadata indicate conflict resolution is set')
def test_restart_revid_conflict_resolution(self):
self.log.info(' Starting test_restart_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
k=0
key = "{0}{1}".format(self.prefix, k)
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
# set a key
value = 'value0'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# Restart Nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
cas_restart = mc_active.getMeta(key)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post))
def test_items_append(self):
self.desired_item_size = self.input.param("desired_item_size", 2048)
self.append_size = self.input.param("append_size", 1024)
self.fixed_append_size = self.input.param("fixed_append_size", True)
self.append_ratio = self.input.param("append_ratio", 0.5)
self._load_all_buckets(self.master,
self.gen_create,
"create",
0,
batch_size=1000,
pause_secs=5,
timeout_secs=100)
for bucket in self.buckets:
self.value_size = self.input.param("value_size", 512)
verify_dict = {}
vkeys, dkeys = bucket.kvs[1].key_set()
key_count = len(vkeys)
app_ratio = self.append_ratio * key_count
selected_keys = []
i = 0
for key in vkeys:
i += 1
if i >= app_ratio:
break
selected_keys.append(key)
awareness = VBucketAwareMemcached(RestConnection(self.master),
bucket.name)
if self.kv_verify:
for key in selected_keys:
value = awareness.memcached(key).get(key)[2]
verify_dict[key] = value
self.log.info("Bucket: {0}".format(bucket.name))
self.log.info("Appending to have items whose initial size was " +
"{0} to equal or cross a size of {1}".format(
self.value_size, self.desired_item_size))
self.log.info("Item-appending of {0} items starting ..".format(
len(selected_keys) + 1))
index = 3
while self.value_size < self.desired_item_size:
str_len = self.append_size
if not self.fixed_append_size:
str_len = int(math.pow(2, index))
for key in selected_keys:
random_string = self.random_str_generator(str_len)
awareness.memcached(key).append(key, random_string)
if self.kv_verify:
verify_dict[key] = verify_dict[key] + random_string
self.log.info(
"for {0} items size was increased to {1} Bytes".format(
len(selected_keys) + 1, self.value_size))
self.value_size += str_len
index += 1
self.log.info("The appending of {0} items ended".format(
len(selected_keys) + 1))
for bucket in self.buckets:
msg = "Bucket:{0}".format(bucket.name)
self.log.info("VERIFICATION <" + msg +
">: Phase 0 - Check the gap between " +
"mem_used by the bucket and total_allocated_bytes")
stats = StatsCommon()
mem_used_stats = stats.get_stats(self.servers, bucket, 'memory',
'mem_used')
total_allocated_bytes_stats = stats.get_stats(
self.servers, bucket, 'memory', 'total_allocated_bytes')
total_fragmentation_bytes_stats = stats.get_stats(
self.servers, bucket, 'memory', 'total_fragmentation_bytes')
for server in self.servers:
self.log.info(
"In {0} bucket {1}, total_fragmentation_bytes + the total_allocated_bytes = {2}"
.format(server.ip, bucket.name,
(int(total_fragmentation_bytes_stats[server]) +
int(total_allocated_bytes_stats[server]))))
self.log.info("In {0} bucket {1}, mem_used = {2}".format(
server.ip, bucket.name, mem_used_stats[server]))
self.log.info(
"In {0} bucket {1}, the difference between actual memory used by memcached and mem_used is {2} times"
.format(
server.ip, bucket.name,
float(
int(total_fragmentation_bytes_stats[server]) +
int(total_allocated_bytes_stats[server])) /
float(mem_used_stats[server])))
self.log.info(
"VERIFICATION <" + msg + ">: Phase1 - Check if any of the " +
"selected keys have value less than the desired value size")
for key in selected_keys:
value = awareness.memcached(key).get(key)[2]
if len(value) < self.desired_item_size:
self.fail(
"Failed to append enough to make value size surpass the "
+ "size {0}, key {1} has size {2}".format(
self.desired_item_size, key, len(value)))
if self.kv_verify:
self.log.info("VERIFICATION <" + msg +
">: Phase2 - Check if the content " +
"after the appends match what's expected")
for k in verify_dict:
if awareness.memcached(k).get(k)[2] != verify_dict[k]:
self.fail(
"Content at key {0}: not what's expected.".format(
k))
self.log.info("VERIFICATION <" + msg + ">: Successful")
shell = RemoteMachineShellConnection(self.master)
shell.execute_cbstats("", "raw", keyname="allocator", vbid="")
shell.disconnect()
class OpsChangeCasTests(BucketConfig):
def setUp(self):
super(OpsChangeCasTests, self).setUp()
self.prefix = "test_"
self.expire_time = self.input.param("expire_time", 35)
self.item_flag = self.input.param("item_flag", 0)
self.value_size = self.input.param("value_size", 256)
self.items = self.input.param("items", 20)
self.rest = RestConnection(self.master)
self.client = VBucketAwareMemcached(self.rest, self.bucket)
def tearDown(self):
super(OpsChangeCasTests, self).tearDown()
def test_meta_rebalance_out(self):
KEY_NAME = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket(vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master])
rebalance.result()
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
get_meta_resp = mc_replica.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
# not supported in 4.6 self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_failover(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_active = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_active)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
time.sleep(60)
replica_CAS = mc_replica.getMeta(KEY_NAME)[4]
#print 'replica CAS {0}'.format(replica_CAS)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
active_CAS = mc_active.getMeta(KEY_NAME)[4]
#print 'active cas {0}'.format(active_CAS)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_soft_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# restart nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
def test_meta_hard_restart(self):
KEY_NAME = 'key2'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
for i in range(10):
# set a key
value = 'value' + str(i)
client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(KEY_NAME)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(KEY_NAME)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
cas_pre = mc_active.getMeta(KEY_NAME)[4]
#print 'cas_a {0} '.format(cas_pre)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] )
self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre))
# reboot nodes
self._reboot_server()
time.sleep(60)
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(KEY_NAME)
cas_post = mc_active.getMeta(KEY_NAME)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
# extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Test Incremental sets on cas and max cas values for keys
'''
def test_cas_set(self):
self.log.info(' Starting test-sets')
self._load_ops(ops='set', mutations=20)
time.sleep(60)
self._check_cas(check_conflict_resolution=False)
''' Test Incremental updates on cas and max cas values for keys
'''
def test_cas_updates(self):
self.log.info(' Starting test-updates')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
self._load_ops(ops='replace',mutations=20)
#self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
''' Test Incremental deletes on cas and max cas values for keys
'''
def test_cas_deletes(self):
self.log.info(' Starting test-deletes')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
self._load_ops(ops='replace',mutations=20)
self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
''' Test expiry on cas and max cas values for keys
'''
def test_cas_expiry(self):
self.log.info(' Starting test-expiry')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
#self._load_ops(ops='replace',mutations=20)
self._load_ops(ops='expiry')
self._check_cas(check_conflict_resolution=False)
self._check_expiry()
''' Test touch on cas and max cas values for keys
'''
def test_cas_touch(self):
self.log.info(' Starting test-touch')
self._load_ops(ops='set', mutations=20)
#self._load_ops(ops='add')
#self._load_ops(ops='replace',mutations=20)
self._load_ops(ops='touch')
self._check_cas(check_conflict_resolution=False)
''' Test getMeta on cas and max cas values for keys
'''
def test_cas_getMeta(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
#self._load_ops(ops='add')
self._load_ops(ops='replace',mutations=20)
self._check_cas(check_conflict_resolution=False)
self._load_ops(ops='delete')
self._check_cas(check_conflict_resolution=False)
def test_cas_setMeta_lower(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
# Select arbit key
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
TEST_CAS = k
cas = mc_active.getMeta(key)[4]
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, cas, '123456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
#print max_cas
self.assertTrue(cas_post_meta >= max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 225, cas+1, '223456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_3 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr3 {0}'.format(get_meta_3)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
#self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
def test_cas_setMeta_higher(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
#print '-'*100
TEST_SEQNO = 123
TEST_CAS = 9966180844186042368
cas = mc_active.getMeta(key)[4]
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, TEST_CAS, '123456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not equal it is {0}'.format(cas_post_meta))
self.assertTrue(max_cas > cas, '[ERROR]Max cas is not higher than original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 125, TEST_CAS+1, '223456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
get_meta_3 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr3 {0}'.format(get_meta_3)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not lower it is higher than {0}'.format(cas_post_meta))
#self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
''' Test deleteMeta on cas and max cas values for keys
'''
def test_cas_deleteMeta(self):
self.log.info(' Starting test-deleteMeta')
# load 20 kvs and check the CAS
self._load_ops(ops='set', mutations=20)
time.sleep(60)
self._check_cas(check_conflict_resolution=False)
k=0
test_cas = 456
while k<1:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
TEST_SEQNO = 123
test_cas = test_cas + 1
# get the meta data
cas = mc_active.getMeta(key)[4]
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, test_cas, '123456789',vbucket_id)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta < max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
print 'cas post {0}'.format(cas_post_meta)
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 125, test_cas, '123456789',vbucket_id)
cas_post_meta = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(cas_post_meta < max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas))
mc_active.set(key, 0, 0,json.dumps({'value':'value3'}))
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas))
self.log.info('Doing delete with meta, using a lower CAS value')
get_meta_pre = mc_active.getMeta(key)[4]
del_with_meta_resp = mc_active.del_with_meta(key, 0, 0, TEST_SEQNO, test_cas, test_cas+1)
get_meta_post = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(max_cas > test_cas+1, '[ERROR]Max cas {0} is not greater than delete cas {1}'.format(max_cas, test_cas))
''' Testing skipping conflict resolution, whereby the last write wins, and it does neither cas CR nor rev id CR
'''
def test_cas_skip_conflict_resolution(self):
self.log.info(' Starting test_cas_skip_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
low_seq=12
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to allow insertion of lower Seq Number')
lower_cas = int(cas)-1
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, low_seq, lower_cas, '123456789',vbucket_id)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
#print 'cas meta data after set_meta_force {0}'.format(cas_post_meta)
#print 'all meta data after set_meta_force {0}'.format(all_post_meta)
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq > post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
'''
def test_revid_conflict_resolution(self):
self.log.info(' Starting test_cas_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, cas, '123456789',vbucket_id,
add_extended_meta_data=True, conflict_resolution_mode=1)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr2 {0}'.format(get_meta_2)
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing conflict resolution, where timeSync is enabled and cas is lower but higher revid, expect Higher Cas to Win
'''
def test_cas_conflict_resolution(self):
self.log.info(' Starting test_cas_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
self._check_cas(check_conflict_resolution=False)
k=0
#Check for first 20 keys
while k<20:
key = "{0}{1}".format(self.prefix, k)
k += 1
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
all = mc_active.getMeta(key)
self.log.info('all meta data before set_meta_force {0}'.format(all))
lower_cas = int(cas)-100
self.log.info('Forcing lower rev-id to win with higher CAS value, instead of higher rev-id with Lower Cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, lower_cas, '123456789',vbucket_id)
cas_post_meta = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect CAS conflict_resolution to occur, and the first mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a restart server'''
def test_restart_revid_conflict_resolution(self):
self.log.info(' Starting test_restart_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
k=0
key = "{0}{1}".format(self.prefix, k)
vbucket_id = self.client._get_vBucket_id(key)
mc_active = self.client.memcached(key)
mc_master = self.client.memcached_for_vbucket( vbucket_id )
mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id)
# set a key
value = 'value0'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# Restart Nodes
self._restart_server(self.servers[:])
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
cas_restart = mc_active.getMeta(key)[4]
#print 'post cas {0}'.format(cas_post)
get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'post CAS {0}'.format(cas_post)
#print 'post ext meta {0}'.format(get_meta_resp)
self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a rebalance server'''
def test_rebalance_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# remove that node
self.log.info('Remove the node with active data')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR
and retains it after a failover server'''
def test_failover_revid_conflict_resolution(self):
self.log.info(' Starting test_rebalance_revid_conflict_resolution ..')
self._load_ops(ops='set', mutations=20)
key = 'key1'
rest = RestConnection(self.master)
client = VBucketAwareMemcached(rest, self.bucket)
value = 'value'
client.memcached(key).set(key, 0, 0,json.dumps({'value':value}))
vbucket_id = client._get_vBucket_id(key)
#print 'vbucket_id is {0}'.format(vbucket_id)
mc_active = client.memcached(key)
mc_master = client.memcached_for_vbucket( vbucket_id )
mc_replica = client.memcached_for_replica_vbucket(vbucket_id)
new_seq=121
pre_cas = mc_active.getMeta(key)[4]
pre_seq = mc_active.getMeta(key)[3]
pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
all = mc_active.getMeta(key)
get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_1)
self.log.info('all meta data before set_meta_force {0}'.format(all))
self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas))
self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ')
set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id)
cas_post = mc_active.getMeta(key)[4]
all_post_meta = mc_active.getMeta(key)
post_seq = mc_active.getMeta(key)[3]
get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False)
#print 'cr {0}'.format(get_meta_2)
max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..')
self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta))
self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas))
self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq))
# failover that node
self.log.info('Failing over node with active data {0}'.format(self.master))
self.cluster.failover(self.servers, [self.master])
self.log.info('Remove the node with active data {0}'.format(self.master))
rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master])
rebalance.result()
time.sleep(120)
replica_CAS = mc_replica.getMeta(key)[4]
get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False)
#print 'replica CAS {0}'.format(replica_CAS)
#print 'replica ext meta {0}'.format(get_meta_resp)
# add the node back
self.log.info('Add the node back, the max_cas should be healed')
rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], [])
rebalance.result()
# verify the CAS is good
client = VBucketAwareMemcached(rest, self.bucket)
mc_active = client.memcached(key)
active_CAS = mc_active.getMeta(key)[4]
#print 'active cas {0}'.format(active_CAS)
self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
#self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set')
''' Test getMeta on cas and max cas values for empty vbucket
'''
def test_cas_getMeta_empty_vBucket(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
k=0
all_keys = []
while k<10:
k+=1
key = "{0}{1}".format(self.prefix, k)
all_keys.append(key)
vbucket_ids = self.client._get_vBucket_ids(all_keys)
print 'bucket_ids'
for v in vbucket_ids:
print v
print 'done'
i=1111
if i not in vbucket_ids and i <= 1023:
vb_non_existing=i
elif i>1023:
i +=1
else:
self.log.info('ERROR generating empty vbucket id')
vb_non_existing=vbucket_ids.pop()
print 'nominated vb_nonexisting is {0}'.format(vb_non_existing)
mc_active = self.client.memcached(all_keys[0]) #Taking a temp connection to the mc.
#max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(vb_non_existing) + ':max_cas'] )
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(all_keys[0])) + ':max_cas'] )
self.assertTrue( max_cas != 0, msg='[ERROR] Max cas is non-zero')
''' Test addMeta on cas and max cas values for keys
'''
def test_meta_backup(self):
self.log.info(' Starting test-getMeta')
self._load_ops(ops='set', mutations=20)
'''Do the backup on the bucket '''
self.shell = RemoteMachineShellConnection(self.master)
self.buckets = RestConnection(self.master).get_buckets()
self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username,
self.input.membase_settings.rest_password)
self.backup_location = "/tmp/backup"
self.command_options = self.input.param("command_options", '')
try:
shell = RemoteMachineShellConnection(self.master)
self.shell.execute_cluster_backup(self.couchbase_login_info, self.backup_location, self.command_options)
time.sleep(5)
shell.restore_backupFile(self.couchbase_login_info, self.backup_location, [bucket.name for bucket in self.buckets])
print 'Done with restore'
finally:
self._check_cas(check_conflict_resolution=False)
''' Common function to verify the expected values on cas
'''
def _check_cas(self, check_conflict_resolution=False, master=None, bucket=None, time_sync=None):
self.log.info(' Verifying cas and max cas for the keys')
#select_count = 20 #Verifying top 20 keys
if master:
self.rest = RestConnection(master)
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
while k < self.items:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] )
#print 'max_cas is {0}'.format(max_cas)
self.assertTrue(cas == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas))
if check_conflict_resolution:
get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=False)
if time_sync == 'enabledWithoutDrift':
self.assertTrue( get_meta_resp[5] == 1, msg='[ERROR] Metadata indicate conflict resolution is not set')
elif time_sync == 'disabled':
self.assertTrue( get_meta_resp[5] == 0, msg='[ERROR] Metadata indicate conflict resolution is set')
''' Common function to add set delete etc operations on the bucket
'''
def _load_ops(self, ops=None, mutations=1, master=None, bucket=None):
if master:
self.rest = RestConnection(master)
if bucket:
self.client = VBucketAwareMemcached(self.rest, bucket)
k=0
payload = MemcachedClientHelper.create_value('*', self.value_size)
while k < self.items:
key = "{0}{1}".format(self.prefix, k)
k += 1
for i in range(mutations):
if ops=='set':
#print 'set'
self.client.memcached(key).set(key, 0, 0,payload)
elif ops=='add':
#print 'add'
self.client.memcached(key).add(key, 0, 0,payload)
elif ops=='replace':
self.client.memcached(key).replace(key, 0, 0,payload)
#print 'Replace'
elif ops=='delete':
#print 'delete'
self.client.memcached(key).delete(key)
elif ops=='expiry':
#print 'expiry'
self.client.memcached(key).set(key, self.expire_time ,0, payload)
elif ops=='touch':
#print 'touch'
self.client.memcached(key).touch(key, 10)
self.log.info("Done with specified {0} ops".format(ops))
'''Check if items are expired as expected'''
def _check_expiry(self):
time.sleep(self.expire_time+30)
k=0
while k<10:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
self.log.info("Try to mutate an expired item with its previous cas {0}".format(cas))
try:
all = mc_active.getMeta(key)
a=self.client.memcached(key).get(key)
self.client.memcached(key).cas(key, 0, self.item_flag, cas, 'new')
all = mc_active.getMeta(key)
raise Exception("The item should already be expired. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
class SyncReplicationTest(unittest.TestCase):
awareness = None
def common_setup(self, replica):
self._input = TestInputSingleton.input
self._servers = self._input.servers
first = self._servers[0]
self.log = logger.Logger().get_logger()
self.log.info(self._input)
rest = RestConnection(first)
for server in self._servers:
RestHelper(RestConnection(server)).is_ns_server_running()
ClusterOperationHelper.cleanup_cluster(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(self._servers, self)
ClusterOperationHelper.add_all_nodes_or_assert(
self._servers[0], self._servers, self._input.membase_settings,
self)
nodes = rest.node_statuses()
otpNodeIds = []
for node in nodes:
otpNodeIds.append(node.id)
info = rest.get_nodes_self()
bucket_ram = info.mcdMemoryReserved * 3 / 4
rest.create_bucket(bucket="default",
ramQuotaMB=int(bucket_ram),
replicaNumber=replica,
proxyPort=rest.get_nodes_self().moxi)
msg = "wait_for_memcached fails"
ready = BucketOperationHelper.wait_for_memcached(first, "default"),
self.assertTrue(ready, msg)
rebalanceStarted = rest.rebalance(otpNodeIds, [])
self.assertTrue(
rebalanceStarted,
"unable to start rebalance on master node {0}".format(first.ip))
self.log.info('started rebalance operation on master node {0}'.format(
first.ip))
rebalanceSucceeded = rest.monitorRebalance()
# without a bucket this seems to fail
self.assertTrue(
rebalanceSucceeded,
"rebalance operation for nodes: {0} was not successful".format(
otpNodeIds))
self.awareness = VBucketAwareMemcached(rest, "default")
def tearDown(self):
if self.awareness:
self.awareness.done()
ClusterOperationHelper.cleanup_cluster(self._servers)
BucketOperationHelper.delete_all_buckets_or_assert(
self._servers, self)
def test_one_replica(self):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 1024)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
mc.sync_replication([{"key": k, "vbucket": vBucket}], 1)
for k in keys:
mc = self.awareness.memcached(k)
mc.get(k)
def test_one_replica_one_node(self):
pass
def test_one_replica_multiple_nodes(self):
pass
def test_one_replica_bucket_replica_one(self):
pass
def test_two_replica(self):
self._unsupported_replicas(2)
def test_three_replica(self):
self._unsupported_replicas(1)
def _unsupported_replicas(self, replica):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 102400)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
mc.get(k)
try:
mc.sync_replication([{"key": k, "vbucket": vBucket}], replica)
msg = "server did not raise an error when running sync_replication with {0} replicas"
self.fail(msg.format(replica))
except MemcachedError as error:
self.log.info("error {0} {1} as expected".format(
error.status, error.msg))
for k in keys:
mc = self.awareness.memcached(k)
mc.get(k)
def test_invalid_key(self):
pass
def test_not_your_vbucket(self):
self.common_setup(1)
keys = ["{0}-{1}".format(str(uuid.uuid4()), i) for i in range(0, 100)]
value = MemcachedClientHelper.create_value("*", 1024)
for k in keys:
vBucket = crc32.crc32_hash(k)
mc = self.awareness.memcached(k)
mc.set(k, 0, 0, value)
not_your_vbucket_mc = self.awareness.not_my_vbucket_memcached(k)
try:
count = 0
expected_error = 0
while count < 100:
a, b, response = not_your_vbucket_mc.sync_replication(
[{
"key": k,
"vbucket": vBucket
}], 1)
count += 1
self.log.info("response : {0}".format(response))
if response and response[0]["event"] != "invalid key":
expected_error += 1
if expected_error is not 100:
self.fail(
msg=
"server did not raise an error when running sync_replication with invalid vbucket"
)
except MemcachedError as error:
self.log.error(error)
def test_some_invalid_keys(self):
pass
def stest_ome_not_your_vbucket(self):
pass
def test_some_large_values(self):
pass
def test_too_many_keys(self):
pass
def test_singlenode(self):
pass
from resourceparser import ServerInfo
sys.path.append("lib")
sys.path.append("pytests")
server = ServerInfo()
server.ip = "127.0.0.1"
server.rest_username = '******'
server.rest_password = '******'
server.port = 9000
rest = RestConnection(server)
nodes = rest.node_statuses()
vm = VBucketAwareMemcached(rest,{"name":"bucket-0","password":""})
key = str(uuid.uuid4())
vm.memcached(key).set(key, 0, 0, "hi")
vm.memcached(key).get(key)
RebalanceHelper.print_taps_from_all_nodes(rest,bucket="bucket-0",password="")
RebalanceHelper.verify_items_count(server,"bucket-0")
RebalanceHelper.verify_items_count(server,"bucket-1")
RebalanceHelper.verify_items_count(server,"bucket-2")
RebalanceHelper.wait_till_total_numbers_match(server,"bucket-0",120,"")
cm = MemcachedClientHelper.proxy_client(server, "bucket-0", "")
key = str(uuid.uuid4())
cm.set(key, 0, 0, "hi")
cm.get(key)
def test_new_response_fields(self):
self.log.info('\n\nStarting test_new_response_fields')
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
h = client.sendHellos(
memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO)
set_resp = self.extract_vbucket_uuid_and_seqno(
client.set('test1key', 0, 0, '123456789'))
# test the inplace operations
test = client.generic_request(
client.memcached('test1key').set, 'test1key', 0, 0,
'totally new value')
replace_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').replace, 'test1key', 0, 0,
'totally new value'))
self.verify_vbucket_and_seqno(set_resp, replace_resp, 'replace')
append_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').append, 'test1key',
'appended data'))
self.verify_vbucket_and_seqno(replace_resp, append_resp, 'append')
prepend_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').prepend, 'test1key',
'prepended data'))
self.verify_vbucket_and_seqno(append_resp, prepend_resp, 'prepend')
# and finally do the delete
delete_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('test1key').delete, 'test1key'))
self.verify_vbucket_and_seqno(set_resp, delete_resp, 'delete')
#meta commands under construction
# test the 'meta' commands
TEST_SEQNO = 123
TEST_CAS = 456
set_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').set_with_meta,
'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, '123456789')
set_meta_vbucket_uuid, set_meta_seqno = struct.unpack(
'>QQ', set_with_meta_resp[2])
set_with_meta_dict = {
'vbucket_uuid': set_meta_vbucket_uuid,
'seqno': set_meta_seqno
}
get_meta_resp = client.generic_request(
client.memcached('test1keyformeta').getMeta, 'test1keyformeta')
self.assertTrue(TEST_SEQNO == get_meta_resp[3], \
msg='get meta seqno does not match as set. Expected {0}, actual {1}'.format(TEST_SEQNO, get_meta_resp[3]) )
self.assertTrue(TEST_CAS == get_meta_resp[4], \
msg='get meta cas does not match as set. Expected {0}, actual {1}'.format(TEST_CAS, get_meta_resp[4]) )
# def del_with_meta(self, key, exp, flags, seqno, old_cas, new_cas, vbucket= -1):
del_with_meta_resp = client.generic_request(
client.memcached('test1keyformeta').del_with_meta,
'test1keyformeta', 0, 0, TEST_SEQNO, TEST_CAS, TEST_CAS + 1)
vbucket_uuid, seqno = struct.unpack('>QQ', del_with_meta_resp[2])
del_with_meta_dict = {'vbucket_uuid': vbucket_uuid, 'seqno': seqno}
self.verify_vbucket_and_seqno(set_with_meta_dict, del_with_meta_dict,
'set/del with meta')
# do some integer operations
set_resp = self.extract_vbucket_uuid_and_seqno(
client.set('key-for-integer-value', 0, 0, '123'))
incr_resp = client.generic_request(
client.memcached('key-for-integer-value').incr,
'key-for-integer-value')
incr_resp_dict = {'vbucket_uuid': incr_resp[2], 'seqno': incr_resp[3]}
self.verify_vbucket_and_seqno(set_resp, incr_resp_dict, 'incr')
decr_resp = client.generic_request(
client.memcached('key-for-integer-value').decr,
'key-for-integer-value')
decr_resp_dict = {'vbucket_uuid': decr_resp[2], 'seqno': decr_resp[3]}
self.verify_vbucket_and_seqno(incr_resp_dict, decr_resp_dict, 'decr')
add_resp = self.extract_vbucket_uuid_and_seqno(
client.generic_request(
client.memcached('totally new key').add, 'totally new key', 0,
0, 'totally new value'))
self.assertTrue(add_resp['vbucket_uuid'] > 0,
msg='Add request vbucket uuid is zero')
self.log.info('\n\nComplete test_new_response_fields\n\n')
class Upgrade_EpTests(UpgradeTests):
def setUp(self):
super(Upgrade_EpTests, self).setUp()
print(self.master)
self.rest = RestConnection(self.master)
self.bucket = 'default' # temp fix
self.client = VBucketAwareMemcached(self.rest, self.bucket)
self.time_synchronization = 'disabled'
print('checking for self.servers '.format(self.servers[1]))
self.prefix = "test_"
self.expire_time = 5
self.item_flag = 0
self.value_size = 256
def tearDown(self):
#super(Upgrade_EpTests, self).tearDown()
self.testcase = '2'
if not "skip_cleanup" in TestInputSingleton.input.test_params:
BucketOperationHelper.delete_all_buckets_or_assert(
self.servers, self.testcase)
ClusterOperationHelper.cleanup_cluster(self.servers)
ClusterOperationHelper.wait_for_ns_servers_or_assert(
self.servers, self.testcase)
def test_upgrade(self):
self.log.info('Starting upgrade tests...')
#o = OpsChangeCasTests()
self.log.info('Inserting few items pre upgrade')
self._load_ops(ops='set',
mutations=20,
master=self.master,
bucket=self.bucket)
self.log.info('Upgrading ..')
try:
UpgradeTests.test_upgrade(self)
finally:
self.log.info(' Done with Upgrade ')
self.log.info('Testing the meta details on items post upgrade')
#self._check_config()
self._check_cas(check_conflict_resolution=True,
master=self.servers[1],
bucket=self.bucket)
def _check_config(self):
result = self.rest.get_bucket_json(self.bucket)["timeSynchronization"]
print(result)
self.assertEqual(
result,
self.time_synchronization,
msg='ERROR, Mismatch on expected time synchronization values')
self.log.info("Verified results")
def _load_ops(self, ops=None, mutations=1, master=None, bucket=None):
if master:
self.rest = RestConnection(master)
if bucket:
self.client = VBucketAwareMemcached(self.rest, bucket)
k = 0
payload = MemcachedClientHelper.create_value('*', self.value_size)
while k < 10:
key = "{0}{1}".format(self.prefix, k)
k += 1
for i in range(mutations):
if ops == 'set':
#print 'set'
self.client.memcached(key).set(key, 0, 0, payload)
elif ops == 'add':
#print 'add'
self.client.memcached(key).add(key, 0, 0, payload)
elif ops == 'replace':
self.client.memcached(key).replace(key, 0, 0, payload)
#print 'Replace'
elif ops == 'delete':
#print 'delete'
self.client.memcached(key).delete(key)
elif ops == 'expiry':
#print 'expiry'
self.client.memcached(key).set(key, self.expire_time, 0,
payload)
elif ops == 'touch':
#print 'touch'
self.client.memcached(key).touch(key, 10)
self.log.info("Done with specified {0} ops".format(ops))
''' Common function to verify the expected values on cas
'''
def _check_cas(self,
check_conflict_resolution=False,
master=None,
bucket=None,
time_sync=None):
self.log.info(' Verifying cas and max cas for the keys')
if master:
self.rest = RestConnection(master)
self.client = VBucketAwareMemcached(self.rest, bucket)
k = 0
while k < 10:
key = "{0}{1}".format(self.prefix, k)
k += 1
mc_active = self.client.memcached(key)
cas = mc_active.getMeta(key)[4]
max_cas = int(
mc_active.stats('vbucket-details')
['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'])
#print 'max_cas is {0}'.format(max_cas)
self.assertTrue(cas == max_cas,
'[ERROR]Max cas is not 0 it is {0}'.format(cas))
if check_conflict_resolution:
get_meta_resp = mc_active.getMeta(
key, request_extended_meta_data=True)
if time_sync == 'enabledWithoutDrift':
self.assertTrue(
get_meta_resp[5] == 1,
msg=
'[ERROR] Metadata indicate conflict resolution is not set'
)
elif time_sync == 'disabled':
self.assertTrue(
get_meta_resp[5] == 0,
msg=
'[ERROR] Metadata indicate conflict resolution is set')
