def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
mc = VBucketAwareMemcached(cb_rest, bucket)
es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count)
# compare values of es documents to documents in couchbase
for row in es_valid[:verification_count]:
key = str(row["meta"]["id"])
try:
_, _, doc = mc.get(key)
val_src = str(json.loads(doc)["site_name"])
val_dest = str(row["doc"]["site_name"])
if val_src != val_dest:
self.xd_ref.fail(
"Document %s has unexpected value (%s) expected (%s)" % (key, val_src, val_dest)
)
except MemcachedError as e:
self.xd_ref.fail("Error during verification. Index contains invalid key: %s" % key)
self._log.info(
"Verified doc values in couchbase bucket (%s) match values in elastic search" % (bucket.name)
)
def _verify_es_values(self,
src_server,
dest_server,
kv_store=1,
verification_count=10000):
cb_rest = RestConnection(src_server)
es_rest = RestConnection(dest_server)
buckets = self.xd_ref._get_cluster_buckets(src_server)
for bucket in buckets:
mc = VBucketAwareMemcached(cb_rest, bucket)
es_valid = es_rest.all_docs(indices=[bucket.name],
size=verification_count)
# compare values of es documents to documents in couchbase
for row in es_valid[:verification_count]:
key = str(row['meta']['id'])
try:
_, _, doc = mc.get(key)
val_src = str(json.loads(doc)['site_name'])
val_dest = str(row['doc']['site_name'])
if val_src != val_dest:
self.xd_ref.fail("Document %s has unexpected value (%s) expected (%s)" % \
(key, val_src, val_dest))
except MemcachedError as e:
self.xd_ref.fail(
"Error during verification. Index contains invalid key: %s"
% key)
self._log.info("Verified doc values in couchbase bucket (%s) match values in elastic search" % \
(bucket.name))
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND,
'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(
err == ERR_NOT_FOUND,
'was able to replace cas on removed key {0}'.format(KEY_NAME))
def verify_missing_keys(self, prefix, count):
"""
verify that none of the keys exist with specified prefix.
"""
client = VBucketAwareMemcached(RestConnection(self.servers[0]),
self.buckets[0])
for i in range(count):
key = "{0}{1}".format(prefix, i)
try:
client.get(key)
except MemcachedError as error:
self.assertEqual(
error.status, constants.ERR_NOT_FOUND,
"expected error NOT_FOUND, got {0}".format(error.status))
else:
self.fail("Able to retrieve expired document")
def key_not_exists_test(self):
self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required')
bucket = self.buckets[0].name
client = VBucketAwareMemcached(RestConnection(self.master), bucket)
KEY_NAME = 'key'
for i in range(1500):
client.set(KEY_NAME, 0, 0, "x")
# delete and verify get fails
client.delete(KEY_NAME)
err = None
try:
rc = client.get(KEY_NAME)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME))
#cas errors do not sleep the test for 10 seconds, plus we need to check that the correct
#error is being thrown
err = None
try:
#For some reason replace instead of cas would not reproduce the bug
mc_active = client.memcached(KEY_NAME)
mc_active.replace(KEY_NAME, 0, 10, "value")
except MemcachedError as error:
err = error.status
self.assertTrue(err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def verify_missing_keys(self, prefix, count):
"""
verify that none of the keys exist with specified prefix.
"""
client = VBucketAwareMemcached(
RestConnection(self.servers[0]),
self.buckets[0])
for i in xrange(count):
key = "{0}{1}".format(prefix, i)
try:
client.get(key)
except MemcachedError as error:
self.assertEquals(
error.status,
constants.ERR_NOT_FOUND,
"expected error NOT_FOUND, got {0}".format(error.status))
else:
self.fail("Able to retrieve expired document")
def _verify_data_all_buckets(self, gen_check):
for bucket in self.buckets:
self.log.info("Check bucket %s" % bucket.name)
gen = copy.deepcopy(gen_check)
rest = RestConnection(self.server_recovery)
client = VBucketAwareMemcached(rest, bucket)
while gen.has_next():
key, value = next(gen)
try:
_, _, d = client.get(key)
self.assertEqual(d.decode("utf-8"), value,\
'Key: %s expected. Value expected %s. Value actual %s' % (key, value, d))
except Exception as ex:
raise Exception('Key %s not found %s' % (key, str(ex)))
def _verify_data_all_buckets(self, gen_check):
for bucket in self.buckets:
self.log.info("Check bucket %s" % bucket.name)
gen = copy.deepcopy(gen_check)
rest = RestConnection(self.server_recovery)
client = VBucketAwareMemcached(rest, bucket)
while gen.has_next():
key, value = gen.next()
try:
_, _, d = client.get(key)
self.assertEquals(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % (
key, value, d))
except Exception, ex:
raise Exception('Key %s not found %s' % (key, str(ex)))
def verify_cas(self, ops, generator):
"""Verify CAS value manipulation.
For update we use the latest CAS value return by set()
to do the mutation again to see if there is any exceptions.
We should be able to mutate that item with the latest CAS value.
For delete(), after it is called, we try to mutate that item with the
cas vavlue returned by delete(). We should see Memcached Error. Otherwise
the test should fail.
For expire, We want to verify using the latest CAS value of that item
can not mutate it because it is expired already."""
client = VBucketAwareMemcached(RestConnection(self.master), 'default')
for bucket in self.buckets:
gen = generator
cas_error_collection = []
data_error_collection = []
while gen.has_next():
key, value = gen.next()
if ops in ["update","touch"]:
for x in range(self.mutate_times):
o_old, cas_old, d_old = client.get(key)
if ops == 'update':
client.memcached(key).cas(key, 0, 0, cas_old, "{0}-{1}".format("mysql-new-value", x))
else:
rc = client.memcached(key).touch(key,10)
o_new, cas_new, d_new = client.memcached(key).get(key)
if cas_old == cas_new:
print 'cas did not change'
cas_error_collection.append(cas_old)
if ops == 'update':
if d_new != "{0}-{1}".format("mysql-new-value", x):
data_error_collection.append((d_new,"{0}-{1}".format("mysql-new-value", x)))
if cas_old != cas_new and d_new == "{0}-{1}".format("mysql-new-value", x):
self.log.info("Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} "
.format(cas_old, key, cas_new))
mc_active = client.memcached( key )
mc_replica = client.memcached( key, replica_index=0 )
active_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
self.assertTrue(active_cas == cas_new,
'cbstats cas mismatch. Expected {0}, actual {1}'.format( cas_new, active_cas))
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
poll_count = 0
while replica_cas != active_cas and poll_count < 5:
time.sleep(1)
replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] )
poll_count = poll_count + 1
if poll_count > 0:
self.log.info('Getting the desired CAS was delayed {0} seconds'.format( poll_count) )
self.assertTrue(active_cas == replica_cas,
'replica cas mismatch. Expected {0}, actual {1}'.format( cas_new, replica_cas))
elif ops == "delete":
o, cas, d = client.memcached(key).delete(key)
time.sleep(10)
self.log.info("Delete operation set item cas with key {0} to {1}".format(key, cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas, value)
raise Exception("The item should already be deleted. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError because the key is deleted.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
elif ops == "expire":
o, cas, d = client.memcached(key).set(key, self.expire_time, 0, value)
time.sleep(self.expire_time+1)
self.log.info("Try to mutate an expired item with its previous cas {0}".format(cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas, value)
raise Exception("The item should already be expired. We can't mutate it anymore")
except MemcachedError as error:
#It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg))
pass
else:
raise Exception(error)
if len(cas_error_collection) > 0:
for cas_value in cas_error_collection:
self.log.error("Set operation fails to modify the CAS {0}".format(cas_value))
raise Exception("Set operation fails to modify the CAS value")
if len(data_error_collection) > 0:
for data_value in data_error_collection:
self.log.error("Set operation fails. item-value is {0}, expected is {1}".format(data_value[0], data_value[1]))
raise Exception("Set operation fails to change item value")
class GenericLoadingTask(Thread, Task):
def __init__(self, server, bucket, kv_store):
Thread.__init__(self)
Task.__init__(self, "load_gen_task")
self.kv_store = kv_store
self.client = VBucketAwareMemcached(RestConnection(server), bucket)
def execute(self, task_manager):
self.start()
self.state = EXECUTING
def check(self, task_manager):
pass
def run(self):
while self.has_next() and not self.done():
self.next()
self.state = FINISHED
self.set_result(True)
def has_next(self):
raise NotImplementedError
def next(self):
raise NotImplementedError
def _unlocked_create(self, partition, key, value):
try:
value_json = json.loads(value)
value_json['mutated'] = 0
value = json.dumps(value_json)
except ValueError:
index = random.choice(range(len(value)))
value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:]
try:
self.client.set(key, self.exp, 0, value)
partition.set(key, value, self.exp)
except MemcachedError as error:
self.state = FINISHED
self.set_exception(error)
def _unlocked_read(self, partition, key):
try:
o, c, d = self.client.get(key)
except MemcachedError as error:
if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None:
pass
else:
self.state = FINISHED
self.set_exception(error)
def _unlocked_update(self, partition, key):
value = partition.get_valid(key)
if value is None:
return
try:
value_json = json.loads(value)
value_json['mutated'] += 1
value = json.dumps(value_json)
except ValueError:
index = random.choice(range(len(value)))
value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:]
try:
self.client.set(key, self.exp, 0, value)
partition.set(key, value, self.exp)
except MemcachedError as error:
self.state = FINISHED
self.set_exception(error)
def _unlocked_delete(self, partition, key):
try:
self.client.delete(key)
partition.delete(key)
except MemcachedError as error:
if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None:
pass
else:
self.state = FINISHED
self.set_exception(error)
def verify_cas(self, ops, generator):
"""Verify CAS value manipulation.
For update we use the latest CAS value return by set()
to do the mutation again to see if there is any exceptions.
We should be able to mutate that item with the latest CAS value.
For delete(), after it is called, we try to mutate that item with the
cas vavlue returned by delete(). We should see Memcached Error. Otherwise
the test should fail.
For expire, We want to verify using the latest CAS value of that item
can not mutate it because it is expired already."""
for bucket in self.buckets:
client = VBucketAwareMemcached(RestConnection(self.master),
bucket.name)
gen = generator
cas_error_collection = []
data_error_collection = []
while gen.has_next():
key, value = next(gen)
if ops in ["update", "touch"]:
for x in range(self.mutate_times):
o_old, cas_old, d_old = client.get(key)
if ops == 'update':
client.memcached(key).cas(
key, 0, 0, cas_old,
"{0}-{1}".format("mysql-new-value", x))
else:
client.memcached(key).touch(key, 10)
o_new, cas_new, d_new = client.memcached(key).get(key)
if cas_old == cas_new:
print('cas did not change')
cas_error_collection.append(cas_old)
if ops == 'update':
if d_new != "{0}-{1}".format("mysql-new-value", x):
data_error_collection.append(
(d_new,
"{0}-{1}".format("mysql-new-value", x)))
if cas_old != cas_new and d_new == "{0}-{1}".format(
"mysql-new-value", x):
self.log.info(
"Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} "
.format(cas_old, key, cas_new))
mc_active = client.memcached(key)
mc_replica = client.memcached(key, replica_index=0)
active_cas = int(
mc_active.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
self.assertTrue(
active_cas == cas_new,
'cbstats cas mismatch. Expected {0}, actual {1}'.
format(cas_new, active_cas))
replica_cas = int(
mc_replica.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
poll_count = 0
while replica_cas != active_cas and poll_count < 5:
time.sleep(1)
replica_cas = int(
mc_replica.stats('vbucket-details')
['vb_' + str(client._get_vBucket_id(key)) +
':max_cas'])
poll_count = poll_count + 1
if poll_count > 0:
self.log.info(
'Getting the desired CAS was delayed {0} seconds'
.format(poll_count))
self.assertTrue(
active_cas == replica_cas,
'replica cas mismatch. Expected {0}, actual {1}'.
format(cas_new, replica_cas))
elif ops == "delete":
o, cas, d = client.memcached(key).delete(key)
time.sleep(10)
self.log.info(
"Delete operation set item cas with key {0} to {1}".
format(key, cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas,
value)
raise Exception(
"The item should already be deleted. We can't mutate it anymore"
)
except MemcachedError as error:
# It is expected to raise MemcachedError because the key is deleted.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" %
(error.status, error.msg))
pass
else:
raise Exception(error)
elif ops == "expire":
o, cas, d = client.memcached(key).set(
key, self.expire_time, 0, value)
time.sleep(self.expire_time + 1)
self.log.info(
"Try to mutate an expired item with its previous cas {0}"
.format(cas))
try:
client.memcached(key).cas(key, 0, self.item_flag, cas,
value)
raise Exception(
"The item should already be expired. We can't mutate it anymore"
)
except MemcachedError as error:
# It is expected to raise MemcachedError becasue the key is expired.
if error.status == ERR_NOT_FOUND:
self.log.info("<MemcachedError #%d ``%s''>" %
(error.status, error.msg))
pass
else:
raise Exception(error)
if len(cas_error_collection) > 0:
for cas_value in cas_error_collection:
self.log.error(
"Set operation fails to modify the CAS {0}".format(
cas_value))
raise Exception("Set operation fails to modify the CAS value")
if len(data_error_collection) > 0:
for data_value in data_error_collection:
self.log.error(
"Set operation fails. item-value is {0}, expected is {1}"
.format(data_value[0], data_value[1]))
raise Exception("Set operation fails to change item value")
