def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000): cb_rest = RestConnection(src_server) es_rest = RestConnection(dest_server) buckets = self.xd_ref._get_cluster_buckets(src_server) for bucket in buckets: mc = VBucketAwareMemcached(cb_rest, bucket) es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count) # compare values of es documents to documents in couchbase for row in es_valid[:verification_count]: key = str(row["meta"]["id"]) try: _, _, doc = mc.get(key) val_src = str(json.loads(doc)["site_name"]) val_dest = str(row["doc"]["site_name"]) if val_src != val_dest: self.xd_ref.fail( "Document %s has unexpected value (%s) expected (%s)" % (key, val_src, val_dest) ) except MemcachedError as e: self.xd_ref.fail("Error during verification. Index contains invalid key: %s" % key) self._log.info( "Verified doc values in couchbase bucket (%s) match values in elastic search" % (bucket.name) )
def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000): cb_rest = RestConnection(src_server) es_rest = RestConnection(dest_server) buckets = self.xd_ref._get_cluster_buckets(src_server) for bucket in buckets: mc = VBucketAwareMemcached(cb_rest, bucket) es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count) # compare values of es documents to documents in couchbase for row in es_valid[:verification_count]: key = str(row['meta']['id']) try: _, _, doc = mc.get(key) val_src = str(json.loads(doc)['site_name']) val_dest = str(row['doc']['site_name']) if val_src != val_dest: self.xd_ref.fail("Document %s has unexpected value (%s) expected (%s)" % \ (key, val_src, val_dest)) except MemcachedError as e: self.xd_ref.fail( "Error during verification. Index contains invalid key: %s" % key) self._log.info("Verified doc values in couchbase bucket (%s) match values in elastic search" % \ (bucket.name))
def key_not_exists_test(self): self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required') bucket = self.buckets[0].name client = VBucketAwareMemcached(RestConnection(self.master), bucket) KEY_NAME = 'key' for i in range(1500): client.set(KEY_NAME, 0, 0, "x") # delete and verify get fails client.delete(KEY_NAME) err = None try: rc = client.get(KEY_NAME) except MemcachedError as error: # It is expected to raise MemcachedError because the key is deleted. err = error.status self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME)) #cas errors do not sleep the test for 10 seconds, plus we need to check that the correct #error is being thrown err = None try: #For some reason replace instead of cas would not reproduce the bug mc_active = client.memcached(KEY_NAME) mc_active.replace(KEY_NAME, 0, 10, "value") except MemcachedError as error: err = error.status self.assertTrue( err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def verify_missing_keys(self, prefix, count): """ verify that none of the keys exist with specified prefix. """ client = VBucketAwareMemcached(RestConnection(self.servers[0]), self.buckets[0]) for i in range(count): key = "{0}{1}".format(prefix, i) try: client.get(key) except MemcachedError as error: self.assertEqual( error.status, constants.ERR_NOT_FOUND, "expected error NOT_FOUND, got {0}".format(error.status)) else: self.fail("Able to retrieve expired document")
def key_not_exists_test(self): self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required') bucket = self.buckets[0].name client = VBucketAwareMemcached(RestConnection(self.master), bucket) KEY_NAME = 'key' for i in range(1500): client.set(KEY_NAME, 0, 0, "x") # delete and verify get fails client.delete(KEY_NAME) err = None try: rc = client.get(KEY_NAME) except MemcachedError as error: # It is expected to raise MemcachedError because the key is deleted. err = error.status self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME)) #cas errors do not sleep the test for 10 seconds, plus we need to check that the correct #error is being thrown err = None try: #For some reason replace instead of cas would not reproduce the bug mc_active = client.memcached(KEY_NAME) mc_active.replace(KEY_NAME, 0, 10, "value") except MemcachedError as error: err = error.status self.assertTrue(err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def verify_missing_keys(self, prefix, count): """ verify that none of the keys exist with specified prefix. """ client = VBucketAwareMemcached( RestConnection(self.servers[0]), self.buckets[0]) for i in xrange(count): key = "{0}{1}".format(prefix, i) try: client.get(key) except MemcachedError as error: self.assertEquals( error.status, constants.ERR_NOT_FOUND, "expected error NOT_FOUND, got {0}".format(error.status)) else: self.fail("Able to retrieve expired document")
def _verify_data_all_buckets(self, gen_check): for bucket in self.buckets: self.log.info("Check bucket %s" % bucket.name) gen = copy.deepcopy(gen_check) rest = RestConnection(self.server_recovery) client = VBucketAwareMemcached(rest, bucket) while gen.has_next(): key, value = next(gen) try: _, _, d = client.get(key) self.assertEqual(d.decode("utf-8"), value,\ 'Key: %s expected. Value expected %s. Value actual %s' % (key, value, d)) except Exception as ex: raise Exception('Key %s not found %s' % (key, str(ex)))
def _verify_data_all_buckets(self, gen_check): for bucket in self.buckets: self.log.info("Check bucket %s" % bucket.name) gen = copy.deepcopy(gen_check) rest = RestConnection(self.server_recovery) client = VBucketAwareMemcached(rest, bucket) while gen.has_next(): key, value = gen.next() try: _, _, d = client.get(key) self.assertEquals(d, value, 'Key: %s expected. Value expected %s. Value actual %s' % ( key, value, d)) except Exception, ex: raise Exception('Key %s not found %s' % (key, str(ex)))
def verify_cas(self, ops, generator): """Verify CAS value manipulation. For update we use the latest CAS value return by set() to do the mutation again to see if there is any exceptions. We should be able to mutate that item with the latest CAS value. For delete(), after it is called, we try to mutate that item with the cas vavlue returned by delete(). We should see Memcached Error. Otherwise the test should fail. For expire, We want to verify using the latest CAS value of that item can not mutate it because it is expired already.""" client = VBucketAwareMemcached(RestConnection(self.master), 'default') for bucket in self.buckets: gen = generator cas_error_collection = [] data_error_collection = [] while gen.has_next(): key, value = gen.next() if ops in ["update","touch"]: for x in range(self.mutate_times): o_old, cas_old, d_old = client.get(key) if ops == 'update': client.memcached(key).cas(key, 0, 0, cas_old, "{0}-{1}".format("mysql-new-value", x)) else: rc = client.memcached(key).touch(key,10) o_new, cas_new, d_new = client.memcached(key).get(key) if cas_old == cas_new: print 'cas did not change' cas_error_collection.append(cas_old) if ops == 'update': if d_new != "{0}-{1}".format("mysql-new-value", x): data_error_collection.append((d_new,"{0}-{1}".format("mysql-new-value", x))) if cas_old != cas_new and d_new == "{0}-{1}".format("mysql-new-value", x): self.log.info("Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} " .format(cas_old, key, cas_new)) mc_active = client.memcached( key ) mc_replica = client.memcached( key, replica_index=0 ) active_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] ) replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(active_cas == cas_new, 'cbstats cas mismatch. Expected {0}, actual {1}'.format( cas_new, active_cas)) replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] ) poll_count = 0 while replica_cas != active_cas and poll_count < 5: time.sleep(1) replica_cas = int( mc_replica.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(key)) + ':max_cas'] ) poll_count = poll_count + 1 if poll_count > 0: self.log.info('Getting the desired CAS was delayed {0} seconds'.format( poll_count) ) self.assertTrue(active_cas == replica_cas, 'replica cas mismatch. Expected {0}, actual {1}'.format( cas_new, replica_cas)) elif ops == "delete": o, cas, d = client.memcached(key).delete(key) time.sleep(10) self.log.info("Delete operation set item cas with key {0} to {1}".format(key, cas)) try: client.memcached(key).cas(key, 0, self.item_flag, cas, value) raise Exception("The item should already be deleted. We can't mutate it anymore") except MemcachedError as error: #It is expected to raise MemcachedError because the key is deleted. if error.status == ERR_NOT_FOUND: self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg)) pass else: raise Exception(error) elif ops == "expire": o, cas, d = client.memcached(key).set(key, self.expire_time, 0, value) time.sleep(self.expire_time+1) self.log.info("Try to mutate an expired item with its previous cas {0}".format(cas)) try: client.memcached(key).cas(key, 0, self.item_flag, cas, value) raise Exception("The item should already be expired. We can't mutate it anymore") except MemcachedError as error: #It is expected to raise MemcachedError becasue the key is expired. if error.status == ERR_NOT_FOUND: self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg)) pass else: raise Exception(error) if len(cas_error_collection) > 0: for cas_value in cas_error_collection: self.log.error("Set operation fails to modify the CAS {0}".format(cas_value)) raise Exception("Set operation fails to modify the CAS value") if len(data_error_collection) > 0: for data_value in data_error_collection: self.log.error("Set operation fails. item-value is {0}, expected is {1}".format(data_value[0], data_value[1])) raise Exception("Set operation fails to change item value")
class GenericLoadingTask(Thread, Task): def __init__(self, server, bucket, kv_store): Thread.__init__(self) Task.__init__(self, "load_gen_task") self.kv_store = kv_store self.client = VBucketAwareMemcached(RestConnection(server), bucket) def execute(self, task_manager): self.start() self.state = EXECUTING def check(self, task_manager): pass def run(self): while self.has_next() and not self.done(): self.next() self.state = FINISHED self.set_result(True) def has_next(self): raise NotImplementedError def next(self): raise NotImplementedError def _unlocked_create(self, partition, key, value): try: value_json = json.loads(value) value_json['mutated'] = 0 value = json.dumps(value_json) except ValueError: index = random.choice(range(len(value))) value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:] try: self.client.set(key, self.exp, 0, value) partition.set(key, value, self.exp) except MemcachedError as error: self.state = FINISHED self.set_exception(error) def _unlocked_read(self, partition, key): try: o, c, d = self.client.get(key) except MemcachedError as error: if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None: pass else: self.state = FINISHED self.set_exception(error) def _unlocked_update(self, partition, key): value = partition.get_valid(key) if value is None: return try: value_json = json.loads(value) value_json['mutated'] += 1 value = json.dumps(value_json) except ValueError: index = random.choice(range(len(value))) value = value[0:index] + random.choice(string.ascii_uppercase) + value[index+1:] try: self.client.set(key, self.exp, 0, value) partition.set(key, value, self.exp) except MemcachedError as error: self.state = FINISHED self.set_exception(error) def _unlocked_delete(self, partition, key): try: self.client.delete(key) partition.delete(key) except MemcachedError as error: if error.status == ERR_NOT_FOUND and partition.get_valid(key) is None: pass else: self.state = FINISHED self.set_exception(error)
def verify_cas(self, ops, generator): """Verify CAS value manipulation. For update we use the latest CAS value return by set() to do the mutation again to see if there is any exceptions. We should be able to mutate that item with the latest CAS value. For delete(), after it is called, we try to mutate that item with the cas vavlue returned by delete(). We should see Memcached Error. Otherwise the test should fail. For expire, We want to verify using the latest CAS value of that item can not mutate it because it is expired already.""" for bucket in self.buckets: client = VBucketAwareMemcached(RestConnection(self.master), bucket.name) gen = generator cas_error_collection = [] data_error_collection = [] while gen.has_next(): key, value = next(gen) if ops in ["update", "touch"]: for x in range(self.mutate_times): o_old, cas_old, d_old = client.get(key) if ops == 'update': client.memcached(key).cas( key, 0, 0, cas_old, "{0}-{1}".format("mysql-new-value", x)) else: client.memcached(key).touch(key, 10) o_new, cas_new, d_new = client.memcached(key).get(key) if cas_old == cas_new: print('cas did not change') cas_error_collection.append(cas_old) if ops == 'update': if d_new != "{0}-{1}".format("mysql-new-value", x): data_error_collection.append( (d_new, "{0}-{1}".format("mysql-new-value", x))) if cas_old != cas_new and d_new == "{0}-{1}".format( "mysql-new-value", x): self.log.info( "Use item cas {0} to mutate the same item with key {1} successfully! Now item cas is {2} " .format(cas_old, key, cas_new)) mc_active = client.memcached(key) mc_replica = client.memcached(key, replica_index=0) active_cas = int( mc_active.stats('vbucket-details') ['vb_' + str(client._get_vBucket_id(key)) + ':max_cas']) self.assertTrue( active_cas == cas_new, 'cbstats cas mismatch. Expected {0}, actual {1}'. format(cas_new, active_cas)) replica_cas = int( mc_replica.stats('vbucket-details') ['vb_' + str(client._get_vBucket_id(key)) + ':max_cas']) poll_count = 0 while replica_cas != active_cas and poll_count < 5: time.sleep(1) replica_cas = int( mc_replica.stats('vbucket-details') ['vb_' + str(client._get_vBucket_id(key)) + ':max_cas']) poll_count = poll_count + 1 if poll_count > 0: self.log.info( 'Getting the desired CAS was delayed {0} seconds' .format(poll_count)) self.assertTrue( active_cas == replica_cas, 'replica cas mismatch. Expected {0}, actual {1}'. format(cas_new, replica_cas)) elif ops == "delete": o, cas, d = client.memcached(key).delete(key) time.sleep(10) self.log.info( "Delete operation set item cas with key {0} to {1}". format(key, cas)) try: client.memcached(key).cas(key, 0, self.item_flag, cas, value) raise Exception( "The item should already be deleted. We can't mutate it anymore" ) except MemcachedError as error: # It is expected to raise MemcachedError because the key is deleted. if error.status == ERR_NOT_FOUND: self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg)) pass else: raise Exception(error) elif ops == "expire": o, cas, d = client.memcached(key).set( key, self.expire_time, 0, value) time.sleep(self.expire_time + 1) self.log.info( "Try to mutate an expired item with its previous cas {0}" .format(cas)) try: client.memcached(key).cas(key, 0, self.item_flag, cas, value) raise Exception( "The item should already be expired. We can't mutate it anymore" ) except MemcachedError as error: # It is expected to raise MemcachedError becasue the key is expired. if error.status == ERR_NOT_FOUND: self.log.info("<MemcachedError #%d ``%s''>" % (error.status, error.msg)) pass else: raise Exception(error) if len(cas_error_collection) > 0: for cas_value in cas_error_collection: self.log.error( "Set operation fails to modify the CAS {0}".format( cas_value)) raise Exception("Set operation fails to modify the CAS value") if len(data_error_collection) > 0: for data_value in data_error_collection: self.log.error( "Set operation fails. item-value is {0}, expected is {1}" .format(data_value[0], data_value[1])) raise Exception("Set operation fails to change item value")