def corrupt_cas_is_healed_on_reboot(self): self.log.info('Start corrupt_cas_is_healed_on_reboot') KEY_NAME = 'key1' rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') # set a key client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':'value1'})) # client.memcached(KEY_NAME).set('k2', 0, 0,json.dumps({'value':'value2'})) # figure out which node it is on mc_active = client.memcached(KEY_NAME) # set the CAS to -2 and then mutate to increment to -1 and then it should stop there self._corrupt_max_cas(mc_active,KEY_NAME) # print 'max cas k2', mc_active.getMeta('k2')[4] # CAS should be 0 now, do some gets and sets to verify that nothing bad happens # self._restart_memcache('default') remote = RemoteMachineShellConnection(self.master) remote.stop_server() time.sleep(30) remote.start_server() time.sleep(30) rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') mc_active = client.memcached(KEY_NAME) maxCas = mc_active.getMeta(KEY_NAME)[4] self.assertTrue(maxCas == 0, 'max cas after reboot is not 0 it is {0}'.format(maxCas))
def corrupt_cas_is_healed_on_reboot(self): self.log.info('Start corrupt_cas_is_healed_on_reboot') KEY_NAME = 'key1' rest = RestConnection(self.master) # set a key client = VBucketAwareMemcached(rest, 'default') client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value': 'value1'})) # client.memcached(KEY_NAME).set('k2', 0, 0,json.dumps({'value':'value2'})) # figure out which node it is on mc_active = client.memcached(KEY_NAME) # set the CAS to -2 and then mutate to increment to -1 and then it should stop there self._corrupt_max_cas(mc_active, KEY_NAME) corrupt_cas = mc_active.getMeta(KEY_NAME)[4] # self._restart_memcache('default') remote = RemoteMachineShellConnection(self.master) remote.stop_server() time.sleep(30) remote.start_server() time.sleep(30) client = VBucketAwareMemcached(rest, 'default') mc_active = client.memcached(KEY_NAME) curr_cas = mc_active.getMeta(KEY_NAME)[4] self.assertTrue( curr_cas == corrupt_cas, 'Corrupted cas (%s) != curr_cas (%s)' % (corrupt_cas, curr_cas))
def test_meta_rebalance_out(self): KEY_NAME = 'key1' rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, self.bucket) for i in range(10): # set a key value = 'value' + str(i) client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value': value})) vbucket_id = client._get_vBucket_id(KEY_NAME) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(KEY_NAME) mc_master = client.memcached_for_vbucket(vbucket_id) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) cas_active = mc_active.getMeta(KEY_NAME)[4] #print 'cas_a {0} '.format(cas_active) max_cas = int( mc_active.stats('vbucket-details')[ 'vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas']) self.assertTrue( cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active)) # remove that node self.log.info('Remove the node with active data') rebalance = self.cluster.async_rebalance(self.servers[-1:], [], [self.master]) rebalance.result() replica_CAS = mc_replica.getMeta(KEY_NAME)[4] get_meta_resp = mc_replica.getMeta(KEY_NAME, request_extended_meta_data=False) #print 'replica CAS {0}'.format(replica_CAS) #print 'replica ext meta {0}'.format(get_meta_resp) # add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) rebalance.result() # verify the CAS is good client = VBucketAwareMemcached(rest, self.bucket) mc_active = client.memcached(KEY_NAME) active_CAS = mc_active.getMeta(KEY_NAME)[4] #print 'active cas {0}'.format(active_CAS) self.assertTrue( replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format( active_CAS, replica_CAS))
def corrupt_cas_is_healed_on_rebalance_out_in(self): self.log.info('Start corrupt_cas_is_healed_on_rebalance_out_in') KEY_NAME = 'key1' rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') # set a key client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':'value1'})) # figure out which node it is on mc_active = client.memcached(KEY_NAME) mc_replica = client.memcached( KEY_NAME, replica_index=0 ) # set the CAS to -2 and then mutate to increment to -1 and then it should stop there self._corrupt_max_cas(mc_active,KEY_NAME) # CAS should be 0 now, do some gets and sets to verify that nothing bad happens resp = mc_active.get(KEY_NAME) self.log.info( 'get for {0} is {1}'.format(KEY_NAME, resp)) # remove that node self.log.info('Remove the node with -1 max cas') rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master]) #rebalance = self.cluster.async_rebalance([self.master], [], self.servers[-1:]) rebalance.result() replica_CAS = mc_replica.getMeta(KEY_NAME)[4] # add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) #rebalance = self.cluster.async_rebalance([self.master], self.servers[-1:],[]) rebalance.result() # verify the CAS is good client = VBucketAwareMemcached(rest, 'default') mc_active = client.memcached(KEY_NAME) active_CAS = mc_active.getMeta(KEY_NAME)[4] self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS))
def test_CASnotzero(self): # MB-31149 # observe.observeseqnotests.ObserveSeqNoTests.test_CASnotzero # set value, append and check CAS value self.log.info('Starting test_CASnotzero') # without hello(mutationseqencenumber) client = VBucketAwareMemcached(RestConnection(self.master), 'default') KEY_NAME = "test1key" client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'})) client.generic_request( client.memcached(KEY_NAME).append, 'test1key', 'appended data') get_meta_resp = client.generic_request( client.memcached(KEY_NAME).getMeta, 'test1key') self.log.info( 'the CAS value without hello(mutationseqencenumber): {} '.format( get_meta_resp[4])) self.assertNotEqual(get_meta_resp[4], 0) # with hello(mutationseqencenumber) KEY_NAME = "test2key" client.set(KEY_NAME, 0, 0, json.dumps({'value': 'value1'})) h = client.sendHellos( memcacheConstants.PROTOCOL_BINARY_FEATURE_MUTATION_SEQNO) client.generic_request( client.memcached(KEY_NAME).append, 'test2key', 'appended data456') get_meta_resp = client.generic_request( client.memcached(KEY_NAME).getMeta, 'test2key') self.log.info( 'the CAS value with hello(mutationseqencenumber): {} '.format( get_meta_resp[4])) self.assertNotEqual(get_meta_resp[4], 0)
def _getr_items(self, item_count, replica_count, prefix, vprefix=""): time_start = time.time() get_count = 0 last_error = "" error_count = 0 awareness = VBucketAwareMemcached(self.rest, self.default_bucket_name) for r in range(replica_count): for i in range(item_count): retry = True key = prefix + "_key_" + str(i) while retry: client = awareness.memcached(key, r) try: value = client.getr(prefix + "_key_" + str(i))[2] assert(value == vprefix + "_value_" + str(i)) get_count += 1 retry = False except mc_bin_client.MemcachedError as e: last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e) error_count += 1 if e.status == 7: self.log.info("getting new vbucket map {0}") awareness.reset(self.rest) else: retry = False except Exception as e: last_error = "failed to getr key {0}, error: {1}".format(prefix + "_key_" + str(i), e) error_count += 1 retry = False if error_count > 0: self.log.error("got {0} errors, last error: {1}".format(error_count, last_error)) self.log.info("got {0} replica items in {1} seconds".format(get_count, time.time() - time_start)) awareness.done() return get_count
def verify_single_node(self, server, kv_store=1): """This is the verification function for single node backup. Args: server: the master server in the cluster as self.master. kv_store: default value is 1. This is the key of the kv_store of each bucket. If --single-node flag appears in backup commad line, we just backup all the items from a single node (the master node in this case). For each bucket, we request for the vBucketMap. For every key in the kvstore of that bucket, we use hash function to get the vBucketId corresponding to that key. By using the vBucketMap, we can know whether that key is in master node or not. If yes, keep it. Otherwise delete it.""" rest = RestConnection(server) for bucket in self.buckets: VBucketAware = VBucketAwareMemcached(rest, bucket.name) memcacheds, vBucketMap, vBucketMapReplica = VBucketAware.request_map(rest, bucket.name) valid_keys, deleted_keys = bucket.kvs[kv_store].key_set() for key in valid_keys: vBucketId = VBucketAware._get_vBucket_id(key) which_server = vBucketMap[vBucketId] sub = which_server.find(":") which_server_ip = which_server[:sub] if which_server_ip != server.ip: partition = bucket.kvs[kv_store].acquire_partition(key) partition.delete(key) bucket.kvs[kv_store].release_partition(key) self._verify_all_buckets(server, kv_store, self.wait_timeout * 50, self.max_verify, True, 1)
def _verify_es_values(self, src_server, dest_server, kv_store=1, verification_count=10000): cb_rest = RestConnection(src_server) es_rest = RestConnection(dest_server) buckets = self.xd_ref._get_cluster_buckets(src_server) for bucket in buckets: mc = VBucketAwareMemcached(cb_rest, bucket) es_valid = es_rest.all_docs(indices=[bucket.name], size=verification_count) # compare values of es documents to documents in couchbase for row in es_valid[:verification_count]: key = str(row['meta']['id']) try: _, _, doc = mc.get(key) val_src = str(json.loads(doc)['site_name']) val_dest = str(row['doc']['site_name']) if val_src != val_dest: self.xd_ref.fail("Document %s has unexpected value (%s) expected (%s)" % \ (key, val_src, val_dest)) except MemcachedError as e: self.xd_ref.fail( "Error during verification. Index contains invalid key: %s" % key) self._log.info("Verified doc values in couchbase bucket (%s) match values in elastic search" % \ (bucket.name))
def run_load(rest, bucket, task, kv_store): smart = VBucketAwareMemcached(rest, bucket) docs_iterators = task["docs"] do_sets, do_gets, do_deletes, do_with_expiration = RebalanceDataGenerator.decode_ops( task) doc_ids = [] expiration = 0 if do_with_expiration: expiration = task["expiration"] for docs_iterator in docs_iterators: for value in docs_iterator: _value = value.encode("ascii", "ignore") _json = json.loads(_value, encoding="utf-8") _id = _json["meta"]["id"].encode("ascii", "ignore") _value = json.dumps(_json["json"]).encode("ascii", "ignore") # _value = json.dumps(_json) try: RebalanceDataGenerator.do_mc(rest, smart, _id, _value, kv_store, doc_ids, do_sets, do_gets, do_deletes, do_with_expiration, expiration) except: traceback.print_exc(file=sys.stdout) #post the results into the queue return
def test_capi_with_checkpointing(self): repl_id = self._start_es_replication(xdcr_params={"checkpointInterval":"60"}) rest_conn = RestConnection(self.src_master) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true') gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false') self.sleep(120) vb0_node = None nodes = self.src_cluster.get_nodes() ip = VBucketAwareMemcached(rest_conn,'default').vBucketMap[0].split(':')[0] for node in nodes: if ip == node.ip: vb0_node = node if not vb0_node: raise XDCRCheckpointException("Error determining the node containing active vb0") vb0_conn = RestConnection(vb0_node) try: checkpoint_record = vb0_conn.get_recent_xdcr_vb_ckpt(repl_id) self.log.info("Checkpoint record : {0}".format(checkpoint_record)) except Exception as e: raise XDCRCheckpointException("Error retrieving last checkpoint document - {0}".format(e)) self._verify_es_results()
def test_capi_with_checkpointing(self): self.setup_xdcr() self.src_cluster.pause_all_replications() gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', range(100), start=0, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.src_cluster.resume_all_replications() self._wait_for_replication_to_catchup() self.sleep(120) vb0_node = None nodes = self.src_cluster.get_nodes() ip = VBucketAwareMemcached(self.rest, 'default').vBucketMap[0].split(':')[0] for node in nodes: if ip == node.ip: vb0_node = node if not vb0_node: raise XDCRCheckpointException("Error determining the node containing active vb0") vb0_conn = RestConnection(vb0_node) try: repl = vb0_conn.get_replication_for_buckets('default', 'default') checkpoint_record = vb0_conn.get_recent_xdcr_vb_ckpt(repl['id']) self.log.info("Checkpoint record : {0}".format(checkpoint_record)) except Exception as e: raise XDCRCheckpointException("Error retrieving last checkpoint document - {0}".format(e)) self._verify_es_results()
def _check_cas(self, check_conflict_resolution=False, master=None, bucket=None, time_sync=None): self.log.info(' Verifying cas and max cas for the keys') #select_count = 20 #Verifying top 20 keys if master: self.rest = RestConnection(master) self.client = VBucketAwareMemcached(self.rest, bucket) k=0 while k < self.items: key = "{0}{1}".format(self.prefix, k) k += 1 mc_active = self.client.memcached(key) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) #print 'max_cas is {0}'.format(max_cas) self.assertTrue(cas == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas)) if check_conflict_resolution: get_meta_resp = mc_active.getMeta(key, request_extended_meta_data=False) if time_sync == 'enabledWithoutDrift': self.assertTrue( get_meta_resp[5] == 1, msg='[ERROR] Metadata indicate conflict resolution is not set') elif time_sync == 'disabled': self.assertTrue( get_meta_resp[5] == 0, msg='[ERROR] Metadata indicate conflict resolution is set')
def _load_ops(self, ops=None, mutations=1, master=None, bucket=None): if master: self.rest = RestConnection(master) if bucket: self.client = VBucketAwareMemcached(self.rest, bucket) k=0 payload = MemcachedClientHelper.create_value('*', self.value_size) while k < self.items: key = "{0}{1}".format(self.prefix, k) k += 1 for i in range(mutations): if ops=='set': #print 'set' self.client.memcached(key).set(key, 0, 0, payload) elif ops=='add': #print 'add' self.client.memcached(key).add(key, 0, 0, payload) elif ops=='replace': self.client.memcached(key).replace(key, 0, 0, payload) #print 'Replace' elif ops=='delete': #print 'delete' self.client.memcached(key).delete(key) elif ops=='expiry': #print 'expiry' self.client.memcached(key).set(key, self.expire_time, 0, payload) elif ops=='touch': #print 'touch' self.client.memcached(key).touch(key, 10) self.log.info("Done with specified {0} ops".format(ops))
def do_get_random_key(self): # MB-31548, get_Random key gets hung sometimes. self.log.info("Creating few docs in the bucket") rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') key = "test_docs-" for index in range(1000): doc_key = key + str(index) client.memcached(doc_key).set(doc_key, 0, 0, json.dumps({'value': 'value1'})) self.log.info("Performing random_gets") mc = MemcachedClient(self.master.ip, 11210) mc.sasl_auth_plain(self.master.rest_username, self.master.rest_password) mc.bucket_select('default') count = 0 while (count < 1000000): count = count + 1 try: mc.get_random_key() except MemcachedError as error: self.fail("<MemcachedError #%d ``%s''>" % (error.status, error.message)) if count % 1000 == 0: self.log.info('The number of iteration is {}'.format(count))
def test_full_eviction_changed_to_value_eviction(self): KEY_NAME = 'key1' gen_create = BlobGenerator('eviction', 'eviction-', self.value_size, end=self.num_items) gen_create2 = BlobGenerator('eviction2', 'eviction2-', self.value_size, end=self.num_items) self._load_all_buckets(self.master, gen_create, "create", 0) self._wait_for_stats_all_buckets(self.servers[:self.nodes_init]) self._verify_stats_all_buckets(self.servers[:self.nodes_init]) remote = RemoteMachineShellConnection(self.master) for bucket in self.buckets: output, _ = remote.execute_couchbase_cli(cli_command='bucket-edit', cluster_host="localhost:8091", user=self.master.rest_username, password=self.master.rest_password, options='--bucket=%s --bucket-eviction-policy=valueOnly' % bucket.name) self.assertTrue(' '.join(output).find('SUCCESS') != -1, 'Eviction policy wasn\'t changed') ClusterOperationHelper.wait_for_ns_servers_or_assert( self.servers[:self.nodes_init], self, wait_time=self.wait_timeout, wait_if_warmup=True) self.sleep(10, 'Wait some time before next load') # self._load_all_buckets(self.master, gen_create2, "create", 0) rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') mcd = client.memcached(KEY_NAME) try: rc = mcd.set(KEY_NAME, 0, 0, json.dumps({'value': 'value2'})) self.fail('Bucket is incorrectly functional') except MemcachedError as e: pass # this is the exception we are hoping for
def key_not_exists_test(self): self.assertTrue(len(self.buckets) > 0, 'at least 1 bucket required') bucket = self.buckets[0].name client = VBucketAwareMemcached(RestConnection(self.master), bucket) KEY_NAME = 'key' for i in range(1500): client.set(KEY_NAME, 0, 0, "x") # delete and verify get fails client.delete(KEY_NAME) err = None try: rc = client.get(KEY_NAME) except MemcachedError as error: # It is expected to raise MemcachedError because the key is deleted. err = error.status self.assertTrue(err == ERR_NOT_FOUND, 'expected key to be deleted {0}'.format(KEY_NAME)) #cas errors do not sleep the test for 10 seconds, plus we need to check that the correct #error is being thrown err = None try: #For some reason replace instead of cas would not reproduce the bug mc_active = client.memcached(KEY_NAME) mc_active.replace(KEY_NAME, 0, 10, "value") except MemcachedError as error: err = error.status self.assertTrue( err == ERR_NOT_FOUND, 'was able to replace cas on removed key {0}'.format(KEY_NAME))
def do_verification(kv_store, rest, bucket): keys = kv_store.keys() smart = VBucketAwareMemcached(rest, bucket) validation_failures = {} for k in keys: expected = kv_store.read(k) if expected: if expected["status"] == "deleted": # it should not exist there ? try: smart.memcached(k).get(k) validation_failures[k] = ["deleted key"] except MemcachedError: pass elif expected["status"] == "expired": try: smart.memcached(k).get(k) validation_failures[k] = ["expired key"] except MemcachedError: pass else: try: x, y, value = smart.memcached(k).get(k) actualmd5 = hashlib.md5(value).digest() if actualmd5 != expected["value"]: validation_failures[k] = ["value mismatch"] except MemcachedError: validation_failures[k] = ["key not found"] return validation_failures
def verify_one_node_has_time_sync_and_one_does_not(self): # need to explicitly enable and disable sync when it is supported self.log.info( '\n\nStarting verify_one_node_has_time_sync_and_one_does_not') client = VBucketAwareMemcached(RestConnection(self.master), 'default') vbucket_id = client._get_vBucket_id(LWW_EP_Engine.TEST_KEY) mc_master = client.memcached_for_vbucket(vbucket_id) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) # set for master but not for the replica result = mc_master.set_time_drift_counter_state(vbucket_id, 0, 1) mc_master.set(LWW_EP_Engine.TEST_KEY, 0, 0, LWW_EP_Engine.TEST_VALUE) get_meta_resp = mc_master.getMeta(LWW_EP_Engine.TEST_KEY, request_extended_meta_data=True) self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicates conflict resolution is not set') self.log.info( '\n\nEnding verify_one_node_has_time_sync_and_one_does_not')
def insert_docs(self, num_of_docs, prefix='doc', extra_values={}, return_docs=False, scope=None, collection=None): random.seed(12345) rest = RestConnection(self.master) smart = VBucketAwareMemcached(rest, self.bucket) doc_names = [] for i in range(0, num_of_docs): key = doc_name = "{0}-{1}".format(prefix, i) geom = { "type": "Point", "coordinates": [random.randrange(-180, 180), random.randrange(-90, 90)] } value = { "name": doc_name, "age": random.randrange(1, 1000), "geometry": geom, "height": random.randrange(1, 13000), "bloom": random.randrange(1, 6), "shed_leaves": random.randrange(6, 13) } value.update(extra_values) if not return_docs: doc_names.append(doc_name) else: doc_names.append(value) # loop till value is set fail_count = 0 while True: try: smart.set(key, 0, 0, json.dumps(value), scope=scope, collection=collection) break except MemcachedError as e: fail_count += 1 if (e.status == 133 or e.status == 132 or e.status == 134) and fail_count < 60: if i == 0: self.log.error( "waiting 5 seconds. error {0}".format(e)) time.sleep(5) else: self.log.error(e) time.sleep(1) else: raise e self.log.info("Inserted {0} json documents".format(num_of_docs)) return doc_names
def insert_key(serverInfo, bucket_name, count, size): rest = RestConnection(serverInfo) smart = VBucketAwareMemcached(rest, bucket_name) for i in xrange(count * 1000): key = "key_" + str(i) flag = random.randint(1, 999) value = {"value": MemcachedClientHelper.create_value("*", size)} smart.memcached(key).set(key, 0, 0, json.dumps(value))
def _poxi(self): tServer = TestInputServer() tServer.ip = self.server_ip tServer.rest_username = "******" tServer.rest_password = "******" tServer.port = 8091 rest = RestConnection(tServer) return VBucketAwareMemcached(rest, self.bucket_name)
def verify_vBuckets_info(master, bucket="default"): ''' verify vBuckets' state and items count(for active/replica) in them related to vBucketMap for all nodes in cluster ''' log = logging.getLogger("infra") awareness = VBucketAwareMemcached(RestConnection(master), bucket) vb_map = awareness.vBucketMap vb_mapReplica = awareness.vBucketMapReplica replica_num = len(vb_mapReplica[0]) #get state and count items for all vbuckets for each node node_stats = RebalanceHelper.get_vBuckets_info(master) state = True #iterate throught all vbuckets by their numbers for num in vb_map: #verify that active vbucket in memcached is also active in stats("hash) if (node_stats[vb_map[num]]["vb_" + str(num)][0] != "active"): log.info("vBucket {0} in {1} node has wrong state {3}".format( "vb_" + str(num), vb_map[num], node_stats[vb_map[num]]["vb_" + str(num)])) state = False #number of active items for num vBucket vb = node_stats[vb_map[num]]["vb_" + str(num)][1] active_vb = vb_map[num] #list of nodes for wich num vBucket is replica replica_vbs = vb_mapReplica[key] sum_items_replica = 0 #sum of replica items for all nodes for num vBucket for i in range(replica_num): if (node_stats[vb_mapReplica[num][i]]["vb_" + str(num)][0] != "replica"): log.info( "vBucket {0} in {1} node has wrong state {3}".format( "vb_" + str(num), vb_mapReplica[num], node_stats[vb_mapReplica[num]]["vb_" + str(num)])) state = False sum_items_replica += int( node_stats[replica_vbs[i]]["vb_" + str(num)][1]) #print information about the discrepancy of the number of replica and active items for num vBucket if (int(vb) * len(vb_mapReplica[num]) != sum_items_replica): log.info( "sum of active items doesn't correspond to replica's vBucets in {0} vBucket:" .format("vb_" + str(num))) log.info("items in active vBucket {0}:{1}".format( vb_map[num], node_stats[vb_map[num]]["vb_" + str(num)])) for j in range(replica): log.info("items in replica vBucket {0}: {1}".format( vb_mapReplica[num][j], node_stats[vb_mapReplica[num][j]]["vb_" + str(num)])) log.info(node_stats[vb_mapReplica[num][0]]) state = False if not state: log.error("Something is wrong, see log above. See details:") log.error("vBucetMap: {0}".format(vb_map)) log.error("vBucetReplicaMap: {0}".format(vb_mapReplica)) log.error("node_stats: {0}".format(node_stats)) return state
def test_meta_hard_restart(self): KEY_NAME = 'key2' rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, self.bucket) for i in range(10): # set a key value = 'value' + str(i) client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value': value})) vbucket_id = client._get_vBucket_id(KEY_NAME) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(KEY_NAME) mc_master = client.memcached_for_vbucket(vbucket_id) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) cas_pre = mc_active.getMeta(KEY_NAME)[4] #print 'cas_a {0} '.format(cas_pre) max_cas = int( mc_active.stats('vbucket-details')[ 'vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas']) self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre)) # reboot nodes self._reboot_server() time.sleep(60) # verify the CAS is good client = VBucketAwareMemcached(rest, self.bucket) mc_active = client.memcached(KEY_NAME) cas_post = mc_active.getMeta(KEY_NAME)[4] #print 'post cas {0}'.format(cas_post) get_meta_resp = mc_active.getMeta(KEY_NAME, request_extended_meta_data=False) #print 'post CAS {0}'.format(cas_post) #print 'post ext meta {0}'.format(get_meta_resp) self.assertTrue( cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post))
def setUp(self): super(OpsChangeCasTests, self).setUp() self.prefix = "test_" self.expire_time = self.input.param("expire_time", 35) self.item_flag = self.input.param("item_flag", 0) self.value_size = self.input.param("value_size", 256) self.items = self.input.param("items", 20) self.rest = RestConnection(self.master) self.client = VBucketAwareMemcached(self.rest, self.bucket)
def get_active_vb0_node(self, master): nodes = self.src_nodes ip = VBucketAwareMemcached(RestConnection(master), 'default').vBucketMap[0].split(':')[0] if master == self.dest_master: nodes = self.dest_nodes for node in nodes: if ip == node.ip: return node raise XDCRCheckpointException("Error determining the node containing active vb0")
def test_MB_32114(self): try: from sdk_client import SDKClient except: from sdk_client3 import SDKClient import couchbase.subdocument as SD rest = RestConnection(self.master) client = VBucketAwareMemcached(rest, 'default') if self.maxttl: self._expiry_pager(self.master) sdk_client = SDKClient(scheme='couchbase', hosts=[self.master.ip], bucket='default') KEY_NAME = 'key1' for i in range(1000): mcd = client.memcached(KEY_NAME + str(i)) rc = mcd.set(KEY_NAME + str(i), 0, 0, json.dumps({'value': 'value2'})) sdk_client.mutate_in( KEY_NAME + str(i), SD.upsert("subdoc_key", "subdoc_val", xattr=True, create_parents=True)) # wait for it to persist persisted = 0 while persisted == 0: opaque, rep_time, persist_time, persisted, cas = client.observe( KEY_NAME + str(i)) start_time = time.time() self._load_doc_data_all_buckets(batch_size=1000) end_time = time.time() for i in range(1000): try: mcd = client.memcached(KEY_NAME + str(i)) _, flags, exp, seqno, cas = client.memcached( KEY_NAME + str(i)).getMeta(KEY_NAME + str(i)) rc = mcd.del_with_meta(KEY_NAME + str(i), 0, 0, 2, cas + 1) except MemcachedError as exp: self.fail("Exception with del_with meta - {0}".format(exp)) self.cluster.compact_bucket(self.master, "default") if self.maxttl: time_to_sleep = (self.maxttl - (end_time - start_time)) + 20 self.sleep(int(time_to_sleep)) else: self.sleep(60) active_bucket_items = rest.get_active_key_count("default") replica_bucket_items = rest.get_replica_key_count("default") print('active_bucket_items ', active_bucket_items) print('replica_bucket_items ', replica_bucket_items) if active_bucket_items * self.num_replicas != replica_bucket_items: self.fail("Mismatch in data !!!")
def test_rollback_and_persistence_race_condition(self): nodeA = self.servers[0] vbucket_client = VBucketAwareMemcached(RestConnection(self.master), 'default') gen_create = BlobGenerator('dcp', 'dcp-', 64, start=0, end=self.num_items) self._load_all_buckets(nodeA, gen_create, "create", 0) # stop persistence for bucket in self.buckets: for s in self.servers[:self.nodes_init]: client = MemcachedClientHelper.direct_client(s, bucket) try: client.stop_persistence() except MemcachedError as e: if self.bucket_type == 'ephemeral': self.assertTrue( "Memcached error #4 'Invalid': Flusher not running. for vbucket :0 to mc " in e.message) return else: raise vb_uuid, seqno, high_seqno = self.vb_info(self.servers[0], 5) time.sleep(10) # more (non-intersecting) load gen_create = BlobGenerator('dcp-secondgroup', 'dcpsecondgroup-', 64, start=0, end=self.num_items) self._load_all_buckets(nodeA, gen_create, "create", 0) shell = RemoteMachineShellConnection(self.servers[0]) shell.kill_memcached() time.sleep(10) mc1 = MemcachedClientHelper.direct_client(self.servers[0], "default") mc2 = MemcachedClientHelper.direct_client(self.servers[1], "default") node1_items = mc1.stats()["curr_items_tot"] node2_items = mc2.stats()["curr_items_tot"] self.assertTrue( node1_items == node2_items, 'Node items not equal. Node 1:{0}, node 2:{1}'.format( node1_items, node2_items))
def connect_host_port(self, host, port, user, pswd): from membase.api.rest_client import RestConnection from memcached.helper.data_helper import VBucketAwareMemcached info = { "ip": host, "port": port, 'username': user or 'Administrator', 'password': pswd or 'password' } rest = RestConnection(info) self.awareness = VBucketAwareMemcached(rest, user or 'default', info) self.backoff = 0 self.xfer_sent = 0 self.xfer_recv = 0
def _verify_es_results(self, bucket='default'): es_docs = self.esrest_conn.all_docs() self.log.info("Retrieved ES Docs") memcached_conn = VBucketAwareMemcached(self.rest, bucket) self.log.info("Comparing CB and ES data") for doc in es_docs: es_data = doc['doc'] mc_active = memcached_conn.memcached(str(es_data['_id'])) cb_flags, cb_cas, cb_data = mc_active.get(str(es_data['_id'])) self.assertDictEqual(es_data, json.loads(cb_data), "Data mismatch found - es data: {0} cb data: {1}". format(str(es_data), str(cb_data))) self.log.info("Data verified")
def _reader_thread(self, inserted_keys, bucket_data, moxi=False): errors = [] rest = RestConnection(self._servers[0]) smartclient = None for name in bucket_data: for key in inserted_keys: if moxi: moxi = MemcachedClientHelper.proxy_client(self._servers[0], name) else: smartclient = VBucketAwareMemcached(rest, name) try: if moxi: moxi.get(key) else: smartclient.memcached(key).get(key) except Exception as ex: errors.append({"error": ex, "key": key}) self.log.info(ex) if not moxi: smartclient.done() smartclient = VBucketAwareMemcached(rest, name)