def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create', compression=self.sdk_compression) except Exception as e: log.info("error while loading data") self.deploy_function(body,wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() # Wait for bootstrap to complete self.wait_for_bootstrap_to_complete(body['appname']) stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def iostats(self, nodes, frequency, verbosity=False): shells = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name MemcachedClientHelper.direct_client(node, bucket) shells.append(RemoteMachineShellConnection(node)) except: pass self._task["iostats"] = [] log.info("Started capturing io stats") while not self._aborted(): time.sleep(frequency) log.info("Collecting io_stats") for shell in shells: try: kB_read, kB_wrtn, util, iowait, idle = \ self._extract_io_info(shell) except (ValueError, TypeError, IndexError): continue if kB_read and kB_wrtn: self._task["iostats"].append({ "time": time.time(), "ip": shell.ip, "read": kB_read, "write": kB_wrtn, "util": util, "iowait": iowait, "idle": idle }) log.info("Finished capturing io stats")
def system_stats(self, nodes, pnames, frequency, verbosity=False): shells = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name MemcachedClientHelper.direct_client(node, bucket) shells.append(RemoteMachineShellConnection(node)) except: pass d = {"snapshots": []} # "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}] start_time = str(self._task["time"]) while not self._aborted(): time.sleep(frequency) current_time = time.time() i = 0 for shell in shells: node = nodes[i] unique_id = node.ip + '-' + start_time for pname in pnames: obj = RemoteMachineHelper(shell).is_process_running(pname) if obj and obj.pid: value = self._extract_proc_info(shell, obj.pid) value["name"] = pname value["id"] = obj.pid value["unique_id"] = unique_id value["time"] = current_time value["ip"] = node.ip d["snapshots"].append(value) i += 1 self._task["systemstats"] = d["snapshots"] log.info("Finished system_stats")
def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create') except Exception as e: log.info("error while loading data") self.deploy_function(body,wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() # Wait for bootstrap to complete self.wait_for_bootstrap_to_complete(body['appname']) stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def system_stats(self, nodes, pnames, frequency, verbosity=False): shells = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name MemcachedClientHelper.direct_client(node, bucket) shells.append(RemoteMachineShellConnection(node)) except: pass d = {"snapshots": []} # "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}] start_time = str(self._task["time"]) while not self._aborted(): time.sleep(frequency) current_time = time.time() i = 0 for shell in shells: node = nodes[i] unique_id = node.ip + '-' + start_time for pname in pnames: obj = RemoteMachineHelper(shell).is_process_running(pname) if obj and obj.pid: value = self._extract_proc_info(shell, obj.pid) value["name"] = pname value["id"] = obj.pid value["unique_id"] = unique_id value["time"] = current_time value["ip"] = node.ip d["snapshots"].append(value) i += 1 self._task["systemstats"] = d["snapshots"] log.info("Finished system_stats")
def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) if self.is_curl: body['depcfg']['curl'] = [] body['depcfg']['curl'].append({"hostname": self.hostname, "value": "server", "auth_type": self.auth_type, "username": self.curl_username, "password": self.curl_password,"cookies": self.cookies}) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create', compression=self.sdk_compression) except Exception as e: log.info("error while loading data") self.deploy_function(body, wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() self.wait_for_handler_state(body['appname'], "deployed") stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def iostats(self, nodes, frequency, verbosity=False): shells = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name MemcachedClientHelper.direct_client(node, bucket) shells.append(RemoteMachineShellConnection(node)) except: pass self._task["iostats"] = [] log.info("Started capturing io stats") while not self._aborted(): time.sleep(frequency) log.info("Collecting io_stats") for shell in shells: try: kB_read, kB_wrtn, util, iowait, idle = \ self._extract_io_info(shell) except (ValueError, TypeError, IndexError): continue if kB_read and kB_wrtn: self._task["iostats"].append({"time": time.time(), "ip": shell.ip, "read": kB_read, "write": kB_wrtn, "util": util, "iowait": iowait, "idle": idle}) log.info("Finished capturing io stats")
def test_partial_rollback(self): self.multi_create_index() self.sleep(30) self.log.info("Stopping persistence on NodeA & NodeB") data_nodes = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) for data_node in data_nodes: for bucket in self.buckets: mem_client = MemcachedClientHelper.direct_client(data_node, bucket.name) mem_client.stop_persistence() self.run_doc_ops() self.sleep(10) # Get count before rollback bucket_before_item_counts = {} for bucket in self.buckets: bucket_count_before_rollback = self.get_item_count(self.master, bucket.name) bucket_before_item_counts[bucket.name] = bucket_count_before_rollback log.info("Items in bucket {0} before rollback = {1}".format( bucket.name, bucket_count_before_rollback)) # Index rollback count before rollback self._verify_bucket_count_with_index_count() self.multi_query_using_index() # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(data_nodes[0]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") for bucket in self.buckets: mem_client = MemcachedClientHelper.direct_client(data_nodes[1], bucket.name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self.cluster.async_failover( self.servers[:self.nodes_init], [data_nodes[1]], self.graceful, wait_for_pending=120) failover_task.result() # Wait for a couple of mins to allow rollback to complete # self.sleep(120) bucket_after_item_counts = {} for bucket in self.buckets: bucket_count_after_rollback = self.get_item_count(self.master, bucket.name) bucket_after_item_counts[bucket.name] = bucket_count_after_rollback log.info("Items in bucket {0} after rollback = {1}".format( bucket.name, bucket_count_after_rollback)) for bucket in self.buckets: if bucket_after_item_counts[bucket.name] == bucket_before_item_counts[bucket.name]: log.info("Looks like KV rollback did not happen at all.") self._verify_bucket_count_with_index_count() self.multi_query_using_index()
def connect(self): if not self.server or \ not isinstance(self.server, Server): return False self.mc = MemcachedClientHelper.direct_client(self.server, self.bucket) return True
def test_scan_consistency_post_memcached_crash(self): self.log.info('Load documents in the default bucket') self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", 0, self.num_items) self.log.info('Create dataset') self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info('Connect link') self.cbas_util.connect_link() self.log.info('Verify dataset count') self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, self.num_items) self.log.info('Stopping persistence on KV') mem_client = MemcachedClientHelper.direct_client( self.input.servers[0], self.cb_bucket_name) mem_client.stop_persistence() self.log.info('Performing Mutations') self.perform_doc_ops_in_all_cb_buckets(self.num_items, "create", self.num_items, self.num_items * 2) self.log.info('Kill Memcached process') shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() shell.disconnect() self.log.info('Validate count') query = 'select count(*) from %s' % self.cbas_dataset_name dataset_count = 0 start_time = time.time() output = [] while time.time() < start_time + 120: try: response, _, _, results, _ = self.cbas_util.execute_statement_on_cbas_util( query, scan_consistency=self.scan_consistency, scan_wait=self.scan_wait) self.assertEqual(response, "success", "Query failed...") dataset_count = results[0]['$1'] if dataset_count == self.num_items: break except Exception as e: self.log.info('Try again as memcached might be recovering...') self.log.info('Verify dataset count is equal to number of items in KV') count_n1ql = self.rest.query_tool( 'select count(*) from %s' % self.cb_bucket_name)['results'][0]['$1'] self.assertEqual(dataset_count, count_n1ql, msg='KV-CBAS count mismatch. Actual %s, expected %s' % (dataset_count, count_n1ql))
def membase_stats(self, interval=60): mcs = [] for node in self.nodes: try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) mcs.append(mc) except Exception, error: log.error(error)
def connect(self): if not self.server or \ not isinstance(self.server, Server): return False self.mc = MemcachedClientHelper.direct_client( self.server, self.bucket) return True
def membase_stats(self, interval=60): mcs = [] for node in self.nodes: try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) mcs.append(mc) except Exception, error: log.error(error)
def send_json(server_info, job=None): if job: jobs = get_jobs_stats(onlyLastBuild=True, job_url=JENKINS_URL + 'job/' + job) else: jobs = get_jobs_stats(onlyLastBuild=True) jsons = build_json_result(jobs) client = MemcachedClientHelper.direct_client(server_info, CB_BUCKET_NAME) for key, rq in jsons: try: client.set(key, 0, 0, rq) except Exception as ex: sys.exit(str(ex))
def send_json(server_info, job=None): if job: jobs = get_jobs_stats(onlyLastBuild=True, job_url=JENKINS_URL + 'job/' + job) else: jobs = get_jobs_stats(onlyLastBuild=True) jsons = build_json_result(jobs) client = MemcachedClientHelper.direct_client(server_info, CB_BUCKET_NAME) for key, rq in jsons: try: client.set(key, 0, 0, rq) except Exception, ex: sys.exit(str(ex))
def membase_stats(self, nodes, bucket, frequency, verbose=False): mcs = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name mcs.append(MemcachedClientHelper.direct_client(node, bucket)) except: pass self._task["membasestats"] = [] self._task["timings"] = [] self._task["dispatcher"] = [] d = {} # "pname": "x", "pid": "y","snapshots": [{"time": time,"value": value}] for mc in mcs: d[mc.host] = {"snapshots": [], "timings": [], "dispatcher": []} while not self._aborted(): time_left = frequency log.info("Collecting membase stats") timings = None # at minimum we want to check for aborted every minute while not self._aborted() and time_left > 0: time.sleep(min(time_left, 60)) time_left -= 60 for mc in mcs: retries = 0 stats = {} while not stats and retries < RETRIES: try: stats = mc.stats() try: mem_stats = mc.stats('raw memory') except MemcachedError: mem_stats = mc.stats('memory') stats.update(mem_stats) except Exception as e: log.error("{0}, retries = {1}".format(str(e), retries)) time.sleep(2) mc.reconnect() retries += 1 continue stats["time"] = time.time() stats["ip"] = mc.host d[mc.host]["snapshots"].append(stats) try: timings = mc.stats('timings') d[mc.host]["timings"].append(timings) dispatcher = mc.stats('dispatcher') d[mc.host]["dispatcher"].append(dispatcher) except EOFError, e: log.error("Unable to get timings/dispatcher stats {0}: {1}"\ .format(mc.host, e))
def membase_stats(self, nodes, bucket, frequency, verbose=False): mcs = [] for node in nodes: try: bucket = RestConnection(node).get_buckets()[0].name mcs.append(MemcachedClientHelper.direct_client(node, bucket)) except: pass self._task["membasestats"] = [] self._task["timings"] = [] self._task["dispatcher"] = [] d = {} # "pname": "x", "pid": "y","snapshots": [{"time": time,"value": value}] for mc in mcs: d[mc.host] = {"snapshots": [], "timings": [], "dispatcher": []} while not self._aborted(): time_left = frequency log.info("Collecting membase stats") timings = None # at minimum we want to check for aborted every minute while not self._aborted() and time_left > 0: time.sleep(min(time_left, 60)) time_left -= 60 for mc in mcs: retries = 0 stats = {} while not stats and retries < RETRIES: try: stats = mc.stats() try: mem_stats = mc.stats('raw memory') except MemcachedError: mem_stats = mc.stats('memory') stats.update(mem_stats) except Exception as e: log.error("{0}, retries = {1}".format(str(e), retries)) time.sleep(2) mc.reconnect() retries += 1 continue stats["time"] = time.time() stats["ip"] = mc.host d[mc.host]["snapshots"].append(stats) try: timings = mc.stats('timings') d[mc.host]["timings"].append(timings) dispatcher = mc.stats('dispatcher') d[mc.host]["dispatcher"].append(dispatcher) except EOFError, e: log.error("Unable to get timings/dispatcher stats {0}: {1}"\ .format(mc.host, e))
def capture_mb_snapshot(self, node): """Capture membase stats snapshot manually""" log.info("capturing memcache stats snapshot for {0}".format(node.ip)) stats = {} try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) stats = mc.stats() stats.update(mc.stats("warmup")) except Exception, e: log.error(e) return False
def capture_mb_snapshot(self, node): """Capture membase stats snapshot manually""" log.info("capturing memcache stats snapshot for {0}".format(node.ip)) stats = {} try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) stats = mc.stats() stats.update(mc.stats("warmup")) except Exception, e: log.error(e) return False
def capture_mb_snapshot(self, node): """Capture membase stats snapshot manually""" log.info("capturing memcache stats snapshot for {0}".format(node.ip)) stats = {} try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) stats = mc.stats() stats.update(mc.stats("warmup")) except Exception as e: log.error(e) return False finally: stats["time"] = time.time() stats["ip"] = node.ip self._mb_stats["snapshots"].append(stats) print(stats) log.info("memcache stats snapshot captured") return True
def capture_mb_snapshot(self, node): """Capture membase stats snapshot manually""" log.info("Capturing memcache stats snapshot for {0}".format(node.ip)) stats = {} try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) stats = mc.stats() stats.update(mc.stats("warmup")) except Exception as e: log.error("Exception: {0}".format(str(e))) return False finally: stats["time"] = time.time() stats["ip"] = node.ip self._mb_stats["snapshots"].append(stats) print stats log.info("Memcache stats snapshot captured") return True
def test_partial_rollback(self): self.multi_create_index() self.sleep(30) self.log.info("Stopping persistence on NodeA & NodeB") data_nodes = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) for data_node in data_nodes: for bucket in self.buckets: mem_client = MemcachedClientHelper.direct_client( data_node, bucket.name) mem_client.stop_persistence() self.run_doc_ops() self.sleep(10) # Get count before rollback bucket_before_item_counts = {} for bucket in self.buckets: bucket_count_before_rollback = self.get_item_count( self.master, bucket.name) bucket_before_item_counts[ bucket.name] = bucket_count_before_rollback log.info("Items in bucket {0} before rollback = {1}".format( bucket.name, bucket_count_before_rollback)) # Index rollback count before rollback self._verify_bucket_count_with_index_count() self.multi_query_using_index() # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(data_nodes[0]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") for bucket in self.buckets: mem_client = MemcachedClientHelper.direct_client( data_nodes[1], bucket.name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self.cluster.async_failover( self.servers[:self.nodes_init], [data_nodes[1]], self.graceful, wait_for_pending=120) failover_task.result() # Wait for a couple of mins to allow rollback to complete # self.sleep(120) bucket_after_item_counts = {} for bucket in self.buckets: bucket_count_after_rollback = self.get_item_count( self.master, bucket.name) bucket_after_item_counts[bucket.name] = bucket_count_after_rollback log.info("Items in bucket {0} after rollback = {1}".format( bucket.name, bucket_count_after_rollback)) for bucket in self.buckets: if bucket_after_item_counts[ bucket.name] == bucket_before_item_counts[bucket.name]: log.info("Looks like KV rollback did not happen at all.") self._verify_bucket_count_with_index_count() self.multi_query_using_index()
def partial_rollback(self): bucket = self._cb_cluster.get_bucket_by_name("default") self._cb_cluster.flush_buckets([bucket]) index = self.create_index(bucket, "default_index") self.load_data() self.wait_for_indexing_complete() # Stop Persistence on Node A & Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[0], bucket) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.stop_persistence() # Perform mutations on the bucket self.async_perform_update_delete(self.upd_del_fields) if self._update: self.sleep(60, "Waiting for updates to get indexed...") self.wait_for_indexing_complete() # Run FTS Query to fetch the initial count of mutated items query = "{\"query\": \"mutated:>0\"}" query = json.loads(query) for index in self._cb_cluster.get_indexes(): hits1, _, _, _ = index.execute_query(query) self.log.info("Hits before rollback: %s" % hits1) # Fetch count of docs in index and bucket before_index_doc_count = index.get_indexed_doc_count() before_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % ( before_bucket_doc_count, before_index_doc_count)) # Kill memcached on Node A so that Node B becomes master shell = RemoteMachineShellConnection(self._master) shell.kill_memcached() # Start persistence on Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.start_persistence() # Failover Node B failover_task = self._cb_cluster.async_failover( node=self._input.servers[1]) failover_task.result() # Wait for Failover & FTS index rollback to complete self.sleep(10) # Run FTS query to fetch count of mutated items post rollback. for index in self._cb_cluster.get_indexes(): hits2, _, _, _ = index.execute_query(query) self.log.info("Hits after rollback: %s" % hits2) # Fetch count of docs in index and bucket after_index_doc_count = index.get_indexed_doc_count() after_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % (after_bucket_doc_count, after_index_doc_count)) # Validation : If there are deletes, validate the #docs in index goes up post rollback if self._input.param("delete", False): self.assertGreater(after_index_doc_count, before_index_doc_count, "Deletes : Index count after rollback not greater than before rollback") else: # For Updates, validate that #hits goes down in the query output post rollback self.assertGreater(hits1, hits2, "Mutated items before rollback are not more than after rollback") # Failover FTS node failover_fts_node = self._input.param("failover_fts_node", False) if failover_fts_node: failover_task = self._cb_cluster.async_failover( node=self._input.servers[2]) failover_task.result() self.sleep(10) # Run FTS query to fetch count of mutated items post FTS node failover. for index in self._cb_cluster.get_indexes(): hits3, _, _, _ = index.execute_query(query) self.log.info( "Hits after rollback and failover of primary FTS node: %s" % hits3) self.assertEqual(hits2, hits3, "Mutated items after FTS node failover are not equal to that after rollback")
def test_rollback(self): bucket = self.src_cluster.get_buckets()[0] nodes = self.src_cluster.get_nodes() # Stop Persistence on Node A & Node B for node in nodes: mem_client = MemcachedClientHelper.direct_client(node, bucket) mem_client.stop_persistence() goxdcr_log = NodeHelper.get_goxdcr_log_dir(self._input.servers[0])\ + '/goxdcr.log*' self.setup_xdcr() self.src_cluster.pause_all_replications() gen = BlobGenerator("C1-", "C1-", self._value_size, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.src_cluster.resume_all_replications() # Perform mutations on the bucket self.async_perform_update_delete() rest1 = RestConnection(self.src_cluster.get_master_node()) rest2 = RestConnection(self.dest_cluster.get_master_node()) # Fetch count of docs in src and dest cluster _count1 = rest1.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] _count2 = rest2.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] self.log.info("Before rollback src cluster count = {0} dest cluster count = {1}".format(_count1, _count2)) # Kill memcached on Node A so that Node B becomes master shell = RemoteMachineShellConnection(self.src_cluster.get_master_node()) shell.kill_memcached() # Start persistence on Node B mem_client = MemcachedClientHelper.direct_client(nodes[1], bucket) mem_client.start_persistence() # Failover Node B failover_task = self.src_cluster.async_failover() failover_task.result() # Wait for Failover & rollback to complete self.sleep(60) # Fetch count of docs in src and dest cluster _count1 = rest1.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] _count2 = rest2.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] self.log.info("After rollback src cluster count = {0} dest cluster count = {1}".format(_count1, _count2)) self.assertTrue(self.src_cluster.wait_for_outbound_mutations(), "Mutations in source cluster not replicated to target after rollback") self.log.info("Mutations in source cluster replicated to target after rollback") count = NodeHelper.check_goxdcr_log( nodes[0], "Received rollback from DCP stream", goxdcr_log) self.assertGreater(count, 0, "rollback did not happen as expected") self.log.info("rollback happened as expected")
def test_rebalance_kv_rollback_create_ops(self): self.setup_for_test() items_before_persistence_stop = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name)[0] self.log.info("Items in CBAS before persistence stop: %s" % items_before_persistence_stop) # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA") mem_client = MemcachedClientHelper.direct_client( self.master, self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items / 2, "create", self.num_items, self.num_items * 3 / 2) kv_nodes = self.get_kv_nodes(self.servers, self.master) items_in_cb_bucket = 0 if self.where_field and self.where_value: items_in_cb_bucket = RestConnection(self.master).query_tool( 'select count(*) from %s where %s = "%s"' % (self.cb_bucket_name, self.where_field, self.where_value))['results'][0]['$1'] else: for node in kv_nodes: items_in_cb_bucket += self.get_item_count( node, self.cb_bucket_name) # Validate no. of items in CBAS dataset self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, items_in_cb_bucket, 0), "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket" ) if self.CC: self.cluster_util.remove_node([self.otpNodes[0]], wait_for_rebalance=False) self.cbas_util.closeConn() self.cbas_util = cbas_utils(self.master, self.cbas_servers[0]) self.cbas_util.createConn("default") else: self.cluster_util.remove_node([self.otpNodes[1]], wait_for_rebalance=False) # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() self.sleep(2, "Wait for 2 secs for DCP rollback sent to CBAS.") curr = time.time() while items_in_cbas_bucket == -1 or ( items_in_cbas_bucket != 0 and items_in_cbas_bucket > items_before_persistence_stop): try: if curr + 120 < time.time(): break items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info("Items in CBAS: %s" % items_in_cbas_bucket) except: self.log.info( "Probably rebalance is in progress and the reason for queries being failing." ) pass self.assertTrue(items_in_cbas_bucket <= items_before_persistence_stop, "Roll-back did not happen.") self.log.info("#######BINGO########\nROLLBACK HAPPENED") items_in_cb_bucket = 0 curr = time.time() while items_in_cb_bucket != items_in_cbas_bucket or items_in_cb_bucket == 0: items_in_cb_bucket = 0 items_in_cbas_bucket = 0 if self.where_field and self.where_value: try: items_in_cb_bucket = RestConnection( self.master).query_tool( 'select count(*) from %s where %s = "%s"' % (self.cb_bucket_name, self.where_field, self.where_value))['results'][0]['$1'] except: self.log.info( "Indexer in rollback state. Query failed. Pass and move ahead." ) pass else: for node in kv_nodes: items_in_cb_bucket += self.get_item_count( node, self.cb_bucket_name) self.log.info("Items in CB bucket after rollback: %s" % items_in_cb_bucket) try: items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) except: pass if curr + 120 < time.time(): break str_time = time.time() while self.rest._rebalance_progress_status( ) == "running" and time.time() < str_time + 300: self.sleep(1) self.log.info("Waiting for rebalance to complete") self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "After Rollback : # Items in CBAS bucket does not match that in the CB bucket" )
def test_ingestion_after_kv_rollback_cbas_disconnected(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA") mem_client = MemcachedClientHelper.direct_client( self.master, self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items / 2, "delete", 0, self.num_items / 2) # Count no. of items in CB & CBAS Buckets kv_nodes = self.get_kv_nodes(self.servers, self.master) items_in_cb_bucket = 0 for node in kv_nodes: items_in_cb_bucket += self.get_item_count(node, self.cb_bucket_name) items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) items_before_rollback = items_in_cbas_bucket self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket" ) self.cbas_util.disconnect_from_bucket(self.cbas_bucket_name) # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() # self.sleep(10,"Wait for 10 secs for memcached restarts.") if self.input.param('kill_cbas', False): shell = RemoteMachineShellConnection(self.cbas_node) shell.kill_process("/opt/couchbase/lib/cbas/runtime/bin/java", "java") shell.kill_process("/opt/couchbase/bin/cbas", "cbas") tries = 60 result = False while tries > 0 and not result: try: result = self.cbas_util.connect_to_bucket( self.cbas_bucket_name) tries -= 1 except: pass self.sleep(2) self.assertTrue( result, "CBAS connect bucket failed after memcached killed on KV node.") curr = time.time() while items_in_cbas_bucket != 0 and items_in_cbas_bucket <= items_before_rollback: items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) if curr + 120 < time.time(): break self.assertTrue(items_in_cbas_bucket > items_before_rollback, "Roll-back did not happen.") self.log.info("#######BINGO########\nROLLBACK HAPPENED") curr = time.time() while items_in_cb_bucket != items_in_cbas_bucket: items_in_cb_bucket = 0 items_in_cbas_bucket = 0 if self.where_field and self.where_value: try: items_in_cb_bucket = RestConnection( self.master).query_tool( 'select count(*) from %s where %s = "%s"' % (self.cb_bucket_name, self.where_field, self.where_value))['results'][0]['$1'] except: self.log.info( "Indexer in rollback state. Query failed. Pass and move ahead." ) pass else: for node in kv_nodes: items_in_cb_bucket += self.get_item_count( node, self.cb_bucket_name) self.log.info("Items in CB bucket after rollback: %s" % items_in_cb_bucket) items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) if curr + 120 < time.time(): break # Count no. of items in CB & CBAS Buckets items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "After Rollback : # Items in CBAS bucket does not match that in the CB bucket" )
def test_ingestion_after_kv_rollback(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA & NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[0], self.cb_bucket_name) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "delete", 0, self.num_items / 2) # Validate no. of items in CBAS dataset if not self.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items / 2, 0): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket") # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self._cb_cluster.async_failover(self.input.servers, [self.input.servers[1]]) failover_task.result() # Wait for Failover & CBAS rollback to complete self.sleep(120) # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "After Rollback : # Items in CBAS bucket does not match that in the CB bucket")
def membase_stats(self, interval=60): mcs = [] for node in self.nodes: try: bucket = RestConnection(node).get_buckets()[0].name mc = MemcachedClientHelper.direct_client(node, bucket) mcs.append(mc) except Exception as error: log.error(error) self._task["membasestats"] = [] self._task["timings"] = [] self._task["dispatcher"] = [] data = dict() for mc in mcs: data[mc.host] = {"snapshots": [], "timings": [], "dispatcher": []} while not self._aborted(): time.sleep(interval) log.info("collecting membase stats") for mc in mcs: for rerty in range(RETRIES): try: stats = mc.stats() except Exception as e: log.warn("{0}, retries = {1}".format(str(e), rerty)) time.sleep(2) mc.reconnect() else: break else: stats = {} data[mc.host]["snapshots"].append(stats) for arg in ("timings", "dispatcher"): try: stats = mc.stats(arg) data[mc.host][arg].append(stats) except EOFError as e: log.error("unable to get {0} stats {1}: {2}".format( arg, mc.host, e)) for host in (mc.host for mc in mcs): unique_id = host + '-' + str(self._task["time"]) current_time = time.time() if self._mb_stats["snapshots"]: # use manually captured stats self._task["membasestats"] = self._mb_stats["snapshots"] else: # use periodically captured stats for snapshot in data[host]["snapshots"]: snapshot["unique_id"] = unique_id snapshot["time"] = current_time snapshot["ip"] = host self._task["membasestats"].append(snapshot) for timing in data[host]["timings"]: timing["unique_id"] = unique_id timing["time"] = current_time timing["ip"] = host self._task["timings"].append(timing) for dispatcher in data[host]["dispatcher"]: dispatcher["unique_id"] = unique_id dispatcher["time"] = current_time dispatcher["ip"] = host self._task["dispatcher"].append(dispatcher) if data[host]["timings"]: log.info("dumping disk timing stats: {0}".format(host)) latests_timings = data[host]["timings"][-1] for key, value in sorted(latests_timings.items()): if key.startswith("disk"): print("{0:50s}: {1}".format(key, value)) log.info("finished membase_stats")
def partial_rollback(self): bucket = self._cb_cluster.get_bucket_by_name("default") self._cb_cluster.flush_buckets([bucket]) index = self.create_index(bucket, "default_index") self.load_data() self.wait_for_indexing_complete() # Stop Persistence on Node A & Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[0], bucket) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.stop_persistence() # Perform mutations on the bucket self.async_perform_update_delete(self.upd_del_fields) if self._update: self.sleep(60, "Waiting for updates to get indexed...") self.wait_for_indexing_complete() # Run FTS Query to fetch the initial count of mutated items query = "{\"query\": \"mutated:>0\"}" query = json.loads(query) for index in self._cb_cluster.get_indexes(): hits1, _, _, _ = index.execute_query(query) self.log.info("Hits before rollback: %s" % hits1) # Fetch count of docs in index and bucket before_index_doc_count = index.get_indexed_doc_count() before_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % ( before_bucket_doc_count, before_index_doc_count)) # Kill memcached on Node A so that Node B becomes master shell = RemoteMachineShellConnection(self._master) shell.kill_memcached() # Start persistence on Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.start_persistence() # Failover Node B failover_task = self._cb_cluster.async_failover( node=self._input.servers[1]) failover_task.result() # Wait for Failover & FTS index rollback to complete self.sleep(10) # Run FTS query to fetch count of mutated items post rollback. for index in self._cb_cluster.get_indexes(): hits2, _, _, _ = index.execute_query(query) self.log.info("Hits after rollback: %s" % hits2) # Fetch count of docs in index and bucket after_index_doc_count = index.get_indexed_doc_count() after_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % (after_bucket_doc_count, after_index_doc_count)) # Validation : If there are deletes, validate the #docs in index goes up post rollback if self._input.param("delete", False): self.assertGreater(after_index_doc_count, before_index_doc_count, "Deletes : Index count after rollback not greater than before rollback") else: # For Updates, validate that #hits goes down in the query output post rollback self.assertGreater(hits1, hits2, "Mutated items before rollback are not more than after rollback") # Failover FTS node failover_fts_node = self._input.param("failover_fts_node", False) if failover_fts_node: failover_task = self._cb_cluster.async_failover( node=self._input.servers[2]) failover_task.result() self.sleep(10) # Run FTS query to fetch count of mutated items post FTS node failover. for index in self._cb_cluster.get_indexes(): hits3, _, _, _ = index.execute_query(query) self.log.info( "Hits after rollback and failover of primary FTS node: %s" % hits3) self.assertEqual(hits2, hits3, "Mutated items after FTS node failover are not equal to that after rollback")
def test_ingestion_after_kv_rollback(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA & NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[0], self.cb_bucket_name) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "delete", 0, self.num_items / 2) # Validate no. of items in CBAS dataset if not self.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items / 2, 0): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket") # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self._cb_cluster.async_failover(self.input.servers, [self.input.servers[1]]) failover_task.result() # Wait for Failover & CBAS rollback to complete self.sleep(60) # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "After Rollback : # Items in CBAS bucket does not match that in the CB bucket")
def test_rollback(self): bucket = self.src_cluster.get_buckets()[0] nodes = self.src_cluster.get_nodes() # Stop Persistence on Node A & Node B for node in nodes: mem_client = MemcachedClientHelper.direct_client(node, bucket) mem_client.stop_persistence() goxdcr_log = NodeHelper.get_goxdcr_log_dir(self._input.servers[0])\ + '/goxdcr.log*' self.setup_xdcr() self.src_cluster.pause_all_replications() gen = BlobGenerator("C1-", "C1-", self._value_size, end=self._num_items) self.src_cluster.load_all_buckets_from_generator(gen) self.src_cluster.resume_all_replications() # Perform mutations on the bucket self.async_perform_update_delete() rest1 = RestConnection(self.src_cluster.get_master_node()) rest2 = RestConnection(self.dest_cluster.get_master_node()) # Fetch count of docs in src and dest cluster _count1 = rest1.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] _count2 = rest2.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] self.log.info("Before rollback src cluster count = {0} dest cluster count = {1}".format(_count1, _count2)) # Kill memcached on Node A so that Node B becomes master shell = RemoteMachineShellConnection(self.src_cluster.get_master_node()) shell.kill_memcached() # Start persistence on Node B mem_client = MemcachedClientHelper.direct_client(nodes[1], bucket) mem_client.start_persistence() # Failover Node B failover_task = self.src_cluster.async_failover() failover_task.result() # Wait for Failover & rollback to complete self.sleep(60) # Fetch count of docs in src and dest cluster _count1 = rest1.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] _count2 = rest2.fetch_bucket_stats(bucket=bucket.name)["op"]["samples"]["curr_items"][-1] self.log.info("After rollback src cluster count = {0} dest cluster count = {1}".format(_count1, _count2)) self.assertTrue(self.src_cluster.wait_for_outbound_mutations(), "Mutations in source cluster not replicated to target after rollback") self.log.info("Mutations in source cluster replicated to target after rollback") _, count = NodeHelper.check_goxdcr_log( nodes[0], "Received rollback from DCP stream", goxdcr_log, timeout=60) self.assertGreater(count, 0, "rollback did not happen as expected") self.log.info("rollback happened as expected")
def test_ingestion_after_kv_rollback_delete_ops(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA") mem_client = MemcachedClientHelper.direct_client( self.master, self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items / 2, "delete", 0, self.num_items / 2) kv_nodes = self.get_kv_nodes(self.servers, self.master) items_in_cb_bucket = 0 if self.where_field and self.where_value: items_in_cb_bucket = RestConnection(self.master).query_tool( 'select count(*) from %s where %s = "%s"' % (self.cb_bucket_name, self.where_field, self.where_value))['results'][0]['$1'] else: for node in kv_nodes: items_in_cb_bucket += self.get_item_count( node, self.cb_bucket_name) # Validate no. of items in CBAS dataset self.assertTrue( self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, items_in_cb_bucket, 0), "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) items_before_rollback = items_in_cbas_bucket self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket" ) # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() self.sleep(2, "Wait for 2 secs for DCP rollback sent to CBAS.") curr = time.time() while items_in_cbas_bucket != 0 and items_in_cbas_bucket <= items_before_rollback: items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) if curr + 120 < time.time(): break self.assertTrue(items_in_cbas_bucket > items_before_rollback, "Roll-back did not happen.") self.log.info("#######BINGO########\nROLLBACK HAPPENED") items_in_cb_bucket = 0 curr = time.time() while items_in_cb_bucket != items_in_cbas_bucket: items_in_cb_bucket = 0 items_in_cbas_bucket = 0 if self.where_field and self.where_value: try: items_in_cb_bucket = RestConnection( self.master).query_tool( 'select count(*) from %s where %s = "%s"' % (self.cb_bucket_name, self.where_field, self.where_value))['results'][0]['$1'] except: self.log.info( "Indexer in rollback state. Query failed. Pass and move ahead." ) pass else: for node in kv_nodes: items_in_cb_bucket += self.get_item_count( node, self.cb_bucket_name) self.log.info("Items in CB bucket after rollback: %s" % items_in_cb_bucket) items_in_cbas_bucket, _ = self.cbas_util.get_num_items_in_cbas_dataset( self.cbas_dataset_name) if curr + 120 < time.time(): break self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) self.assertTrue( items_in_cb_bucket == items_in_cbas_bucket, "After Rollback : # Items in CBAS bucket does not match that in the CB bucket" )