def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create', compression=self.sdk_compression) except Exception as e: log.info("error while loading data") self.deploy_function(body,wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() # Wait for bootstrap to complete self.wait_for_bootstrap_to_complete(body['appname']) stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) if self.is_curl: body['depcfg']['curl'] = [] body['depcfg']['curl'].append({"hostname": self.hostname, "value": "server", "auth_type": self.auth_type, "username": self.curl_username, "password": self.curl_password,"cookies": self.cookies}) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create', compression=self.sdk_compression) except Exception as e: log.info("error while loading data") self.deploy_function(body, wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() self.wait_for_handler_state(body['appname'], "deployed") stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def test_partial_rollback(self): kv_node = self.get_nodes_from_services_map(service_type="kv", get_all_nodes=True) log.info("kv nodes:{0}".format(kv_node)) for node in kv_node: mem_client = MemcachedClientHelper.direct_client(node, self.src_bucket_name) mem_client.stop_persistence() body = self.create_save_function_body(self.function_name, self.handler_code, worker_count=3) try: task = self.cluster.async_load_gen_docs(self.master, self.src_bucket_name, self.gens_load, self.buckets[0].kvs[1], 'create') except Exception as e: log.info("error while loading data") self.deploy_function(body,wait_for_bootstrap=False) # Kill memcached on Node A self.log.info("Killing memcached on {0}".format(kv_node[1])) shell = RemoteMachineShellConnection(kv_node[1]) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on {0}". format(kv_node[0])) mem_client = MemcachedClientHelper.direct_client(kv_node[0], self.src_bucket_name) mem_client.start_persistence() # Wait for bootstrap to complete self.wait_for_bootstrap_to_complete(body['appname']) stats_src = RestConnection(self.master).get_bucket_stats(bucket=self.src_bucket_name) log.info(stats_src) self.verify_eventing_results(self.function_name, stats_src["curr_items"], skip_stats_validation=True)
def test_ingestion_after_kv_rollback(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA & NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[0], self.cb_bucket_name) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "delete", 0, self.num_items / 2) # Validate no. of items in CBAS dataset if not self.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items / 2, 0): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket") # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self._cb_cluster.async_failover(self.input.servers, [self.input.servers[1]]) failover_task.result() # Wait for Failover & CBAS rollback to complete self.sleep(60) # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "After Rollback : # Items in CBAS bucket does not match that in the CB bucket")
def kill_memcached_service(self, server): remote_client = RemoteMachineShellConnection(server) remote_client.kill_memcached() remote_client.disconnect()
def partial_rollback(self): bucket = self._cb_cluster.get_bucket_by_name("default") self._cb_cluster.flush_buckets([bucket]) index = self.create_index(bucket, "default_index") self.load_data() self.wait_for_indexing_complete() # Stop Persistence on Node A & Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[0], bucket) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.stop_persistence() # Perform mutations on the bucket self.async_perform_update_delete(self.upd_del_fields) if self._update: self.sleep(60, "Waiting for updates to get indexed...") self.wait_for_indexing_complete() # Run FTS Query to fetch the initial count of mutated items query = "{\"query\": \"mutated:>0\"}" query = json.loads(query) for index in self._cb_cluster.get_indexes(): hits1, _, _, _ = index.execute_query(query) self.log.info("Hits before rollback: %s" % hits1) # Fetch count of docs in index and bucket before_index_doc_count = index.get_indexed_doc_count() before_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % ( before_bucket_doc_count, before_index_doc_count)) # Kill memcached on Node A so that Node B becomes master shell = RemoteMachineShellConnection(self._master) shell.kill_memcached() # Start persistence on Node B mem_client = MemcachedClientHelper.direct_client(self._input.servers[1], bucket) mem_client.start_persistence() # Failover Node B failover_task = self._cb_cluster.async_failover( node=self._input.servers[1]) failover_task.result() # Wait for Failover & FTS index rollback to complete self.sleep(10) # Run FTS query to fetch count of mutated items post rollback. for index in self._cb_cluster.get_indexes(): hits2, _, _, _ = index.execute_query(query) self.log.info("Hits after rollback: %s" % hits2) # Fetch count of docs in index and bucket after_index_doc_count = index.get_indexed_doc_count() after_bucket_doc_count = index.get_src_bucket_doc_count() self.log.info("Docs in Bucket : %s, Docs in Index : %s" % (after_bucket_doc_count, after_index_doc_count)) # Validation : If there are deletes, validate the #docs in index goes up post rollback if self._input.param("delete", False): self.assertGreater(after_index_doc_count, before_index_doc_count, "Deletes : Index count after rollback not greater than before rollback") else: # For Updates, validate that #hits goes down in the query output post rollback self.assertGreater(hits1, hits2, "Mutated items before rollback are not more than after rollback") # Failover FTS node failover_fts_node = self._input.param("failover_fts_node", False) if failover_fts_node: failover_task = self._cb_cluster.async_failover( node=self._input.servers[2]) failover_task.result() self.sleep(10) # Run FTS query to fetch count of mutated items post FTS node failover. for index in self._cb_cluster.get_indexes(): hits3, _, _, _ = index.execute_query(query) self.log.info( "Hits after rollback and failover of primary FTS node: %s" % hits3) self.assertEqual(hits2, hits3, "Mutated items after FTS node failover are not equal to that after rollback")
def test_ingestion_after_kv_rollback(self): self.setup_for_test() # Stop Persistence on Node A & Node B self.log.info("Stopping persistence on NodeA & NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[0], self.cb_bucket_name) mem_client.stop_persistence() mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.stop_persistence() # Perform Create, Update, Delete ops in the CB bucket self.log.info("Performing Mutations") self.perform_doc_ops_in_all_cb_buckets(self.num_items, "delete", 0, self.num_items / 2) # Validate no. of items in CBAS dataset if not self.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items / 2, 0): self.fail( "No. of items in CBAS dataset do not match that in the CB bucket") # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "Before Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "Before Rollback : # Items in CBAS bucket does not match that in the CB bucket") # Kill memcached on Node A so that Node B becomes master self.log.info("Kill Memcached process on NodeA") shell = RemoteMachineShellConnection(self.master) shell.kill_memcached() # Start persistence on Node B self.log.info("Starting persistence on NodeB") mem_client = MemcachedClientHelper.direct_client(self.input.servers[1], self.cb_bucket_name) mem_client.start_persistence() # Failover Node B self.log.info("Failing over NodeB") self.sleep(10) failover_task = self._cb_cluster.async_failover(self.input.servers, [self.input.servers[1]]) failover_task.result() # Wait for Failover & CBAS rollback to complete self.sleep(120) # Count no. of items in CB & CBAS Buckets items_in_cb_bucket = self.get_item_count(self.master, self.cb_bucket_name) items_in_cbas_bucket, _ = self.get_num_items_in_cbas_dataset( self.cbas_dataset_name) self.log.info( "After Rollback --- # docs in CB bucket : %s, # docs in CBAS bucket : %s", items_in_cb_bucket, items_in_cbas_bucket) if items_in_cb_bucket != items_in_cbas_bucket: self.fail( "After Rollback : # Items in CBAS bucket does not match that in the CB bucket")