def test_doc_key_size(self): """ Insert document key with min and max key size on each available collection and validate :return: """ min_doc_gen = doc_generator("test_min_key_size", 0, self.num_items, key_size=1, doc_size=self.doc_size, mix_key_size=False, randomize_doc_size=False) max_doc_gen = doc_generator("test_max_key_size", 0, self.num_items, key_size=245, doc_size=self.doc_size, mix_key_size=False, randomize_doc_size=False) # Set to keep track of all inserted CAS values known_cas = set() # Client to insert docs under different collections client = SDKClient([self.cluster.master], self.bucket, compression_settings=self.sdk_compression) for doc_gen in [min_doc_gen, max_doc_gen]: while doc_gen.has_next(): key, value = doc_gen.next() for _, scope in self.bucket.scopes.items(): for _, collection in scope.collections.items(): client.select_collection(scope.name, collection.name) result = client.crud("create", key, value, self.maxttl, durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds") if result["status"] is False: self.log_failure( "Doc create failed for key '%s' " "collection::scope %s::%s - %s" % (key, scope.name, collection.name, result)) else: if result["cas"] in known_cas: self.log_failure( "Same CAS exists under different " "collection: %s" % result) collection.num_items += 1 known_cas.add(result["cas"]) # Close SDK connection client.close() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def perform_crud_ops(): old_cas = 0 client = SDKClient([self.cluster.master], bucket_obj) for op_type in ["create", "update", "read", "replace", "delete"]: crud_desc = "Key %s, doc_op: %s" % (key, op_type) self.log.info(crud_desc) result = client.crud(op_type, key, value, replicate_to=self.replicate_to, persist_to=self.persist_to) if op_type != "read": if op_type != "replace": dict_key = "ops_%s" % op_type else: dict_key = "ops_update" verification_dict[dict_key] += 1 verification_dict["sync_write_committed_count"] += 1 if result["cas"] == old_cas: self.log_failure("CAS didn't get updated: %s" % result["cas"]) elif op_type == "read": if result["cas"] != old_cas: self.log_failure("CAS updated for read operation: %s" % result["cas"]) self.summary.add_step(crud_desc) old_cas = result["cas"] client.close()
def touch_test(self): self.log.info("1. Loading initial set of documents") load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) self._load_all_buckets(load_gen, "create") self.bucket_util.verify_stats_all_buckets(self.num_items) self.bucket_util._wait_for_stats_all_buckets() self.log.info("2. Loading bucket into DGM") dgm_gen = doc_generator( self.key, self.num_items, self.num_items+1) dgm_task = self.task.async_load_gen_docs( self.cluster, self.bucket_util.buckets[0], dgm_gen, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=4, active_resident_threshold=self.active_resident_threshold) self.task_manager.get_task_result(dgm_task) self.log.info("3. Touch intial self.num_items docs which are " "residing on disk due to DGM") client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) while load_gen.has_next(): key, _ = load_gen.next() result = client.crud("touch", key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is not True: self.log_failure("Touch on %s failed: %s" % (key, result)) client.close() self.validate_test_failure()
def test_ops_on_same_key(self): """ 1. Set key 2. Delete a key 3. Set the same key 4. Validate the rev_id for the key is maintained Ref: MB-48179 """ if self.durability_level in ["", Bucket.DurabilityLevel.NONE]: self.fail("Test supported only for sync_write scenarios") crud_pattern = self.input.param("crud_pattern", "async:sync:async") crud_pattern = crud_pattern.split(":") rev_ids = dict() client = SDKClient([self.cluster.master], self.cluster.buckets[0]) # Async create of keys for i in range(self.num_items): key = self.key + str(i) durability = "" if crud_pattern[0] == "sync": durability = self.durability_level client.crud(DocLoading.Bucket.DocOps.CREATE, key, {}, durability=durability) # Sync delete of keys for i in range(self.num_items): key = self.key + str(i) durability = "" if crud_pattern[1] == "sync": durability = self.durability_level client.crud(DocLoading.Bucket.DocOps.DELETE, key, durability=durability) # Async create of keys for i in range(self.num_items): key = self.key + str(i) durability = "" if crud_pattern[2] == "sync": durability = self.durability_level client.crud(DocLoading.Bucket.DocOps.CREATE, key, {}, durability=durability) result = client.collection.lookupIn( key, Collections.singletonList( LookupInSpec.get(LookupInMacro.REV_ID).xattr())) rev_ids[key] = int(result.contentAs(0, String)) client.close() # Rev_id validation for i in range(self.num_items): key = self.key + str(i) if rev_ids[key] != 3: self.fail("Rev id mismatch for key '%s'. RevId: %s" % (key, rev_ids[key]))
def test_transactions(self): tasks = list() self.check_fragmentation_using_magma_stats(self.cluster.buckets[0]) bucket = self.cluster.buckets[0] workers = self.process_concurrency self.tm = TaskManager(workers) transaction_app = Transaction() trans_config = transaction_app.createTransactionConfig( self.transaction_timeout, self.sdk_timeout, self.transaction_durability_level) transaction_items = self.input.param("transaction_items", 1000) workload = dict() workload["keyPrefix"] = "transactions" workload["keySize"] = self.key_size workload["docSize"] = self.doc_size workload["mutated"] = 0 workload["keyRange"] = Tuples.of(0, transaction_items) workload["batchSize"] = 1 workload["workers"] = 3 workload["transaction_pattern"] = [[ CbServer.default_scope, CbServer.default_collection, self.transaction_pattern ]] work_load = WorkLoadSettings(workload["keyPrefix"], workload["keySize"], workload["docSize"], workload["mutated"], workload["keyRange"], workload["batchSize"], workload["transaction_pattern"], workload["workers"]) work_load.setTransactionRollback(self.rollback) client = SDKClient([self.cluster.master], bucket) transaction_obj = transaction_app.createTransaction( client.cluster, trans_config) for index, load_pattern in enumerate(work_load.transaction_pattern): th_name = "Transaction_%s" % index batch_size = load_pattern[0] num_transactions = load_pattern[1] trans_pattern = load_pattern[2] task = TransactionWorkLoadGenerate( th_name, client.cluster, client.bucketObj, transaction_obj, work_load.doc_gen, batch_size, num_transactions, trans_pattern, work_load.commit_transaction, work_load.rollback_transaction, transaction_app) tasks.append(task) self.tm.submit(task) self.tm.getAllTaskResult() client.close() result = \ self.check_fragmentation_using_magma_stats(self.cluster.buckets[0]) self.assertTrue(result, "Magma framentation error") # The final disk check requires randomized documents of size of 4096 # and no additional documents should be loaded. if self.final_disk_check: self.final_fragmentation_check(transaction_items)
def test_nru_eviction_impact_on_cbas(self): self.log.info("Create dataset") self.cbas_util.create_dataset_on_bucket(self.cb_bucket_name, self.cbas_dataset_name) self.log.info("Connect to Local link") self.cbas_util.connect_link() self.log.info("Add documents until ram percentage") self.load_document_until_ram_percentage() self.log.info("Fetch current document count") target_bucket = None self.bucket_util.get_all_buckets() for tem_bucket in self.bucket_util.buckets: if tem_bucket.name == self.cb_bucket_name: target_bucket = tem_bucket break item_count = target_bucket.stats.itemCount self.log.info("Completed base load with %s items" % item_count) self.log.info("Get initial inserted 100 docs, so they aren't removed") client = SDKClient([self.cluster.master], target_bucket) for doc_index in range(100): doc_key = "test_docs-" + str(doc_index) client.read(doc_key) self.log.info("Add 20% more items to trigger NRU") for doc_index in range(item_count, int(item_count * 1.2)): doc_key = "key_id-" + str(doc_index) op_result = client.crud("create", doc_key, '{"name":"dave"}', durability=self.durability_level) if op_result["status"] is False: self.log.warning("Insert failed for %s: %s" % (doc_key, op_result)) # Disconnect the SDK client client.close() self.log.info("Validate document count on CBAS") count_n1ql = self.rest.query_tool( 'select count(*) from %s' % self.cb_bucket_name)['results'][0]['$1'] if self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, count_n1ql): pass else: self.log.info("Document count mismatch might be due to ejection " "of documents on KV. Retry again") count_n1ql = self.rest.query_tool( 'select count(*) from %s' % self.cb_bucket_name)['results'][0]['$1'] self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count( self.cbas_dataset_name, count_n1ql), msg="Count mismatch on CBAS")
def test_no_eviction_impact_on_cbas(self): self.log.info("Add documents until ram percentage") self.load_document_until_ram_percentage() self.log.info("Fetch current document count") target_bucket = None buckets = self.bucket_util.get_all_buckets(self.cluster) for tem_bucket in buckets: if tem_bucket.name == self.bucket_name: target_bucket = tem_bucket break item_count = target_bucket.stats.itemCount self.log.info("Completed base load with %s items" % item_count) self.log.info("Load more until we are out of memory") client = SDKClient([self.cluster.master], target_bucket) i = item_count op_result = {"status": True} while op_result["status"] is True: op_result = client.crud("create", "key-id" + str(i), '{"name":"dave"}', durability=self.durability_level) i += 1 if SDKException.AmbiguousTimeoutException not in op_result["error"] \ or SDKException.RetryReason.KV_TEMPORARY_FAILURE \ not in op_result["error"]: client.close() self.fail("Invalid exception for OOM insert: %s" % op_result) self.log.info('Memory is full at {0} items'.format(i)) self.log.info("As a result added more %s items" % (i - item_count)) self.log.info("Fetch item count") target_bucket = None buckets = self.bucket_util.get_all_buckets(self.cluster) for tem_bucket in buckets: if tem_bucket.name == self.cb_bucket_name: target_bucket = tem_bucket break item_count_when_oom = target_bucket.stats.itemCount mem_when_oom = target_bucket.stats.memUsed self.log.info('Item count when OOM {0} and memory used {1}'.format( item_count_when_oom, mem_when_oom)) self.log.info("Validate document count on CBAS") count_n1ql = self.rest.query_tool('select count(*) from %s' % self.bucket_name)['results'][0]['$1'] dataset = self.cbas_util.list_all_dataset_objs()[0] if not self.cbas_util.validate_cbas_dataset_items_count( self.cluster, dataset.full_name, count_n1ql): self.fail("No. of items in CBAS dataset do not match " "that in the KV bucket")
def test_doc_size(self): """ Insert document with empty content and max size on each available collection and validate :return: """ # Empty docs min_doc_size_gen = doc_generator("test_min_doc_size", 0, self.num_items, key_size=self.key_size, doc_size=0, mix_key_size=False, randomize_doc_size=False) # 20 MB docs max_doc_size_gen = doc_generator("test_max_doc_size", 0, self.num_items, key_size=self.key_size, doc_size=1024 * 1024 * 20, mix_key_size=False, randomize_doc_size=False) # Set to keep track of all inserted CAS values # Format know_cas[CAS] = list(vb_lists) known_cas = dict() # Client to insert docs under different collections client = SDKClient([self.cluster.master], self.bucket, compression_settings=self.sdk_compression) for doc_gen in [min_doc_size_gen, max_doc_size_gen]: while doc_gen.has_next(): key, value = doc_gen.next() for _, scope in self.bucket.scopes.items(): for _, collection in scope.collections.items(): client.select_collection(scope.name, collection.name) result = client.crud("create", key, value, self.maxttl, durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds") if result["status"] is False: self.log_failure("Doc create failed for key '%s' " "collection::scope %s::%s - %s" % (key, scope.name, collection.name, result)) else: self.__validate_cas_for_key(key, result, known_cas) collection.num_items += 1 # Close SDK connection client.close() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def touch_test(self): self.log.info("Loading bucket into DGM") load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) dgm_gen = doc_generator(self.key, self.num_items, self.num_items + 1) dgm_task = self.task.async_load_gen_docs( self.cluster, self.bucket_util.buckets[0], dgm_gen, "create", 0, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=10, process_concurrency=4, active_resident_threshold=self.active_resident_threshold) self.task_manager.get_task_result(dgm_task) self.log.info("Touch intial self.num_items docs which are " "residing on disk due to DGM") client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) collections = BucketUtils.get_random_collections( self.bucket_util.buckets, 2, 2, 1) for self.bucket_name, scope_dict in collections.iteritems(): bucket = BucketUtils.get_bucket_obj(self.bucket_util.buckets, self.bucket_name) scope_dict = scope_dict["scopes"] for scope_name, collection_dict in scope_dict.items(): collection_dict = collection_dict["collections"] for c_name, c_data in collection_dict.items(): self.log.info("CAS test on collection %s: %s" % (scope_name, c_name)) client.select_collection(scope_name, c_name) while load_gen.has_next(): key, _ = load_gen.next() result = client.crud("touch", key, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is not True: self.log_failure("Touch on %s failed: %s" % (key, result)) client.close() self.bucket_util._wait_for_stats_all_buckets() # Validate doc count as per bucket collections self.bucket_util.validate_docs_per_collections_all_buckets() self.validate_test_failure()
def perform_create_deletes(): index = 0 client = SDKClient([self.cluster.master], self.cluster.buckets[0]) self.log.info("Starting ops to create tomb_stones") while not self.stop_thread: key = "temp_key--%s" % index result = client.crud("create", key, "") if result["status"] is False: self.log_failure("Key %s create failed: %s" % (key, result)) break result = client.crud("delete", key) if result["status"] is False: self.log_failure("Key %s delete failed: %s" % (key, result)) break index += 1 client.close() self.log.info("Total keys deleted: %s" % index)
def MB36948(self): node_to_stop = self.servers[0] self.log.info("Adding index/query node") self.task.rebalance([self.cluster.master], [self.servers[2]], [], services=["n1ql,index"]) self.log.info("Creating SDK client connection") client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], compression_settings=self.sdk_compression) self.log.info("Stopping memcached on: %s" % node_to_stop) ssh_conn = RemoteMachineShellConnection(node_to_stop) err_sim = CouchbaseError(self.log, ssh_conn) err_sim.create(CouchbaseError.STOP_MEMCACHED) result = client.crud("create", "abort1", "abort1_val") if not result["status"]: self.log_failure("Async SET failed") result = client.crud("update", "abort1", "abort1_val", durability=self.durability_level, timeout=3, time_unit="seconds") if result["status"]: self.log_failure("Sync write succeeded") if SDKException.DurabilityAmbiguousException not in result["error"]: self.log_failure("Invalid exception for sync_write: %s" % result) self.log.info("Resuming memcached on: %s" % node_to_stop) err_sim.revert(CouchbaseError.STOP_MEMCACHED) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(1) self.log.info("Closing ssh & SDK connections") ssh_conn.disconnect() client.close() self.validate_test_failure()
def test_durability_impossible(self): """ Create bucket with replica > num_kv_nodes. Perform doc insert to make sure we get TimeoutException due to durability_impossible from the server. """ verification_dict = self.get_cb_stat_verification_dict() key, value = doc_generator("test_key", 0, 1).next() for d_level in self.possible_d_levels[self.bucket_type]: if d_level == Bucket.DurabilityLevel.NONE: continue bucket_dict = self.get_bucket_dict(self.bucket_type, d_level) # Object to support performing CRUDs bucket_obj = Bucket(bucket_dict) self.bucket_util.create_bucket(self.cluster, bucket_obj, wait_for_warmup=True) self.summary.add_step("Create bucket with durability %s" % d_level) client = SDKClient([self.cluster.master], bucket_obj) result = client.crud("create", key, value, timeout=3) if result["status"] is True \ or SDKException.DurabilityImpossibleException \ not in result["error"]: self.log_failure("Indirect sync_write succeeded " "without enough nodes") client.close() # Cbstats vbucket-details validation self.cb_stat_verify(verification_dict) # Delete the created bucket self.bucket_util.delete_bucket(self.cluster, bucket_obj) self.summary.add_step("Delete bucket with d_level %s" % d_level)
def verify_cas(self, ops, generator): """ Verify CAS value manipulation. For update we use the latest CAS value return by set() to do the mutation again to see if there is any exceptions. We should be able to mutate that item with the latest CAS value. For delete(), after it is called, we try to mutate that item with the cas value returned by delete(). We should see SDK Error. Otherwise the test should fail. For expire, We want to verify using the latest CAS value of that item can not mutate it because it is expired already. """ for bucket in self.bucket_util.buckets: client = SDKClient(RestConnection(self.cluster.master), bucket) gen = generator while gen.has_next(): key, value = gen.next() vb_of_key = self.bucket_util.get_vbucket_num_for_key(key) active_node_ip = None for node_ip in self.shell_conn.keys(): if vb_of_key in self.vb_details[node_ip]["active"]: active_node_ip = node_ip break self.log.info("Performing %s on key %s" % (ops, key)) if ops in ["update", "touch"]: for x in range(self.mutate_times): old_cas = client.crud("read", key, timeout=10)["cas"] # value = {"val": "mysql-new-value-%s" % x} if ops == 'update': result = client.crud( "replace", key, value, durability=self.durability_level, cas=old_cas) else: result = client.touch( key, 0, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: client.close() self.log_failure("Touch / replace with cas failed") return new_cas = result["cas"] if old_cas == new_cas: self.log_failure("CAS old (%s) == new (%s)" % (old_cas, new_cas)) if ops == 'update': if json.loads(str(result["value"])) \ != json.loads(value): self.log_failure("Value mismatch. " "%s != %s" % (result["value"], value)) else: self.log.debug( "Mutate %s with CAS %s successfully! " "Current CAS: %s" % (old_cas, key, new_cas)) active_read = client.crud("read", key, timeout=self.sdk_timeout) active_cas = active_read["cas"] replica_cas = -1 cas_in_active_node = \ self.cb_stat[active_node_ip].vbucket_details( bucket.name)[str(vb_of_key)]["max_cas"] if str(cas_in_active_node) != str(new_cas): self.log_failure("CbStats CAS mismatch. %s != %s" % (cas_in_active_node, new_cas)) poll_count = 0 max_retry = 5 while poll_count < max_retry: replica_read = client.getFromReplica( key, ReplicaMode.FIRST)[0] replica_cas = replica_read["cas"] if active_cas == replica_cas \ or self.durability_level: break poll_count = poll_count + 1 self.sleep(1, "Retry read CAS from replica..") if active_cas != replica_cas: self.log_failure("Replica cas mismatch. %s != %s" % (new_cas, replica_cas)) elif ops == "delete": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("delete", key, durability=self.durability_level, timeout=self.sdk_timeout) self.log.info("CAS after delete of key %s: %s" % (key, result["cas"])) result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("The item should already be deleted") if DurableExceptions.KeyNotFoundException \ not in result["error"]: self.log_failure("Invalid Excepetion: %s" % result) if result["cas"] != 0: self.log_failure("Delete returned invalid cas: %s, " "Expected 0" % result["cas"]) if result["cas"] == old_cas: self.log_failure("Deleted doc returned old cas: %s " % old_cas) elif ops == "expire": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("touch", key, exp=self.expire_time) if result["status"] is True: if result["cas"] == old_cas: self.log_failure("Touch failed to update CAS") else: self.log_failure("Touch operation failed") self.sleep(self.expire_time + 1, "Wait for item to expire") result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("Able to mutate %s with old cas: %s" % (key, old_cas)) if ClientException.KeyNotFoundException \ not in result["error"]: self.log_failure("Invalid error after expiry: %s" % result)
def test_sync_write_in_progress(self): doc_ops = self.input.param("doc_ops", "create;create").split(';') shell_conn = dict() cbstat_obj = dict() error_sim = dict() vb_info = dict() active_vbs = dict() replica_vbs = dict() # Override d_level, error_simulation type based on d_level self.__get_d_level_and_error_to_simulate() # Acquire SDK client from the pool for performing doc_ops locally client = SDKClient([self.cluster.master], self.bucket) target_nodes = DurabilityHelper.getTargetNodes(self.cluster, self.nodes_init, self.num_nodes_affected) for node in target_nodes: shell_conn[node.ip] = RemoteMachineShellConnection(node) cbstat_obj[node.ip] = Cbstats(node) vb_info["init"] = dict() vb_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno( self.bucket.name) error_sim[node.ip] = CouchbaseError(self.log, shell_conn[node.ip]) # Fetch affected nodes' vb_num which are of type=replica active_vbs[node.ip] = cbstat_obj[node.ip].vbucket_list( self.bucket.name, vbucket_type="active") replica_vbs[node.ip] = cbstat_obj[node.ip].vbucket_list( self.bucket.name, vbucket_type="replica") if self.durability_level \ == Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE: target_vbs = active_vbs target_vbuckets = list() for target_node in target_nodes: target_vbuckets += target_vbs[target_node.ip] else: target_vbuckets = replica_vbs[target_nodes[0].ip] if len(target_nodes) > 1: index = 1 while index < len(target_nodes): target_vbuckets = list( set(target_vbuckets).intersection( set(replica_vbs[target_nodes[index].ip]))) index += 1 doc_load_spec = dict() doc_load_spec["doc_crud"] = dict() doc_load_spec["doc_crud"][MetaCrudParams.DocCrud.COMMON_DOC_KEY] \ = "test_collections" doc_load_spec[MetaCrudParams.TARGET_VBUCKETS] = target_vbuckets doc_load_spec[MetaCrudParams.COLLECTIONS_CONSIDERED_FOR_CRUD] = 5 doc_load_spec[MetaCrudParams.SCOPES_CONSIDERED_FOR_CRUD] = "all" doc_load_spec[MetaCrudParams.DURABILITY_LEVEL] = self.durability_level doc_load_spec[MetaCrudParams.SDK_TIMEOUT] = 60 if doc_ops[0] == DocLoading.Bucket.DocOps.CREATE: doc_load_spec["doc_crud"][ MetaCrudParams.DocCrud.CREATE_PERCENTAGE_PER_COLLECTION] = 1 elif doc_ops[0] == DocLoading.Bucket.DocOps.UPDATE: doc_load_spec["doc_crud"][ MetaCrudParams.DocCrud.UPDATE_PERCENTAGE_PER_COLLECTION] = 1 elif doc_ops[0] == DocLoading.Bucket.DocOps.REPLACE: doc_load_spec["doc_crud"][ MetaCrudParams.DocCrud.REPLACE_PERCENTAGE_PER_COLLECTION] = 1 elif doc_ops[0] == DocLoading.Bucket.DocOps.DELETE: doc_load_spec["doc_crud"][ MetaCrudParams.DocCrud.DELETE_PERCENTAGE_PER_COLLECTION] = 1 # Induce error condition for testing for node in target_nodes: error_sim[node.ip].create(self.simulate_error, bucket_name=self.bucket.name) self.sleep(3, "Wait for error simulation to take effect") doc_loading_task = \ self.bucket_util.run_scenario_from_spec( self.task, self.cluster, self.cluster.buckets, doc_load_spec, async_load=True) self.sleep(5, "Wait for doc ops to reach server") for bucket, s_dict in doc_loading_task.loader_spec.items(): for s_name, c_dict in s_dict["scopes"].items(): for c_name, c_meta in c_dict["collections"].items(): client.select_collection(s_name, c_name) for op_type in c_meta: key, value = c_meta[op_type]["doc_gen"].next() if self.with_non_sync_writes: fail = client.crud(doc_ops[1], key, value, exp=0, timeout=2, time_unit="seconds") else: fail = client.crud( doc_ops[1], key, value, exp=0, durability=self.durability_level, timeout=2, time_unit="seconds") expected_exception = \ SDKException.AmbiguousTimeoutException retry_reason = \ SDKException.RetryReason.KV_SYNC_WRITE_IN_PROGRESS if doc_ops[0] == DocLoading.Bucket.DocOps.CREATE \ and doc_ops[1] in \ [DocLoading.Bucket.DocOps.DELETE, DocLoading.Bucket.DocOps.REPLACE]: expected_exception = \ SDKException.DocumentNotFoundException retry_reason = None # Validate the returned error from the SDK if expected_exception not in str(fail["error"]): self.log_failure("Invalid exception for %s: %s" % (key, fail["error"])) if retry_reason \ and retry_reason not in str(fail["error"]): self.log_failure( "Invalid retry reason for %s: %s" % (key, fail["error"])) # Try reading the value in SyncWrite state fail = client.crud("read", key) if doc_ops[0] == "create": # Expected KeyNotFound in case of CREATE op if fail["status"] is True: self.log_failure( "%s returned value during SyncWrite %s" % (key, fail)) else: # Expects prev val in case of other operations if fail["status"] is False: self.log_failure( "Key %s read failed for prev value: %s" % (key, fail)) # Revert the introduced error condition for node in target_nodes: error_sim[node.ip].revert(self.simulate_error, bucket_name=self.bucket.name) # Wait for doc_loading to complete self.task_manager.get_task_result(doc_loading_task) self.bucket_util.validate_doc_loading_results(doc_loading_task) if doc_loading_task.result is False: self.log_failure("Doc CRUDs failed") # Release the acquired SDK client client.close() self.validate_test_failure()
class basic_ops(BaseTestCase): def setUp(self): super(basic_ops, self).setUp() self.test_log = logging.getLogger("test") self.key = 'test_docs'.rjust(self.key_size, '0') nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.bucket_util.add_rbac_user() if self.default_bucket: self.bucket_util.create_default_bucket(replica=self.num_replicas, compression_mode=self.compression_mode, ram_quota=100) time.sleep(10) self.def_bucket= self.bucket_util.get_all_buckets() self.client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0]) self.__durability_level() self.create_Transaction() self._stop = threading.Event() self.log.info("==========Finished Basic_ops base setup========") def tearDown(self): self.client.close() super(basic_ops, self).tearDown() def __durability_level(self): if self.durability_level == "MAJORITY": self.durability = 1 elif self.durability_level == "MAJORITY_AND_PERSIST_ON_MASTER": self.durability = 2 elif self.durability_level == "PERSIST_TO_MAJORITY": self.durability = 3 elif self.durability_level == "ONLY_NONE": self.durability = 4 else: self.durability = 0 def get_doc_generator(self, start, end): age = range(5) first = ['james', 'sharon'] body = [''.rjust(self.doc_size - 10, 'a')] template = '{{ "age": {0}, "first_name": "{1}", "body": "{2}"}}' generator = DocumentGenerator(self.key, template, age, first, body, start=start, end=end) return generator def set_exception(self, exception): self.exception = exception raise BaseException("Got an exception {}".format(self.exception)) def __chunks(self, l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] def create_Transaction(self, client=None): if not client: client = self.client transaction_config = Transaction().createTransactionConfig(self.transaction_timeout, self.durability) try: self.transaction = Transaction().createTansaction(client.cluster, transaction_config) except Exception as e: self.set_exception(e) def __thread_to_transaction(self, transaction, op_type, doc, txn_commit, update_count=1, sync=True, set_exception=True, client=None): if not client: client = self.client if op_type == "create": exception = Transaction().RunTransaction(transaction, [client.collection], doc, [], [], txn_commit, sync, update_count) elif op_type == "update": self.test_log.info("updating all the keys through threads") exception = Transaction().RunTransaction(transaction, [client.collection], [], doc, [], txn_commit, sync, update_count) elif op_type == "delete": exception = Transaction().RunTransaction(transaction, [client.collection], [], [], doc, txn_commit, sync, update_count) if set_exception: if exception: self.set_exception("Failed") def doc_gen(self, num_items, start=0, value={'value':'value1'}): self.docs = [] self.keys = [] self.content = self.client.translate_to_json_object(value) for i in range(start, self.num_items): key = "test_docs-" + str(i) doc = Tuples.of(key, self.content) self.keys.append(key) self.docs.append(doc) def verify_doc(self, num_items, client): for i in range(num_items): key = "test_docs-" + str(i) result = client.read(key) actual_val = self.client.translate_to_json_object(result['value']) self.assertEquals(self.content, actual_val) def test_MultiThreadTxnLoad(self): # Atomicity.basic_retry.basic_ops.test_MultiThreadTxnLoad,num_items=1000 ''' Load data through txn, update half the items through different threads and delete half the items through different threads. if update_retry then update and delete the same key in two different transaction and make sure update fails ''' self.num_txn = self.input.param("num_txn", 9) self.update_retry = self.input.param("update_retry", False) self.doc_gen(self.num_items) threads = [] # create the docs exception = Transaction().RunTransaction(self.transaction, [self.client.collection], self.docs, [], [], self.transaction_commit, True, self.update_count) if exception: self.set_exception("Failed") if self.update_retry: threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", self.keys, self.transaction_commit, self.update_count))) threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, self.update_count))) else: update_docs = self.__chunks(self.keys[:self.num_items/2], self.num_txn) delete_docs = self.__chunks(self.keys[self.num_items/2:], self.num_txn) for keys in update_docs: threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", keys, self.transaction_commit, self.update_count))) for keys in delete_docs: threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", keys, self.transaction_commit, self.update_count))) for thread in threads: thread.start() for thread in threads: thread.join() self.sleep(60) if self.update_retry: for key in self.keys: result = self.client.read(key) self.assertEquals(result['status'], False) else: self.value = {'mutated':1, 'value':'value1'} self.content = self.client.translate_to_json_object(self.value) self.verify_doc(self.num_items/2, self.client) for key in self.keys[self.num_items/2:]: result = self.client.read(key) self.assertEquals(result['status'], False) def test_basic_retry(self): ''' Load set of data to the cluster, update through 2 different threads, make sure transaction maintains the order of update''' self.write_conflict = self.input.param("write_conflict", 2) self.test_log.info("going to create and execute the task") self.gen_create = self.get_doc_generator(0, self.num_items) task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket, self.gen_create, "create" , exp=0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True,durability=self.durability_level,sync=self.sync) self.task.jython_task_manager.get_task_result(task) self.test_log.info("get all the keys in the cluster") self.doc_gen(self.num_items) threads = [] for update_count in [2, 4, 6]: threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, update_count))) # Add verification task if self.transaction_commit: self.update_count = 6 else: self.update_count = 0 for thread in threads: thread.start() self.sleep(2) for thread in threads: thread.join() self.sleep(10) task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket, self.gen_create, "verify" , exp=0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True,durability=self.durability_level) self.task.jython_task_manager.get_task_result(task) def test_basic_retry_async(self): self.test_log.info("going to create and execute the task") self.gen_create = self.get_doc_generator(0, self.num_items) task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket, self.gen_create, "create" , exp=0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries,update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True,durability=self.durability_level,sync=True,num_threads=1) self.task.jython_task_manager.get_task_result(task) self.test_log.info("get all the keys in the cluster") keys = ["test_docs-0"]*20 exception = Transaction().RunTransaction(self.transaction, [self.client.collection], [], keys, [], self.transaction_commit, False, 0) if exception: self.set_exception(Exception(exception)) def basic_concurrency(self): self.crash = self.input.param("crash", False) self.doc_gen(self.num_items) # run transaction thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count, True, False)) thread.start() self.sleep(1) if self.crash: self.client.cluster.shutdown() self.transaction.close() print "going to create a new transaction" self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0]) self.create_Transaction(self.client1) self.sleep(self.transaction_timeout+60) exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], self.docs, [], [], self.transaction_commit, self.sync, self.update_count) if exception: time.sleep(60) self.verify_doc(self.num_items, self.client1) self.client1.close() else: key = "test_docs-0" # insert will fail result = self.client.insert(key, "value") self.assertEqual(result["status"], False) # Update should pass result = self.client.upsert(key,"value") self.assertEqual(result["status"], True) # delete should pass result = self.client.delete(key) self.assertEqual(result["status"], True) thread.join() def test_stop_loading(self): ''' Load through transactions and close the transaction abruptly, create a new transaction sleep for 60 seconds and perform create on the same set of docs ''' self.num_txn = self.input.param("num_txn", 9) self.doc_gen(self.num_items) threads = [] docs = list(self.__chunks(self.docs, len(self.docs)/self.num_txn)) for doc in docs: threads.append(threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", doc, self.transaction_commit, self.update_count, True, False))) for thread in threads: thread.start() self.client.cluster.shutdown() self.transaction.close() self.client1 = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0]) self.create_Transaction(self.client1) self.sleep(self.transaction_timeout+60) # sleep for 60 seconds so that transaction cleanup can happen self.test_log.info("going to start the load") for doc in docs: exception = Transaction().RunTransaction(self.transaction, [self.client1.collection], doc, [], [], self.transaction_commit, self.sync, self.update_count) if exception: time.sleep(60) self.verify_doc(self.num_items, self.client1) self.client1.close() def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value): _, failed_items = self.client.crud( op_type, doc_id, [key, value], durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds", create_path=True, xattr=True) self.assertFalse(failed_items, "Subdoc Xattr insert failed") def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None): if subdoc_key: success, failed_items = self.client.crud("subdoc_read", doc_id, subdoc_key, xattr=True) self.assertFalse(failed_items, "Xattr read failed") self.assertEqual(expected_val, str(success[doc_id]["value"][0]), "Sub_doc value mismatch: %s != %s" % (success[doc_id]["value"][0], expected_val)) def test_TxnWithXattr(self): self.system_xattr = self.input.param("system_xattr", False) if self.system_xattr: xattr_key = "my._attr" else: xattr_key="my.attr" val = "v" * self.doc_size self.doc_gen(self.num_items) threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count)) threads.start() self.sleep(1) self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert", xattr_key, val) threads.join() if self.transaction_commit: self.__read_doc_and_validate("test_docs-0", val, xattr_key) self.sleep(60) self.verify_doc(self.num_items, self.client) def test_TxnWithMultipleXattr(self): xattrs_to_insert = [["my.attr", "value"], ["new_my.attr", "new_value"]] self.doc_gen(self.num_items) threads = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count)) threads.start() self.sleep(1) for key, val in xattrs_to_insert: self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert", key, val) threads.join() if self.transaction_commit: for key, val in xattrs_to_insert: self.__read_doc_and_validate("test_docs-0", val, key) self.sleep(60) self.verify_doc(self.num_items, self.client)
def test_bucket_durability_upgrade(self): update_task = None self.sdk_timeout = 60 create_batch_size = 10000 if self.atomicity: create_batch_size = 10 # To make sure sync_write can we supported by initial cluster version sync_write_support = True if float(self.initial_version[0:3]) < 6.5: sync_write_support = False if sync_write_support: self.verification_dict["rollback_item_count"] = 0 self.verification_dict["sync_write_aborted_count"] = 0 if self.upgrade_with_data_load: self.log.info("Starting async doc updates") update_task = self.task.async_continuous_doc_ops( self.cluster, self.bucket, self.gen_load, op_type=DocLoading.Bucket.DocOps.UPDATE, process_concurrency=1, persist_to=1, replicate_to=1, timeout_secs=30) self.log.info("Upgrading cluster nodes to target version") node_to_upgrade = self.fetch_node_to_upgrade() while node_to_upgrade is not None: self.log.info("Selected node for upgrade: %s" % node_to_upgrade.ip) self.upgrade_function[self.upgrade_type](node_to_upgrade, self.upgrade_version) try: self.cluster.update_master_using_diag_eval( self.cluster.servers[0]) except Exception: self.cluster.update_master_using_diag_eval( self.cluster.servers[self.nodes_init-1]) create_gen = doc_generator(self.key, self.num_items, self.num_items+create_batch_size) # Validate sync_write results after upgrade if self.atomicity: sync_write_task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, create_gen, DocLoading.Bucket.DocOps.CREATE, process_concurrency=1, transaction_timeout=self.transaction_timeout, record_fail=True) else: sync_write_task = self.task.async_load_gen_docs( self.cluster, self.bucket, create_gen, DocLoading.Bucket.DocOps.CREATE, timeout_secs=self.sdk_timeout, process_concurrency=4, sdk_client_pool=self.sdk_client_pool, skip_read_on_error=True, suppress_error_table=True) self.task_manager.get_task_result(sync_write_task) self.num_items += create_batch_size retry_index = 0 while retry_index < 5: self.sleep(3, "Wait for num_items to match") current_items = self.bucket_util.get_bucket_current_item_count( self.cluster, self.bucket) if current_items == self.num_items: break self.log.debug("Num_items mismatch. Expected: %s, Actual: %s" % (self.num_items, current_items)) # Doc count validation self.cluster_util.print_cluster_stats() self.verification_dict["ops_create"] += create_batch_size self.summary.add_step("Upgrade %s" % node_to_upgrade.ip) # Halt further upgrade if test has failed during current upgrade if self.test_failure: break node_to_upgrade = self.fetch_node_to_upgrade() if self.upgrade_with_data_load: # Wait for update_task to complete update_task.end_task() self.task_manager.get_task_result(update_task) else: self.verification_dict["ops_update"] = 0 # Cb_stats vb-details validation failed = self.durability_helper.verify_vbucket_details_stats( self.bucket_util.buckets[0], self.cluster_util.get_kv_nodes(), vbuckets=self.cluster_util.vbuckets, expected_val=self.verification_dict) if failed: self.log_failure("Cbstat vbucket-details validation failed") self.summary.add_step("Cbstats vb-details verification") self.validate_test_failure() possible_d_levels = dict() possible_d_levels[Bucket.Type.MEMBASE] = \ self.bucket_util.get_supported_durability_levels() possible_d_levels[Bucket.Type.EPHEMERAL] = [ Bucket.DurabilityLevel.NONE, Bucket.DurabilityLevel.MAJORITY] len_possible_d_levels = len(possible_d_levels[self.bucket_type]) - 1 if not sync_write_support: self.verification_dict["rollback_item_count"] = 0 self.verification_dict["sync_write_aborted_count"] = 0 # Perform bucket_durability update key, value = doc_generator("b_durability_doc", 0, 1).next() client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) for index, d_level in enumerate(possible_d_levels[self.bucket_type]): self.log.info("Updating bucket_durability=%s" % d_level) self.bucket_util.update_bucket_property( self.bucket_util.buckets[0], bucket_durability=BucketDurability[d_level]) self.bucket_util.print_bucket_stats() buckets = self.bucket_util.get_all_buckets() if buckets[0].durability_level != BucketDurability[d_level]: self.log_failure("New bucket_durability not taken") self.summary.add_step("Update bucket_durability=%s" % d_level) self.sleep(10, "MB-39678: Bucket_d_level change to take effect") if index == 0: op_type = DocLoading.Bucket.DocOps.CREATE self.verification_dict["ops_create"] += 1 elif index == len_possible_d_levels: op_type = DocLoading.Bucket.DocOps.DELETE self.verification_dict["ops_delete"] += 1 else: op_type = DocLoading.Bucket.DocOps.UPDATE if "ops_update" in self.verification_dict: self.verification_dict["ops_update"] += 1 result = client.crud(op_type, key, value, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Doc_op %s failed on key %s: %s" % (op_type, key, result["error"])) self.summary.add_step("Doc_op %s" % op_type) client.close() # Cb_stats vb-details validation failed = self.durability_helper.verify_vbucket_details_stats( self.bucket_util.buckets[0], self.cluster_util.get_kv_nodes(), vbuckets=self.cluster_util.vbuckets, expected_val=self.verification_dict) if failed: self.log_failure("Cbstat vbucket-details validation failed") self.summary.add_step("Cbstats vb-details verification") self.validate_test_failure()
class BasicCrudTests(MagmaBaseTest): def setUp(self): super(BasicCrudTests, self).setUp() self.enable_disable_swap_space(self.servers) start = 0 end = self.num_items start_read = 0 end_read = self.num_items if self.rev_write: start = -int(self.num_items - 1) end = 1 if self.rev_read: start_read = -int(self.num_items - 1) end_read = 1 self.gen_create = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.result_task = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size, dgm_batch=self.dgm_batch) if self.active_resident_threshold != 100: for task in self.result_task.keys(): self.num_items = task.doc_index self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.disk_usage = dict() if self.standard_buckets == 1 or self.standard_buckets == self.magma_buckets: for bucket in self.bucket_util.get_all_buckets(): disk_usage = self.get_disk_usage(bucket, self.servers) self.disk_usage[bucket.name] = disk_usage[0] self.log.info( "For bucket {} disk usage after initial creation is {}MB\ ".format(bucket.name, self.disk_usage[bucket.name])) self.gen_read = doc_generator( self.key, start_read, end_read, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() def tearDown(self): super(BasicCrudTests, self).tearDown() def test_expiry(self): result = True self.gen_create = doc_generator(self.key, 0, 10, doc_size=20, doc_type=self.doc_type, key_size=self.key_size) tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", 10, batch_size=10, process_concurrency=1, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, ) self.task.jython_task_manager.get_task_result(tasks_info.keys()[0]) self.sleep(20) self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) for i in range(10): key = (self.key + "-" + str(i).zfill(self.key_size - len(self.key))) try: getReplicaResult = self.client.collection.getAnyReplica( key, GetAnyReplicaOptions.getAnyReplicaOptions()) if getReplicaResult: result = False try: self.log.info( "Able to retreive: %s" % { "key": key, "value": getReplicaResult.contentAsObject(), "cas": getReplicaResult.cas() }) except Exception as e: print str(e) except DocumentUnretrievableException as e: pass if len(self.client.getFromAllReplica(key)) > 0: result = False self.client.close() self.assertTrue(result, "SDK is able to retrieve expired documents") def test_basic_create_read(self): """ Write and Read docs parallely , While reading we are using old doc generator (self.gen_create) using which we already created docs in magam_base for writing we are creating a new doc generator. Befor we start read, killing memcached to make sure, all reads happen from magma/storage """ self.log.info("Loading and Reading docs parallel") count = 0 init_items = self.num_items while count < self.test_itr: self.log.info("Create Iteration count == {}".format(count)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.doc_ops = "create:read" start = self.num_items end = self.num_items + init_items start_read = self.num_items end_read = self.num_items + init_items if self.rev_write: start = -int(self.num_items + init_items - 1) end = -int(self.num_items - 1) if self.rev_read: start_read = -int(self.num_items + init_items - 1) end_read = -int(self.num_items - 1) self.gen_create = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.gen_read = doc_generator( self.key, start_read, end_read, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) if self.doc_size <= 32: for bucket in self.bucket_util.get_all_buckets(): disk_usage = self.get_disk_usage(bucket, self.servers) self.assertIs( disk_usage[2] > disk_usage[3], True, "For Bucket {} , Disk Usage for seqIndex'\n' \ After new Creates count {}'\n' \ exceeds keyIndex disk'\n' \ usage".format(bucket.name, count + 1)) if self.standard_buckets > 1 and self.standard_buckets == self.magma_buckets: disk_usage = dict() for bucket in self.bucket_util.get_all_buckets(): usage = self.get_disk_usage(bucket, self.servers) disk_usage[bucket.name] = usage[0] self.assertTrue( all([ disk_usage[disk_usage.keys()[0]] == disk_usage[key] for key in disk_usage.keys() ]), '''Disk Usage for magma buckets is not equal for same number of docs ''') count += 1 self.log.info("====test_basic_create_read ends====") def test_update_multi(self): """ Update all the docs n times, and after each iteration check for space amplificationa and data validation """ count = 0 mutated = 1 update_doc_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("Count of docs to be updated is {}\ ".format(update_doc_count)) num_update = list() while update_doc_count > self.num_items: num_update.append(self.num_items) update_doc_count -= self.num_items if update_doc_count > 0: num_update.append(update_doc_count) while count < self.test_itr: self.log.info("Update Iteration count == {}".format(count)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) self.log.debug("List of docs to be updated {}\ ".format(num_update)) for itr in num_update: self.doc_ops = "update" start = 0 end = itr if self.rev_update: start = -int(itr - 1) end = 1 self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After update count {} disk usage is {}\ ".format(count + 1, _res)) usage_factor = ( (float(self.num_items + sum(num_update)) / self.num_items) + 0.5) self.log.debug("Disk usage factor is {}".format(usage_factor)) self.assertIs( _res > usage_factor * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After Update'\n' \ Count {} exceeds Actual'\n' \ disk usage {}MB by {}'\n' \ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]], usage_factor)) count += 1 data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.enable_disable_swap_space(self.servers, disable=False) self.log.info("====test_update_multi ends====") def test_multi_update_delete(self): """ Step 1: Kill memcached and Update all the docs update_itr times After each iteration check for space amplification and for last iteration of test_itr validate docs Step 2: Delete half the docs, check sapce amplification Step 3 Recreate check for space amplification. Repeat all above steps test_itr times Step 4 : Do data validation for newly create docs """ count = 0 mutated = 1 for i in range(self.test_itr): while count < self.update_itr: self.log.debug( "Iteration {}: Step 1 of test_multi_update_delete \ ".format(self.test_itr + 1)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) self.doc_ops = "update" start = 0 end = self.num_items if self.rev_update: start = -int(self.num_items - 1) end = 1 self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After update count {} disk usage is {}MB\ ".format(count + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ Update Count {} exceeds Actual \ disk usage {}MB by 2.5\ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]])) count += 1 # Will check data validatio only in the last # iteration of test_tr to avoid multiple # data validation, that is why below if check if i + 1 == self.test_itr: data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.update_itr += self.update_itr self.log.debug("Iteration {}: Step 2 of test_multi_update_delete \ ".format(self.test_itr + 1)) start_del = 0 end_del = self.num_items // 2 if self.rev_del: start_del = -int(self.num_items // 2 - 1) end_del = 1 self.gen_delete = doc_generator( self.key, start_del, end_del, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.log.info("Deleting num_items//2 docs") self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After delete count {} disk usage is {}MB\ ".format(i + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ Delete count {} exceeds Actual \ disk usage {}MB by 2.5 \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.debug("Iteration{}: Step 3 of test_multi_update_delete \ ".format(self.test_itr + 1)) self.gen_create = copy.deepcopy(self.gen_delete) self.log.info("Recreating num_items//2 docs") self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("disk usage after new create \ is {}".format(_res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ new Creates count {} exceeds \ Actual disk usage {}MB by \ 2.5 times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.debug("Iteration{}: Step 4 of test_multi_update_delete \ ".format(self.test_itr + 1)) data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.log.info("====test_multiUpdate_delete ends====") def test_update_rev_update(self): count = 0 mutated = 1 for i in range(self.test_itr): while count < self.update_itr: for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) tasks_info = dict() data_validation = [] g_update = doc_generator( self.key, 0, self.num_items // 2, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, g_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(tem_tasks_info.items()) start = -(self.num_items // 2 - 1) end = 1 r_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 if self.next_half: mutated -= 2 start = -(self.num_items - 1) end = -(self.num_items // 2 - 1) r_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(tem_tasks_info.items()) for task in tasks_info: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tasks_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(tasks_info) if not self.next_half: tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) for task in tem_tasks_info: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tem_tasks_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(tem_tasks_info) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() if self.next_half: data_validation.extend([ self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], g_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout), self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) ]) else: data_validation.append( self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout)) for task in data_validation: self.task.jython_task_manager.get_task_result(task) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after update count {}\ is {}".format(count + 1, _res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ Update Count {} exceeds \ Actual disk usage {} by four \ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]])) count += 1 self.update_itr += self.update_itr start_del = 0 end_del = self.num_items // 2 if self.rev_del: start_del = -int(self.num_items // 2 - 1) end_del = 1 self.gen_delete = doc_generator( self.key, start_del, end_del, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.log.info("Deleting num_items//2 docs") self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after delete is {}".format(_res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ Delete count {} exceeds Actual \ disk usage {} by four \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.gen_create = copy.deepcopy(self.gen_delete) self.log.info("Recreating num_items//2 docs") self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) d_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(d_validation) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after new create \ is {}".format(_res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ new Creates count {} exceeds \ Actual disk usage {} by four \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.info("====test_update_rev_update ends====") def test_update_single_doc_n_times(self): """ Update same document n times, where n is number which gets derived from given fragmentation value and after updates check for space amplification and data validation """ count = 0 self.assertIs((self.fragmentation <= 0 or self.fragmentation >= 100), False, msg="Fragmentation value can't be <=0 or >=100") update_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("{} is the count with which doc will be updated \ ".format(update_count)) self.doc_ops = "update" self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) self.gen_update = doc_generator( self.key, 0, 1, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=count, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) key, val = self.gen_update.next() for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) while count < (update_count + 1): self.log.debug("Update Iteration count == {}".format(count)) val.put("mutated", count + 1) self.client.upsert(key, val) count += 1 self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After all updates disk usage is {}MB\ ".format(_res)) usage_factor = ( (float(self.num_items + update_count) / self.num_items) + 0.5) self.log.debug("Disk usage factor is {}".format(usage_factor)) self.assertIs( _res > usage_factor * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After all Updates'\n' \ exceeds Actual'\n' \ disk usage {}MB by {}'\n' \ times".format(_res, self.disk_usage[self.disk_usage.keys()[0]], usage_factor)) data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.enable_disable_swap_space(self.servers, disable=False) self.log.info("====test_update_single_doc_n_times====") def test_read_docs_using_multithreads(self): """ Read same docs together using multithreads. """ self.log.info("Reading docs parallelly using multi threading") tasks_info = dict() update_doc_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("Count of docs to be updated is {}\ ".format(update_doc_count)) num_update = list() while update_doc_count > self.num_items: num_update.append(self.num_items) update_doc_count -= self.num_items if update_doc_count > 0: num_update.append(update_doc_count) for itr in num_update: self.doc_ops = "update" start = 0 end = itr self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=0, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) update_task_info = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=False) tasks_info.update(update_task_info.items()) count = 0 self.doc_ops = "read" # if self.next_half is true then one thread will read # in ascending order and other in descending order if self.next_half: start = -int(self.num_items - 1) end = 1 g_read = doc_generator(self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() while count < self.read_thread_count: read_task_info = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=False) tasks_info.update(read_task_info.items()) count += 1 if self.next_half and count < self.read_thread_count: read_tasks_info = self.bucket_util._async_validate_docs( self.cluster, g_read, "read", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(read_task_info.items()) count += 1 for task in tasks_info: self.task_manager.get_task_result(task) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() self.log.info("test_read_docs_using_multithreads ends") def test_basic_create_delete(self): """ CREATE(n)-> DELETE(n)->DISK_USAGE_CHECK REPEAT ABove test_itr_times """ self.log.info("Cretaing and Deletes docs n times ") count = 0 start = 0 end = self.num_items self.gen_delete = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) while count < self.test_itr: self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Verifying doc counts after delete doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("disk usage after delete count {} \ is {}MB".format(count + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ delete count {} exceeds \ Actual disk usage {}MB by \ 2.5 times".format(_res, count + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) count += 1 self.log.info("====test_basic_create_delete ends====")
class basic_ops(ClusterSetup): def setUp(self): super(basic_ops, self).setUp() if self.default_bucket: # Over-ride bucket ram quota=100MB self.bucket_size = 100 self.create_bucket(self.cluster) self.sleep(10, "Wait for bucket to become ready for ops") self.def_bucket = self.bucket_util.get_all_buckets(self.cluster) self.client = SDKClient([self.cluster.master], self.def_bucket[0]) self.__durability_level() self.create_Transaction() self._stop = threading.Event() self.log.info("==========Finished Basic_ops base setup========") def tearDown(self): self.client.close() super(basic_ops, self).tearDown() def __durability_level(self): if self.durability_level == Bucket.DurabilityLevel.MAJORITY: self.durability = 1 elif self.durability_level \ == Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE: self.durability = 2 elif self.durability_level \ == Bucket.DurabilityLevel.PERSIST_TO_MAJORITY: self.durability = 3 elif self.durability_level == "ONLY_NONE": self.durability = 4 else: self.durability = 0 def get_doc_generator(self, start, end): age = range(5) name = ['james', 'sharon'] body = [''.rjust(self.doc_size - 10, 'a')] template = JsonObject.create() template.put("age", age) template.put("first_name", name) template.put("body", body) generator = DocumentGenerator(self.key, template, start=start, end=end, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, randomize=True, age=age, first_name=name) return generator def set_exception(self, exception): self.exception = exception raise BaseException("Got an exception %s" % self.exception) def __chunks(self, l, n): """Yield successive n-sized chunks from l.""" for i in range(0, len(l), n): yield l[i:i + n] def create_Transaction(self, client=None): if not client: client = self.client transaction_config = Transaction().createTransactionConfig( self.transaction_timeout, self.durability) try: self.transaction = Transaction().createTansaction( client.cluster, transaction_config) except Exception as e: self.set_exception(e) def __thread_to_transaction(self, transaction, op_type, doc, txn_commit, update_count=1, sync=True, set_exception=True, client=None): exception = None if client is None: client = self.client if op_type == "create": exception = Transaction().RunTransaction(client.cluster, transaction, [client.collection], doc, [], [], txn_commit, sync, update_count) elif op_type == "update": self.log.info("updating all the keys through threads") exception = Transaction().RunTransaction(client.cluster, transaction, [client.collection], [], doc, [], txn_commit, sync, update_count) elif op_type == "delete": exception = Transaction().RunTransaction(client.cluster, transaction, [client.collection], [], [], doc, txn_commit, sync, update_count) if set_exception and exception: self.set_exception("Failed") def doc_gen(self, num_items, start=0, value={'value': 'value1'}, op_type="create"): self.docs = [] self.keys = [] self.content = self.client.translate_to_json_object(value) for i in range(start, self.num_items): key = "test_docs-" + str(i) if op_type == "create": doc = Tuples.of(key, self.content) self.keys.append(key) self.docs.append(doc) else: self.docs.append(key) def verify_doc(self, num_items, client): for i in range(num_items): key = "test_docs-" + str(i) result = client.read(key) actual_val = self.client.translate_to_json_object(result['value']) self.assertEquals(self.content, actual_val) def test_MultiThreadTxnLoad(self): """ Load data through txn, update half the items through different threads and delete half the items through different threads. if update_retry then update and delete the same key in two different transaction and make sure update fails """ self.num_txn = self.input.param("num_txn", 9) self.update_retry = self.input.param("update_retry", False) self.doc_gen(self.num_items) threads = [] # create the docs exception = Transaction().RunTransaction(self.client.cluster, self.transaction, [self.client.collection], self.docs, [], [], self.transaction_commit, True, self.update_count) if exception: self.set_exception("Failed") if self.update_retry: threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", self.keys, self.transaction_commit, self.update_count))) threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, 10, self.update_count))) else: update_docs = self.__chunks(self.keys[:self.num_items / 2], self.num_txn) delete_docs = self.__chunks(self.keys[self.num_items / 2:], self.num_txn) for keys in update_docs: threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", keys, self.transaction_commit, self.update_count))) for keys in delete_docs: threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "delete", keys, self.transaction_commit, self.update_count))) for thread in threads: thread.start() for thread in threads: thread.join() self.sleep(60, "Wait for transactions to complete") if self.update_retry: for key in self.keys: result = self.client.read(key) self.assertEquals(result['status'], False) else: self.value = {'mutated': 1, 'value': 'value1'} self.content = self.client.translate_to_json_object(self.value) self.verify_doc(self.num_items / 2, self.client) for key in self.keys[self.num_items / 2:]: result = self.client.read(key) self.assertEquals(result['status'], False) def test_basic_retry(self): """ Load set of data to the cluster, update through 2 different threads, make sure transaction maintains the order of update :return: """ self.write_conflict = self.input.param("write_conflict", 2) self.log.info("going to create and execute the task") self.gen_create = self.get_doc_generator(0, self.num_items) task = self.task.async_load_gen_docs_atomicity( self.cluster, self.def_bucket, self.gen_create, "create", exp=0, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, durability=self.durability_level, sync=self.sync) self.task.jython_task_manager.get_task_result(task) self.log.info("Get all the keys in the cluster") self.doc_gen(self.num_items) threads = [] for update_count in [2, 4, 6]: threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.keys, self.transaction_commit, update_count))) # Add verification task if self.transaction_commit: self.update_count = 6 else: self.update_count = 0 for thread in threads: thread.start() self.sleep(2, "Wait for transaction thread to start") for thread in threads: thread.join() def test_basic_retry_async(self): self.log.info("going to create and execute the task") self.gen_create = self.get_doc_generator(0, self.num_items) task = self.task.async_load_gen_docs_atomicity( self.cluster, self.def_bucket, self.gen_create, "create", exp=0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, durability=self.durability_level, sync=True, num_threads=1) self.task.jython_task_manager.get_task_result(task) self.log.info("get all the keys in the cluster") keys = ["test_docs-0"] * 2 exception = Transaction().RunTransaction( self.client.cluster, self.transaction, [self.client.collection], [], keys, [], self.transaction_commit, False, 0) if exception: self.set_exception(Exception(exception)) def basic_concurrency(self): self.crash = self.input.param("crash", False) self.doc_gen(self.num_items) # run transaction thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count, True, False)) thread.start() self.sleep(1, "Wait for transaction thread to start") if self.crash: self.client.cluster.disconnect() self.transaction.close() self.client1 = SDKClient([self.cluster.master], self.def_bucket[0]) self.create_Transaction(self.client1) self.sleep(self.transaction_timeout + 60, "Wait for transaction cleanup to complete") exception = Transaction().RunTransaction( self.client.cluster, self.transaction, [self.client1.collection], self.docs, [], [], self.transaction_commit, self.sync, self.update_count) if exception: self.sleep(60, "Wait for transaction cleanup to happen") self.verify_doc(self.num_items, self.client1) self.client1.close() else: key = "test_docs-0" # insert will succeed due to doc_isoloation feature result = self.client.insert(key, "value") self.assertEqual(result["status"], True) # Update should pass result = self.client.upsert(key, "value") self.assertEqual(result["status"], True) # delete should pass result = self.client.delete(key) self.assertEqual(result["status"], True) thread.join() def test_stop_loading(self): """ Load through transactions and close the transaction abruptly, create a new transaction sleep for 60 seconds and perform create on the same set of docs """ self.num_txn = self.input.param("num_txn", 9) self.doc_gen(self.num_items) threads = [] docs = list(self.__chunks(self.docs, len(self.docs) / self.num_txn)) for doc in docs: threads.append( threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", doc, self.transaction_commit, self.update_count, True, False))) for thread in threads: thread.start() self.client.cluster.disconnect() self.transaction.close() self.client1 = SDKClient([self.cluster.master], self.def_bucket[0]) self.create_Transaction(self.client1) self.sleep(self.transaction_timeout + 60, "Wait for transaction cleanup to happen") self.log.info("going to start the load") for doc in docs: exception = Transaction().RunTransaction( self.client1.cluster, self.transaction, [self.client1.collection], doc, [], [], self.transaction_commit, self.sync, self.update_count) if exception: self.sleep(60, "Wait for transaction cleanup to happen") self.verify_doc(self.num_items, self.client1) self.client1.close() def __insert_sub_doc_and_validate(self, doc_id, op_type, key, value): _, failed_items = self.client.crud(op_type, doc_id, [key, value], durability=self.durability_level, timeout=self.sdk_timeout, time_unit="seconds", create_path=True, xattr=True) self.assertFalse(failed_items, "Subdoc Xattr insert failed") def __read_doc_and_validate(self, doc_id, expected_val, subdoc_key=None): if subdoc_key: success, failed_items = self.client.crud("subdoc_read", doc_id, subdoc_key, xattr=True) self.assertFalse(failed_items, "Xattr read failed") self.assertEqual( expected_val, str(success[doc_id]["value"][0]), "Sub_doc value mismatch: %s != %s" % (success[doc_id]["value"][0], expected_val)) def test_TxnWithXattr(self): self.system_xattr = self.input.param("system_xattr", False) if self.system_xattr: xattr_key = "my._attr" else: xattr_key = "my.attr" val = "v" * self.doc_size self.doc_gen(self.num_items) thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count)) thread.start() thread.join() self.doc_gen(self.num_items, op_type="update", value={ "mutated": 1, "value": "value1" }) thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.docs, self.transaction_commit, self.update_count)) thread.start() self.sleep(1) self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert", xattr_key, val) thread.join() if self.transaction_commit: self.__read_doc_and_validate("test_docs-0", val, xattr_key) self.sleep(60, "Wait for transaction to complete") self.verify_doc(self.num_items, self.client) def test_TxnWithMultipleXattr(self): xattrs_to_insert = [["my.attr", "value"], ["new_my.attr", "new_value"]] self.doc_gen(self.num_items) thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "create", self.docs, self.transaction_commit, self.update_count)) thread.start() thread.join() self.doc_gen(self.num_items, op_type="update", value={ "mutated": 1, "value": "value1" }) thread = threading.Thread(target=self.__thread_to_transaction, args=(self.transaction, "update", self.docs, self.transaction_commit, self.update_count)) thread.start() self.sleep(1, "Wait for transx-thread to start") for key, val in xattrs_to_insert: self.__insert_sub_doc_and_validate("test_docs-0", "subdoc_insert", key, val) thread.join() if self.transaction_commit: for key, val in xattrs_to_insert: self.__read_doc_and_validate("test_docs-0", val, key) self.sleep(60, "Wait for transaction to complete") self.verify_doc(self.num_items, self.client)
class OpsChangeCasTests(CasBaseTest): def setUp(self): super(OpsChangeCasTests, self).setUp() self.key = "test_cas" self.expire_time = self.input.param("expire_time", 35) self.item_flag = self.input.param("item_flag", 0) self.load_gen = doc_generator(self.key, 0, self.num_items, doc_size=self.doc_size) self.node_data = dict() for node in self.cluster_util.get_kv_nodes(): shell = RemoteMachineShellConnection(node) cb_stat = Cbstats(shell) self.node_data[node.ip] = dict() self.node_data[node.ip]["shell"] = shell self.node_data[node.ip]["cb_stat"] = Cbstats(shell) self.node_data[node.ip]["active"] = cb_stat.vbucket_list( self.bucket, "active") self.node_data[node.ip]["replica"] = cb_stat.vbucket_list( self.bucket, "replica") if self.sdk_client_pool: self.client = self.sdk_client_pool.get_client_for_bucket( self.bucket) else: self.client = SDKClient([self.cluster.master], self.bucket) def tearDown(self): # Close opened shell connections for node_ip in self.node_data.keys(): self.node_data[node_ip]["shell"].disconnect() # Close SDK client connection self.client.close() super(OpsChangeCasTests, self).tearDown() def test_meta_rebalance_out(self): KEY_NAME = 'key1' for i in range(10): # set a key value = 'value' + str(i) self.client.memcached(KEY_NAME).set(KEY_NAME, 0, 0, json.dumps({'value':value})) vbucket_id = self.client._get_vBucket_id(KEY_NAME) mc_active = self.client.memcached(KEY_NAME) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) cas_active = mc_active.getMeta(KEY_NAME)[4] max_cas = int(mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(KEY_NAME)) + ':max_cas'] ) self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active)) # remove that node self.log.info('Remove the node with active data') rebalance = self.cluster.async_rebalance(self.servers[-1:], [], [self.master]) rebalance.result() replica_cas = mc_replica.getMeta(KEY_NAME)[4] get_meta_resp = mc_replica.getMeta(KEY_NAME, request_extended_meta_data=False) # Add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) if rebalance.result is False: self.log_failure("Node add-back rebalance failed") # verify the CAS is good mc_active = self.client.memcached(KEY_NAME) active_cas = mc_active.getMeta(KEY_NAME)[4] if replica_cas == active_cas: self.log_failure("CAS mismatch. Active: %s, Replica: %s" % (active_cas, replica_cas)) self.validate_test_failure() def test_meta_failover(self): KEY_NAME = 'key2' for i in range(10): # set a key value = 'value' + str(i) client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(KEY_NAME) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(KEY_NAME) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) cas_active = mc_active.getMeta(KEY_NAME)[4] #print 'cas_a {0} '.format(cas_active) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] ) self.assertTrue(cas_active == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_active)) # failover that node self.log.info('Failing over node with active data {0}'.format(self.master)) self.cluster.failover(self.servers, [self.master]) self.log.info('Remove the node with active data {0}'.format(self.master)) rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master]) rebalance.result() self.sleep(60) replica_CAS = mc_replica.getMeta(KEY_NAME)[4] #print 'replica CAS {0}'.format(replica_CAS) # add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) rebalance.result() # verify the CAS is good mc_active = client.memcached(KEY_NAME) active_CAS = mc_active.getMeta(KEY_NAME)[4] #print 'active cas {0}'.format(active_CAS) get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False) #print 'replica CAS {0}'.format(replica_CAS) #print 'replica ext meta {0}'.format(get_meta_resp) self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS)) self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') def test_meta_soft_restart(self): KEY_NAME = 'key2' for i in range(10): # set a key value = 'value' + str(i) client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(KEY_NAME) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(KEY_NAME) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) cas_pre = mc_active.getMeta(KEY_NAME)[4] #print 'cas_a {0} '.format(cas_pre) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] ) self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre)) # restart nodes self._restart_server(self.servers[:]) # verify the CAS is good mc_active = client.memcached(KEY_NAME) cas_post = mc_active.getMeta(KEY_NAME)[4] #print 'post cas {0}'.format(cas_post) get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False) #print 'post CAS {0}'.format(cas_post) #print 'post ext meta {0}'.format(get_meta_resp) self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post)) # extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') def test_meta_hard_restart(self): KEY_NAME = 'key2' for i in range(10): # set a key value = 'value' + str(i) client.memcached(KEY_NAME).set(KEY_NAME, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(KEY_NAME) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(KEY_NAME) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) cas_pre = mc_active.getMeta(KEY_NAME)[4] #print 'cas_a {0} '.format(cas_pre) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(client._get_vBucket_id(KEY_NAME)) + ':max_cas'] ) self.assertTrue(cas_pre == max_cas, '[ERROR]Max cas is not 0 it is {0}'.format(cas_pre)) # reboot nodes self._reboot_server() self.sleep(60) # verify the CAS is good mc_active = client.memcached(KEY_NAME) cas_post = mc_active.getMeta(KEY_NAME)[4] #print 'post cas {0}'.format(cas_post) get_meta_resp = mc_active.getMeta(KEY_NAME,request_extended_meta_data=False) #print 'post CAS {0}'.format(cas_post) #print 'post ext meta {0}'.format(get_meta_resp) self.assertTrue(cas_pre == cas_post, 'cas mismatch active: {0} replica {1}'.format(cas_pre, cas_post)) # extended meta is not supported self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') def test_cas_set(self): """ Test Incremental sets on cas and max cas values for keys """ self._load_ops(self.load_gen, 'update', mutations=20) self._check_cas(self.load_gen, check_conflict_resolution=False) self.validate_test_failure() def test_cas_updates(self): """ Test Incremental updates on cas and max cas values for keys """ self._load_ops(self.load_gen, 'update', mutations=20) self._load_ops(self.load_gen, 'replace', mutations=20) self._check_cas(self.load_gen, check_conflict_resolution=False) self.validate_test_failure() def test_cas_deletes(self): """ Test Incremental deletes on cas and max cas values for keys """ self._load_ops(self.load_gen, 'create') self._load_ops(self.load_gen, 'replace', mutations=20) self._check_cas(self.load_gen, check_conflict_resolution=False) self._load_ops(self.load_gen, 'delete') self._check_cas(self.load_gen, check_conflict_resolution=False, docs_deleted=True) self.validate_test_failure() ''' Test expiry on cas and max cas values for keys ''' def test_cas_expiry(self): self._load_ops(self.load_gen, 'create') self._load_ops(self.load_gen, 'expiry') self._check_cas(self.load_gen, check_conflict_resolution=False) self._check_expiry(self.load_gen) self.validate_test_failure() def test_cas_touch(self): """ Test touch on cas and max cas values for keys """ self.log.info('Starting test-touch') self._load_ops(self.load_gen, 'update', mutations=20) self._load_ops(self.load_gen, 'touch') self._check_cas(self.load_gen, check_conflict_resolution=False) self.validate_test_failure() def test_cas_getMeta(self): """ Test getMeta on cas and max cas values for keys """ self.log.info('Starting test-getMeta') self._load_ops(ops='set', mutations=20) self._check_cas(check_conflict_resolution=False) #self._load_ops(ops='add') self._load_ops(ops='replace',mutations=20) self._check_cas(check_conflict_resolution=False) self._load_ops(ops='delete') self._check_cas(check_conflict_resolution=False) def test_cas_setMeta_lower(self): self.log.info(' Starting test-getMeta') # set some kv self._load_ops(ops='set', mutations=1) #self._check_cas(check_conflict_resolution=False) k=0 while k<10: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) self.log.info('For key {0} the vbucket is {1}'.format( key,vbucket_id )) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) #mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) TEST_SEQNO = 123 TEST_CAS = k rc = mc_active.getMeta(key) cas = rc[4] + 1 self.log.info('Key {0} retrieved CAS is {1} and will set CAS to {2}'.format(key, rc[4], cas)) rev_seqno = rc[3] # do a set meta based on the existing CAS set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 123, cas) # check what get meta say rc = mc_active.getMeta(key) cas_post_meta = rc[4] self.log.info('Getmeta CAS is {0}'.format(cas_post_meta)) self.assertTrue( cas_post_meta == cas, 'Meta expected {0} actual {1}'.format( cas, cas_post_meta)) # and what stats says max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Max CAS for key {0} vbucket is {1}'.format( key, max_cas)) self.assertTrue(cas_post_meta >= max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta)) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas)) # do another mutation and compare mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) # and then mix in a set with meta set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, 225, max_cas+1) cas_post_meta = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta)) # and one more mutation for good measure mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) def test_cas_setMeta_higher(self): self.log.info(' Starting test-getMeta') self._load_ops(ops='set', mutations=20) self._check_cas(check_conflict_resolution=False) k=0 while k<10: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) #mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_1) #print '-'*100 TEST_SEQNO = 123 TEST_CAS = 9966180844186042368 cas = mc_active.getMeta(key)[4] #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, TEST_SEQNO, TEST_CAS, '123456789',vbucket_id, # add_extended_meta_data=True, conflict_resolution_mode=1) set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, TEST_CAS) cas_post_meta = mc_active.getMeta(key)[4] get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr2 {0}'.format(get_meta_2) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not equal it is {0}'.format(cas_post_meta)) self.assertTrue(max_cas > cas, '[ERROR]Max cas is not higher than original cas {0}'.format(cas)) mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, max_cas+1) #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, 125, TEST_CAS+1, '223456789',vbucket_id, # add_extended_meta_data=True, conflict_resolution_mode=1) cas_post_meta = mc_active.getMeta(key)[4] get_meta_3 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr3 {0}'.format(get_meta_3) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(cas_post_meta == max_cas, '[ERROR]Max cas is not lower it is higher than {0}'.format(cas_post_meta)) #self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas)) mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) ''' Test deleteMeta on cas and max cas values for keys ''' def test_cas_deleteMeta(self): self.log.info(' Starting test-deleteMeta') # load 20 kvs and check the CAS self._load_ops(ops='set', mutations=20) self.sleep(60) self._check_cas(check_conflict_resolution=False) k = 0 test_cas = 456 while k < 1: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) TEST_SEQNO = 123 test_cas = test_cas + 1 # get the meta data cas = mc_active.getMeta(key)[4] + 1 set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, cas) cas_post_meta = mc_active.getMeta(key)[4] # verify the observed CAS is as set max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas)) mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) # what is test cas for? Commenting out for now """ set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, TEST_SEQNO, test_cas) cas_post_meta = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(cas_post_meta < max_cas, '[ERROR]Max cas is not higher it is lower than {0}'.format(cas_post_meta)) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to original cas {0}'.format(cas)) """ # test the delete mc_active.set(key, 0, 0,json.dumps({'value':'value3'})) cas = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas == cas, '[ERROR]Max cas is not equal to cas {0}'.format(cas)) # self.log.info('Doing delete with meta, using a lower CAS value') get_meta_pre = mc_active.getMeta(key)[4] del_with_meta_resp = mc_active.del_with_meta(key, 0, 0, TEST_SEQNO, test_cas, test_cas+1) get_meta_post = mc_active.getMeta(key)[4] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.assertTrue(max_cas > test_cas+1, '[ERROR]Max cas {0} is not greater than delete cas {1}'.format(max_cas, test_cas)) ''' Testing skipping conflict resolution, whereby the last write wins, and it does neither cas CR nor rev id CR ''' def test_cas_skip_conflict_resolution(self): self.log.info(' Starting test_cas_skip_conflict_resolution ..') self._load_ops(ops='set', mutations=20) self._check_cas(check_conflict_resolution=False) k=0 #Check for first 20 keys while k<20: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) low_seq=12 cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] all = mc_active.getMeta(key) self.log.info('all meta data before set_meta_force {0}'.format(all)) self.log.info('Forcing conflict_resolution to allow insertion of lower Seq Number') lower_cas = int(cas)-1 #import pdb;pdb.set_trace() #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, low_seq, lower_cas, '123456789',vbucket_id) set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, low_seq, lower_cas, 3) cas_post_meta = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..') #print 'cas meta data after set_meta_force {0}'.format(cas_post_meta) #print 'all meta data after set_meta_force {0}'.format(all_post_meta) self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas)) self.assertTrue(pre_seq > post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) ''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR ''' def test_revid_conflict_resolution(self): self.log.info(' Starting test_cas_revid_conflict_resolution ..') self._load_ops(ops='set', mutations=20) self._check_cas(check_conflict_resolution=False) k=0 #Check for first 20 keys while k<20: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) new_seq=121 cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] all = mc_active.getMeta(key) self.log.info('all meta data before set_meta_force {0}'.format(all)) self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ') set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, cas, '123456789',vbucket_id, add_extended_meta_data=True, conflict_resolution_mode=1) cas_post_meta = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr2 {0}'.format(get_meta_2) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect No conflict_resolution to occur, and the last updated mutation to be the winner..') self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas)) self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) ''' Testing conflict resolution, where timeSync is enabled and cas is lower but higher revid, expect Higher Cas to Win ''' def test_cas_conflict_resolution(self): self.log.info(' Starting test_cas_conflict_resolution ..') self._load_ops(ops='set', mutations=20) self._check_cas(check_conflict_resolution=False) k=0 #Check for first 20 keys while k<20: key = "{0}{1}".format(self.prefix, k) k += 1 vbucket_id = self.client._get_vBucket_id(key) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) new_seq=121 cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] all = mc_active.getMeta(key) self.log.info('all meta data before set_meta_force {0}'.format(all)) lower_cas = int(cas)-100 self.log.info('Forcing lower rev-id to win with higher CAS value, instead of higher rev-id with Lower Cas ') #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, lower_cas, '123456789',vbucket_id) try: set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, lower_cas) except mc_bin_client.MemcachedError as e: # this is expected pass cas_post_meta = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect CAS conflict_resolution to occur, and the first mutation to be the winner..') self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) self.assertTrue(max_cas == cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas, cas)) #self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) ''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR and retains it after a restart server''' def test_restart_revid_conflict_resolution(self): self.log.info(' Starting test_restart_revid_conflict_resolution ..') self._load_ops(ops='set', mutations=20) k = 0 key = "{0}{1}".format(self.prefix, k) vbucket_id = self.client._get_vBucket_id(key) mc_active = self.client.memcached(key) mc_master = self.client.memcached_for_vbucket( vbucket_id ) mc_replica = self.client.memcached_for_replica_vbucket(vbucket_id) # set a key value = 'value0' client.memcached(key).set(key, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(key) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(key) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) new_seq=121 pre_cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) all = mc_active.getMeta(key) get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_1) self.log.info('all meta data before set_meta_force {0}'.format(all)) self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas)) self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ') try: set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id) except mc_bin_client.MemcachedError as e: # this is expected pass cas_post = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_2) max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..') self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) #self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas)) #self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) # Restart Nodes self._restart_server(self.servers[:]) # verify the CAS is good mc_active = client.memcached(key) cas_restart = mc_active.getMeta(key)[4] #print 'post cas {0}'.format(cas_post) get_meta_resp = mc_active.getMeta(key,request_extended_meta_data=False) #print 'post CAS {0}'.format(cas_post) #print 'post ext meta {0}'.format(get_meta_resp) self.assertTrue(pre_cas == cas_post, 'cas mismatch active: {0} replica {1}'.format(pre_cas, cas_post)) #self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') ''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR and retains it after a rebalance server''' def test_rebalance_revid_conflict_resolution(self): self.log.info(' Starting test_rebalance_revid_conflict_resolution ..') self._load_ops(ops='set', mutations=20) key = 'key1' value = 'value' client.memcached(key).set(key, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(key) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(key) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) new_seq=121 pre_cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) all = mc_active.getMeta(key) get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_1) self.log.info('all meta data before set_meta_force {0}'.format(all)) self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas)) self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ') #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id) set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas) cas_post = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_2) max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..') self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas)) self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) # remove that node self.log.info('Remove the node with active data') rebalance = self.cluster.async_rebalance(self.servers[-1:], [] ,[self.master]) rebalance.result() self.sleep(120) replica_CAS = mc_replica.getMeta(key)[4] get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False) #print 'replica CAS {0}'.format(replica_CAS) #print 'replica ext meta {0}'.format(get_meta_resp) # add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) rebalance.result() # verify the CAS is good mc_active = client.memcached(key) active_CAS = mc_active.getMeta(key)[4] print 'active cas {0}'.format(active_CAS) self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS)) #self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') ''' Testing revid based conflict resolution with timeSync enabled, where cas on either mutations match and it does rev id CR and retains it after a failover server''' def test_failover_revid_conflict_resolution(self): self.log.info(' Starting test_rebalance_revid_conflict_resolution ..') self._load_ops(ops='set', mutations=20) key = 'key1' value = 'value' self.client.memcached(key).set(key, 0, 0,json.dumps({'value':value})) vbucket_id = client._get_vBucket_id(key) #print 'vbucket_id is {0}'.format(vbucket_id) mc_active = client.memcached(key) mc_master = client.memcached_for_vbucket( vbucket_id ) mc_replica = client.memcached_for_replica_vbucket(vbucket_id) new_seq=121 pre_cas = mc_active.getMeta(key)[4] pre_seq = mc_active.getMeta(key)[3] pre_max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) all = mc_active.getMeta(key) get_meta_1 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_1) self.log.info('all meta data before set_meta_force {0}'.format(all)) self.log.info('max_cas before set_meta_force {0}'.format(pre_max_cas)) self.log.info('Forcing conflict_resolution to rev-id by matching inserting cas ') #set_with_meta_resp = mc_active.set_with_meta(key, 0, 0, new_seq, pre_cas, '123456789',vbucket_id) set_with_meta_resp = mc_active.setWithMeta(key, '123456789', 0, 0, new_seq, pre_cas) cas_post = mc_active.getMeta(key)[4] all_post_meta = mc_active.getMeta(key) post_seq = mc_active.getMeta(key)[3] get_meta_2 = mc_active.getMeta(key,request_extended_meta_data=False) #print 'cr {0}'.format(get_meta_2) max_cas_post = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(key)) + ':max_cas'] ) self.log.info('Expect RevId conflict_resolution to occur, and the last updated mutation to be the winner..') self.log.info('all meta data after set_meta_force {0}'.format(all_post_meta)) self.assertTrue(max_cas_post == pre_cas, '[ERROR]Max cas {0} is not equal to original cas {1}'.format(max_cas_post, pre_cas)) self.assertTrue(pre_seq < post_seq, '[ERROR]Pre rev id {0} is not greater than post rev id {1}'.format(pre_seq, post_seq)) # failover that node self.log.info('Failing over node with active data {0}'.format(self.master)) self.cluster.failover(self.servers, [self.master]) self.log.info('Remove the node with active data {0}'.format(self.master)) rebalance = self.cluster.async_rebalance(self.servers[:], [] ,[self.master]) rebalance.result() self.sleep(120) replica_CAS = mc_replica.getMeta(key)[4] get_meta_resp = mc_replica.getMeta(key,request_extended_meta_data=False) #print 'replica CAS {0}'.format(replica_CAS) #print 'replica ext meta {0}'.format(get_meta_resp) # add the node back self.log.info('Add the node back, the max_cas should be healed') rebalance = self.cluster.async_rebalance(self.servers[-1:], [self.master], []) rebalance.result() # verify the CAS is good mc_active = client.memcached(key) active_CAS = mc_active.getMeta(key)[4] #print 'active cas {0}'.format(active_CAS) self.assertTrue(replica_CAS == active_CAS, 'cas mismatch active: {0} replica {1}'.format(active_CAS,replica_CAS)) #self.assertTrue( get_meta_resp[5] == 1, msg='Metadata indicate conflict resolution is not set') ''' Test getMeta on cas and max cas values for empty vbucket ''' def test_cas_getMeta_empty_vBucket(self): self.log.info(' Starting test-getMeta') self._load_ops(ops='set', mutations=20) k=0 all_keys = [] while k<10: k+=1 key = "{0}{1}".format(self.prefix, k) all_keys.append(key) vbucket_ids = self.client._get_vBucket_ids(all_keys) print 'bucket_ids' for v in vbucket_ids: print v print 'done' i=1111 if i not in vbucket_ids and i <= 1023: vb_non_existing=i elif i>1023: i +=1 else: self.log.info('ERROR generating empty vbucket id') vb_non_existing=vbucket_ids.pop() print 'nominated vb_nonexisting is {0}'.format(vb_non_existing) mc_active = self.client.memcached(all_keys[0]) #Taking a temp connection to the mc. #max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(vb_non_existing) + ':max_cas'] ) max_cas = int( mc_active.stats('vbucket-details')['vb_' + str(self.client._get_vBucket_id(all_keys[0])) + ':max_cas'] ) self.assertTrue( max_cas != 0, msg='[ERROR] Max cas is non-zero') def test_meta_backup(self): """ Test addMeta on cas and max cas values for keys """ self.log.info('Starting test-getMeta') self._load_ops(ops='set', mutations=20) # Do the backup on the bucket self.shell = RemoteMachineShellConnection(self.cluster.master) self.buckets = self.bucket_util.buckets self.couchbase_login_info = "%s:%s" % (self.input.membase_settings.rest_username, self.input.membase_settings.rest_password) self.backup_location = "/tmp/backup" self.command_options = self.input.param("command_options", '') try: shell = RemoteMachineShellConnection(self.cluster.master) self.shell.execute_cluster_backup( self.couchbase_login_info, self.backup_location, self.command_options) self.sleep(5, "Wait before restore backup") shell.restore_backupFile(self.couchbase_login_info, self.backup_location, [bucket.name for bucket in self.buckets]) self.log.info("Done with restore") finally: self._check_cas(check_conflict_resolution=False) def _check_cas(self, load_gen, check_conflict_resolution=False, time_sync=None, docs_deleted=False): """ Common function to verify the expected values on cas """ load_gen = copy.deepcopy(load_gen) self.log.info('Verifying CAS and max-cas for the keys') while load_gen.has_next(): key, value = load_gen.next() cas = self.client.crud("read", key)["cas"] if docs_deleted: if cas != 0: self.log_failure("Max CAS mismatch. %s != 0" % cas) else: max_cas = None vb_for_key = self.bucket_util.get_vbucket_num_for_key(key) for _, data in self.node_data.items(): if vb_for_key in data["active"]: vb_stat = data["cb_stat"].vbucket_details( self.bucket.name) max_cas = long(vb_stat[str(vb_for_key)]["max_cas"]) break if cas != max_cas: self.log_failure("Max CAS mismatch. %s != %s" % (cas, max_cas)) if check_conflict_resolution: get_meta_resp = mc_active.getMeta( key, request_extended_meta_data=False) if time_sync == 'enabledWithoutDrift': if get_meta_resp[5] != 1: self.log_failure("Metadata indicates " "conflict resolution is not set") elif time_sync == 'disabled': if get_meta_resp[5] != 0: self.log_failure("Metadata indicates " "conflict resolution is set") def _load_ops(self, load_gen, op_type, mutations=1): """ Common function to add set delete etc operations on the bucket """ mutations = 1 self.log.info("Performing %s for %s times" % (op_type, mutations)) exp = 0 if op_type == "expiry": op_type = "touch" exp = self.expire_time for _ in range(mutations): tasks_info = self.bucket_util.sync_load_all_buckets( self.cluster, load_gen, op_type, exp=exp, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, batch_size=self.num_items, sdk_client_pool=self.sdk_client_pool) for task, _ in tasks_info.items(): if task.fail: self.log_failure("Failures observed during %s" % op_type) def _check_expiry(self, load_gen): """ Check if num_items are expired as expected """ self.sleep(self.expire_time, "Wait for docs to expire") while load_gen.has_next(): key, value = load_gen.next() vb_for_key = self.bucket_util.get_vbucket_num_for_key(key) cas = None for _, data in self.node_data.items(): if vb_for_key in data["active"]: vb_stat = data["cb_stat"].vbucket_details(self.bucket.name) cas = long(vb_stat[str(vb_for_key)]["max_cas"]) break replace_result = self.client.crud( "replace", key, value, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout=self.sdk_timeout, cas=cas) if replace_result["status"] is True: self.log_failure("Replace on %s succeeded using old CAS %s" % (key, cas)) if SDKException.DocumentNotFoundException \ not in str(replace_result["error"]): self.log_failure("Invalid exception for %s: %s" % (key, replace_result))
def test_scenario(bucket, doc_ops, with_sync_write_val=None): # Set crud_batch_size crud_batch_size = 4 simulate_error = CouchbaseError.STOP_MEMCACHED # Fetch target_vbs for CRUDs node_vb_info = self.vbs_in_node target_vbuckets = node_vb_info[target_nodes[0]]["replica"] if len(target_nodes) > 1: index = 1 while index < len(target_nodes): target_vbuckets = list( set(target_vbuckets).intersection( set(node_vb_info[target_nodes[index]]["replica"])) ) index += 1 # Variable to hold one of the doc_generator objects gen_loader_1 = None gen_loader_2 = None # Initialize doc_generators to use for testing self.log.info("Creating doc_generators") gen_create = doc_generator( self.key, self.num_items, crud_batch_size, vbuckets=self.cluster.vbuckets, target_vbucket=target_vbuckets) gen_update = doc_generator( self.key, 0, crud_batch_size, vbuckets=self.cluster.vbuckets, target_vbucket=target_vbuckets, mutate=1) gen_delete = doc_generator( self.key, 0, crud_batch_size, vbuckets=self.cluster.vbuckets, target_vbucket=target_vbuckets) self.log.info("Done creating doc_generators") # Start CRUD operation based on the given 'doc_op' type if doc_ops[0] == "create": self.num_items += crud_batch_size gen_loader_1 = gen_create elif doc_ops[0] in ["update", "replace", "touch"]: gen_loader_1 = gen_update elif doc_ops[0] == "delete": gen_loader_1 = gen_delete self.num_items -= crud_batch_size if doc_ops[1] == "create": gen_loader_2 = gen_create elif doc_ops[1] in ["update", "replace", "touch"]: gen_loader_2 = gen_update elif doc_ops[1] == "delete": gen_loader_2 = gen_delete # Load required docs for doc_op_1 in case of type != create if doc_op[2] == "load_initial_docs": doc_loading_task = self.task.async_load_gen_docs( self.cluster, bucket, gen_loader_1, "create", 0, batch_size=crud_batch_size, process_concurrency=1, timeout_secs=10, print_ops_rate=False, sdk_client_pool=self.sdk_client_pool) self.task_manager.get_task_result(doc_loading_task) if doc_loading_task.fail: self.log_failure("Failure while loading initial docs") self.summary.add_step("Create docs for %s" % doc_op[0]) verification_dict["ops_create"] += crud_batch_size verification_dict["sync_write_committed_count"] \ += crud_batch_size # Initialize tasks and store the task objects doc_loader_task = self.task.async_load_gen_docs( self.cluster, bucket, gen_loader_1, doc_ops[0], 0, batch_size=crud_batch_size, process_concurrency=8, timeout_secs=60, print_ops_rate=False, start_task=False, sdk_client_pool=self.sdk_client_pool) # SDK client for performing individual ops client = SDKClient([self.cluster.master], bucket) # Perform specified action for node in target_nodes: error_sim = CouchbaseError(self.log, self.vbs_in_node[node]["shell"]) error_sim.create(simulate_error, bucket_name=bucket.name) self.sleep(5, "Wait for error simulation to take effect") self.task_manager.add_new_task(doc_loader_task) self.sleep(5, "Wait for task_1 CRUDs to reach server") # Perform specified CRUD operation on sync_write docs tem_gen = deepcopy(gen_loader_2) while tem_gen.has_next(): key, value = tem_gen.next() for retry_strategy in [ SDKConstants.RetryStrategy.FAIL_FAST, SDKConstants.RetryStrategy.BEST_EFFORT]: if with_sync_write_val: fail = client.crud(doc_ops[1], key, value=value, exp=0, durability=with_sync_write_val, timeout=3, time_unit="seconds", sdk_retry_strategy=retry_strategy) else: fail = client.crud(doc_ops[1], key, value=value, exp=0, timeout=3, time_unit="seconds", sdk_retry_strategy=retry_strategy) expected_exception = SDKException.AmbiguousTimeoutException retry_reason = \ SDKException.RetryReason.KV_SYNC_WRITE_IN_PROGRESS if retry_strategy == SDKConstants.RetryStrategy.FAIL_FAST: expected_exception = \ SDKException.RequestCanceledException retry_reason = \ SDKException.RetryReason \ .KV_SYNC_WRITE_IN_PROGRESS_NO_MORE_RETRIES # Validate the returned error from the SDK if expected_exception not in str(fail["error"]): self.log_failure("Invalid exception for {0}: {1}" .format(key, fail["error"])) if retry_reason not in str(fail["error"]): self.log_failure("Invalid retry reason for {0}: {1}" .format(key, fail["error"])) # Try reading the value in SyncWrite in-progress state fail = client.crud("read", key) if doc_ops[0] == "create": # Expected KeyNotFound in case of CREATE operation if fail["status"] is True: self.log_failure( "%s returned value during SyncWrite state: %s" % (key, fail)) else: # Expects prev value in case of other operations if fail["status"] is False: self.log_failure( "Key %s read failed for previous value: %s" % (key, fail)) # Revert the introduced error condition for node in target_nodes: error_sim = CouchbaseError(self.log, self.vbs_in_node[node]["shell"]) error_sim.revert(simulate_error, bucket_name=bucket.name) # Wait for doc_loader_task to complete self.task.jython_task_manager.get_task_result(doc_loader_task) verification_dict["ops_%s" % doc_op[0]] += crud_batch_size verification_dict["sync_write_committed_count"] \ += crud_batch_size # Disconnect the client client.close()
def test_MB_41944(self): num_index = self.input.param("num_index", 1) # Create doc_gen for loading doc_gen = doc_generator(self.key, 0, 1) # Get key for delete op and reset the gen key, v = doc_gen.next() doc_gen.reset() # Open SDK client connection client = SDKClient([self.cluster.master], self.bucket_util.buckets[0]) query = list() query.append("CREATE PRIMARY INDEX index_0 on %s USING GSI" % self.bucket_util.buckets[0].name) if num_index == 2: query.append("CREATE INDEX index_1 on %s(name,age) " "WHERE mutated=0 USING GSI" % self.bucket_util.buckets[0].name) # Create primary index on the bucket for q in query: client.cluster.query(q) # Wait for index to become online` for index, _ in enumerate(query): query = "SELECT state FROM system:indexes WHERE name='index_%s'" \ % index index = 0 state = None while index < 30: state = client.cluster.query(query) \ .rowsAsObject()[0].get("state") if state == "online": break self.sleep(1) if state != "online": self.log_failure("Index 'index_%s' not yet online" % index) # Start transaction to create the doc trans_task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, doc_gen, DocLoading.Bucket.DocOps.CREATE) self.task_manager.get_task_result(trans_task) # Perform sub_doc operation on same key _, fail = client.crud(DocLoading.Bucket.SubDocOps.INSERT, key=key, value=["_sysxattr", "sysxattr-payload"], xattr=True) if fail: self.log_failure("Subdoc insert failed: %s" % fail) else: self.log.info("Subdoc insert success") # Delete the created doc result = client.crud(DocLoading.Bucket.DocOps.DELETE, key) if result["status"] is False: self.log_failure("Doc delete failed: %s" % result["error"]) else: self.log.info("Document deleted") # Re-insert same doc through transaction trans_task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, doc_gen, DocLoading.Bucket.DocOps.CREATE) self.task_manager.get_task_result(trans_task) # Close SDK Client connection client.close() self.validate_test_failure()
def validate_durability_for_bucket(index, bucket): # Fetch all the failed docs from the tasks failed_docs = dict() failed_docs["create"] = self.loadgen_tasks[0].fail failed_docs["update"] = self.loadgen_tasks[1].fail failed_docs["delete"] = self.loadgen_tasks[2].fail try: failed_docs["read"] = self.loadgen_tasks[3].fail except: pass # Detect whether durability is going to succeed or not # based on the fail_over type and num_server to fail durability_success = self.durability_helper.durability_succeeds( bucket.name, self.cluster.master, induced_error=self.failover_action, failed_nodes=self.server_to_fail) no_error_ops = ["read"] error_expected_ops = [] if durability_success: no_error_ops += ["create", "update", "delete"] # If durability is expected to pass, # we should not see any failure in create/delete/update self.assertTrue( len(failed_docs["create"]) == 0, msg="Unexpected exception during 'create' {0}".format( failed_docs["create"])) self.assertTrue( len(failed_docs["update"]) == 0, msg="Unexpected exception during 'update' {0}".format( failed_docs["update"])) self.assertTrue( len(failed_docs["delete"]) == 0, msg="Unexpected exception during 'delete' {0}".format( failed_docs["delete"])) else: error_expected_ops += ["create", "update", "delete"] # If durability is expected to fail, # we should see atleast one failure in create/delete/update self.assertTrue(len(failed_docs["create"]) != 0, msg="No exceptions in 'create' operation") self.assertTrue(len(failed_docs["update"]) != 0, msg="No exceptions in 'update' operation") self.assertTrue(len(failed_docs["delete"]) != 0, msg="No exceptions in 'delete' operation") # Fail the cases, if the errors seen in the unexpected op_type for op_type in no_error_ops: # Verification of CRUD failures self.assertTrue(len(failed_docs[op_type] == 0), msg="Read operation failed: {0}".format( failed_docs["read"])) # Create SDK client for doc CRUD retry client = SDKClient(RestConnection(self.cluster.master), bucket) # Loop over all failed docs as per their op_types for op_type in error_expected_ops: for failed_doc in failed_docs[op_type]: # Failures should be observed only in the vbuckets # placed in the failure nodes key = failed_doc["key"] vb_num = self.bucket_util.get_vbucket_num_for_key(key) msg = "Key '{0}' not found in failed nodes' vbucket list" \ .format(key) err_msg = "Invalid exception {0} for vb failure" key_in_active_vb = vb_num in self.active_vb_in_failover_nodes key_in_replica_vb = vb_num in self.replica_vb_in_failover_nodes # Checks whether the error happened on target vbucket only self.assetTrue(key_in_active_vb or key_in_replica_vb, msg=msg) # Validate the received exception for CRUD failure if key_in_active_vb: result = "ambiguous abort" in failed_doc["error"] self.assertTrue(result, msg=err_msg.format( failed_doc["error"], "active")) elif key_in_replica_vb: result = "ambiguous abort" in failed_doc["error"] self.assertTrue(result, msg=err_msg.format( failed_doc["error"], "active")) # Retry failed docs to verify it succeeds with no failure self.durability_helper.retry_with_no_error( client, failed_docs[op_type], op_type, timeout=self.sdk_timeout) # Closing the SDK client client.close()
def test_sub_doc_op_with_bucket_level_durability(self): """ Create Buckets with durability_levels set and perform Sub_doc CRUDs from client without durability settings and validate the ops to make sure respective durability is honored """ key, value = doc_generator("test_key", 0, 1).next() sub_doc_key = "sub_doc_key" sub_doc_vals = ["val_1", "val_2", "val_3", "val_4", "val_5"] for d_level in self.get_supported_durability_for_bucket(): # Avoid creating bucket with durability=None if d_level == Bucket.DurabilityLevel.NONE: continue step_desc = "Creating %s bucket with level '%s'" \ % (self.bucket_type, d_level) verification_dict = self.get_cb_stat_verification_dict() self.log.info(step_desc) # Object to support performing CRUDs and create Bucket bucket_dict = self.get_bucket_dict(self.bucket_type, d_level) bucket_obj = Bucket(bucket_dict) self.bucket_util.create_bucket(self.cluster, bucket_obj, wait_for_warmup=True) self.summary.add_step(step_desc) # SDK client to perform sub_doc ops client = SDKClient([self.cluster.master], bucket_obj) result = client.crud("create", key, value) verification_dict["ops_create"] += 1 verification_dict["sync_write_committed_count"] += 1 if result["status"] is False: self.log_failure("Doc insert failed for key: %s" % key) # Perform sub_doc CRUD for sub_doc_op in ["subdoc_insert", "subdoc_upsert", "subdoc_replace"]: sub_doc_val = choice(sub_doc_vals) _, fail = client.crud(sub_doc_op, key, [sub_doc_key, sub_doc_val]) if fail: self.log_failure("%s failure. Key %s, sub_doc (%s, %s): %s" % (sub_doc_op, key, sub_doc_key, sub_doc_val, result)) else: verification_dict["ops_update"] += 1 verification_dict["sync_write_committed_count"] += 1 success, fail = client.crud("subdoc_read", key, sub_doc_key) if fail or str(success[key]["value"].get(0)) != sub_doc_val: self.log_failure("%s failed. Expected: %s, Actual: %s" % (sub_doc_op, sub_doc_val, success[key]["value"].get(0))) self.summary.add_step("%s for key %s" % (sub_doc_op, key)) # Subdoc_delete and verify sub_doc_op = "subdoc_delete" _, fail = client.crud(sub_doc_op, key, sub_doc_key) if fail: self.log_failure("%s failure. Key %s, sub_doc (%s, %s): %s" % (sub_doc_op, key, sub_doc_key, sub_doc_val, result)) verification_dict["ops_update"] += 1 verification_dict["sync_write_committed_count"] += 1 _, fail = client.crud(sub_doc_op, key, sub_doc_key) if SDKException.PathNotFoundException \ not in str(fail[key]["error"]): self.log_failure("Invalid error after sub_doc_delete") self.summary.add_step("%s for key %s" % (sub_doc_op, key)) # Validate doc_count self.bucket_util._wait_for_stats_all_buckets(self.cluster, self.cluster.buckets) self.bucket_util.verify_stats_all_buckets(self.cluster, 1) # Cbstats vbucket-details validation self.cb_stat_verify(verification_dict) # Close SDK client client.close() # Delete the bucket on server self.bucket_util.delete_bucket(self.cluster, bucket_obj) self.summary.add_step("Delete %s bucket" % self.bucket_type)
def test_staged_doc_read(self): self.verify = self.input.param("verify", True) bucket = self.cluster.buckets[0] expected_exception = SDKException.DocumentNotFoundException # Create SDK client for transactions client = SDKClient([self.cluster.master], bucket) if self.doc_op in ["update", "delete"]: for doc in self.docs: result = client.crud("create", doc.getT1(), doc.getT2(), durability=self.durability_level, timeout=60) if result["status"] is False: self.log_failure("Key %s create failed: %s" % (doc.getT1(), result)) break expected_exception = None read_thread = Thread( target=self.__perform_read_on_doc_keys, args=(bucket, self.keys), kwargs=dict(expected_exception=expected_exception)) read_thread.start() # Transaction load exception = self.__run_mock_test(client, self.doc_op) if SDKException.TransactionExpired not in str(exception): self.log_failure("Expected exception not found") self.log.info("Terminating reader thread") self.stop_thread = True read_thread.join() self.transaction_fail_count = 2 exception = self.__run_mock_test(client, self.doc_op) if exception: self.log_failure(exception) # verify the values for key in self.keys: result = client.read(key) if "Remove" in self.operation \ or self.transaction_commit is False \ or self.verify is False: if result['status']: actual_val = client.translate_to_json_object( result['value']) self.log.info("Actual value for key %s is %s" % (key, actual_val)) self.log_failure( "Key '%s' should be deleted but present in the bucket" % key) else: actual_val = client.translate_to_json_object(result['value']) if self.doc_op == "update": self.content.put("mutated", 1) elif self.doc_op == "delete": self.content.removeKey("value") if self.content != actual_val: self.log.info("Key %s Actual: %s, Expected: %s" % (key, actual_val, self.content)) self.log_failure("Mismatch in doc content") # Close SDK client client.close() if self.read_failed[self.cluster.buckets[0]] is True: self.log_failure("Failure in read thread for bucket: %s" % self.cluster.buckets[0].name) self.validate_test_failure()
def test_staged_doc_query_from_index(self): self.verify = self.input.param("verify", True) expected_val = dict() bucket = self.cluster.buckets[0] # Create SDK client for transactions client = SDKClient([self.cluster.master], bucket) if self.doc_op in ["update", "delete"]: for doc in self.docs: result = client.crud("create", doc.getT1(), doc.getT2(), durability=self.durability_level, timeout=60) if result["status"] is False: self.log_failure("Key %s create failed: %s" % (doc.getT1(), result)) break expected_val[doc.getT1()] = json.loads(str(doc.getT2())) # Create primary Index on all buckets for t_bucket in self.cluster.buckets: q_result = client.cluster.query("CREATE PRIMARY INDEX ON `%s`" % t_bucket.name) if q_result.metaData().status().toString() != "SUCCESS": client.close() self.fail("Create primary index failed for bucket %s" % t_bucket.name) self.sleep(10, "Wait for primary indexes to get warmed up") query_thread = Thread(target=self.__perform_query_on_doc_keys, args=(bucket, self.keys, expected_val)) query_thread.start() # Transaction load exception = self.__run_mock_test(client, self.doc_op) if SDKException.TransactionExpired not in str(exception): self.log_failure("Expected exception not found") self.log.info("Terminating query thread") self.stop_thread = True query_thread.join() self.transaction_fail_count = 2 exception = self.__run_mock_test(client, self.doc_op) if exception: self.log_failure(exception) # verify the values for key in self.keys: result = client.read(key) if "Remove" in self.operation \ or self.transaction_commit is False \ or self.verify is False: if result['status']: actual_val = client.translate_to_json_object( result['value']) self.log.info("Actual value for key %s is %s" % (key, actual_val)) self.log_failure( "Key '%s' should be deleted but present in the bucket" % key) else: actual_val = client.translate_to_json_object(result['value']) if self.doc_op == "update": self.content.put("mutated", 1) elif self.doc_op == "delete": self.content.removeKey("value") if self.content != actual_val: self.log.info("Key %s Actual: %s, Expected: %s" % (key, actual_val, self.content)) self.log_failure("Mismatch in doc content") # Close SDK client client.close() if self.read_failed[self.cluster.buckets[0]] is True: self.log_failure("Failure in read thread for bucket: %s" % self.cluster.buckets[0].name) self.validate_test_failure()
class MagmaExpiryTests(MagmaBaseTest): def setUp(self): super(MagmaExpiryTests, self).setUp() self.gen_delete = None self.gen_create = None self.gen_update = None self.gen_expiry = None self.exp_pager_stime = self.input.param("exp_pager_stime", 10) self.iterations = self.input.param("iterations", 5) self.expiry_perc = self.input.param("expiry_perc", 100) self.items = self.num_items def load_bucket(self): tasks = dict() for collection in self.collections: self.generate_docs(doc_ops="create", target_vbucket=None) tasks.update(self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions, skip_read_on_error=False, scope=self.scope_name, collection=collection, monitor_stats=self.monitor_stats)) for task in tasks: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tasks, self.cluster) self.bucket_util.log_doc_ops_task_failures(tasks) self.bucket_util._wait_for_stats_all_buckets(timeout=1200) def tearDown(self): super(MagmaExpiryTests, self).tearDown() def test_read_expired_replica(self): result = True self.gen_create = doc_generator( self.key, 0, 10, doc_size=20, doc_type=self.doc_type, key_size=self.key_size) tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", exp=10, batch_size=10, process_concurrency=1, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, ) self.task.jython_task_manager.get_task_result(tasks_info.keys()[0]) self.sleep(20) self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) for i in range(10): key = (self.key + "-" + str(i).zfill(self.key_size-len(self.key))) try: getReplicaResult = self.client.collection.getAnyReplica( key, GetAnyReplicaOptions.getAnyReplicaOptions()) if getReplicaResult: result = False try: self.log.info("Able to retreive: %s" % {"key": key, "value": getReplicaResult.contentAsObject(), "cas": getReplicaResult.cas()}) except Exception as e: print str(e) except DocumentUnretrievableException as e: pass if len(self.client.get_from_all_replicas(key)) > 0: result = False self.client.close() self.assertTrue(result, "SDK is able to retrieve expired documents") def test_expiry(self): ''' Test Focus: Expire items (for n iterations) and verify tombstone count after expiry pagers time and meta data purge interval Steps: --- Expired already created items/create expiry load --- Wait for docs to expire --- Check for tombstones count --- Verify doc count --- Set meta data purge age and verify tomb stone count --- Repeat above steps n times ''' self.log.info("test_expiry starts") self.expiry_start = 0 self.expiry_end = self.num_items self.doc_ops = "expiry" for it in range(self.iterations): self.log.info("Iteration {}".format(it)) self.expiry_perc = self.input.param("expiry_perc", 100) self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1)*(it+1) self.log.info("Iterations - {}, expected_ts_count - {}".format(it, expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) # self.bucket_util.set_metadata_purge_interval(str(self.meta_purge_interval), # buckets=self.buckets) # self.sleep(self.meta_purge_interval*60*60*2/0.04, "Wait for Metadata Purge Interval to drop \ # tomb-stones from storage") self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) #Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) self.assertTrue(self.vbuckets * (self.num_replicas+1)>=ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_create_expire_same_items(self): ''' Test Focus: Create and expire n items Steps: --- Create items == num_items (init_loading will be set to False) --- Check Disk Usage after creates --- Expire all the items --- Check for tombstones count --- Check Disk Usage --- Repeat above steps n times ''' self.log.info("test_create_expire_same_items starts") self.create_start = 0 self.create_end = self.num_items self.expiry_start = 0 self.expiry_end = self.num_items #self.create_perc = 100 #self.expiry_perc = 100 for _iter in range(self.iterations): self.maxttl = random.randint(5, 20) self.log.info("Test Iteration: {}".format(_iter)) # Create items which are expired self.generate_docs(doc_ops="create", create_start=self.create_start, create_end=self.create_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="create") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after creates {}".format(disk_usage)) size_before = disk_usage[0] self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="expiry") self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) # Space amplification check msg_stats = "Fragmentation value for {} stats exceeds\ the configured value" result = self.check_fragmentation_using_magma_stats(self.buckets[0], self.cluster.nodes_in_cluster) self.assertIs(result, True, msg_stats.format("magma")) result = self.check_fragmentation_using_bucket_stats( self.buckets[0], self.cluster.nodes_in_cluster) self.assertIs(result, True, msg_stats.format("KV")) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after expiry {}".format(disk_usage)) size_after = disk_usage[0] self.assertTrue(size_after < size_before * 0.8, "Data Size before(%s) and after expiry(%s)" .format(size_before, size_after)) # Metadata Purge Interva self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) self.sleep(60, "wait after compaction") disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) disk_usage_after_compaction = disk_usage[0] self.log.info("Iteration--{}, disk usage after compaction--{}". format(_iter, disk_usage[0])) self.assertTrue(disk_usage_after_compaction < 500, "Disk size after compaction exceeds 500MB") def test_expiry_no_wait_update(self): ''' Test Focus: Upsert and expire n items Steps: --- Upsert items == num_items (init_loading will be in magma base) --- Expire all the load -- Wait for docs to expire and conver to tomb stones --- Check Disk Usage after expiry --- Check for tombstones count --- Set meta data purge age and verify tomb stone count --- Check Disk Usage --- Repeat above steps n times ''' self.log.info(" test_expiry_no_wait_update starts") self.update_start = 0 self.update_end = self.num_items self.expiry_start = 0 self.expiry_end = self.num_items self.update_perc = 100 self.expiry_perc = 100 for _iter in range(self.iterations): self.log.info("Iteration--{}".format(_iter)) self.generate_docs(doc_ops="update", update_start=self.update_start, update_end=self.update_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="update") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.debug("Disk usage after updates {}".format(disk_usage)) size_before = disk_usage[0] self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="expiry") self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1)*(_iter+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after expiry {}".format(disk_usage)) size_after = disk_usage[0] self.assertTrue(size_after < self.disk_usage[self.disk_usage.keys()[0]] * 0.6, "Data Size before(%s) and after expiry(%s)" .format(self.disk_usage[self.disk_usage.keys()[0]], size_after)) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after compaction {}".format(disk_usage)) size_after_compaction = disk_usage[0] self.log.info("disk usage after compaction {}".format(size_after_compaction)) self.sleep(300, "sleeping after test") # below assert is only applicable if we expire all the items self.assertTrue(size_after_compaction < 500, "size after compaction shouldn't be more than 500") def test_docs_expired_wait_for_magma_purge(self): pass def test_expiry_disk_full(self): self.expiry_perc = self.input.param("expiry_perc", 100) self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") def _get_disk_usage_in_MB(remote_client, path): disk_info = remote_client.get_disk_info(in_MB=True, path=path) disk_space = disk_info[1].split()[-3][:-1] return disk_space # Fill up the disk remote_client = RemoteMachineShellConnection(self.cluster.master) du = int(_get_disk_usage_in_MB(remote_client, self.cluster.master.data_path)) - 50 _file = os.path.join(self.cluster.master.data_path, "full_disk_") cmd = "dd if=/dev/zero of={0}{1} bs=1024M count=1" while int(du) > 0: cmd = cmd.format(_file, str(du) + "MB_" + str(time.time())) output, error = remote_client.execute_command(cmd, use_channel=True) remote_client.log_command_output(output, error) du -= 1024 if du < 1024: cmd = "dd if=/dev/zero of={0}{1} bs=" + str(du) + "M count=1" # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) # Metadata Purge Interval self.meta_purge_interval = 60 self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # free up the disk output, error = remote_client.execute_command("rm -rf full_disk*", use_channel=True) # Wait for expiry pager to insert tombstones again self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage after disk is available ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) # Metadata Purge Interval after disk space is available self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction() ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) def test_random_expiry(self): ''' Test Focus: Expire items (Items will have different ttl value) and verify tombstone count after expiry pagers time and meta data purge interval Steps: --- Expired already created items/create expiry load (Items will have different ttl values) --- Wait for docs to expire --- Check for tombstones count --- Verify doc count --- Set meta data purge age and verify tomb stone count --- Repeat above steps n times ''' self.random_exp = True self.doc_ops = "expiry" self.expiry_perc = self.input.param("expiry_perc", 100) self.generate_docs() _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(self.num_items) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) self.assertTrue(self.vbuckets * (self.num_replicas+1)>=ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_expire_read_validate_meta(self): self.expiry_perc = self.input.param("expiry_perc", 100) self.doc_ops = "expiry" self.bucket_util._expiry_pager(216000) self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") self.sigkill_memcached() self.bucket_util._expiry_pager(216000) # Read all the docs to ensure they get converted to tombstones self.generate_docs(doc_ops="read", read_start=self.expiry_start, read_end=self.expiry_end) self.gen_delete = copy.deepcopy(self.gen_read) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="delete") self.sleep(180, "wait after get ops") #data_validation = self.task.async_validate_docs( # self.cluster, self.bucket_util.buckets[0], # self.gen_read, "delete", 0, # batch_size=self.batch_size, # process_concurrency=self.process_concurrency, # pause_secs=5, timeout_secs=self.sdk_timeout) #self.task.jython_task_manager.get_task_result(data_validation) # All docs converted to tomb-stone # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(expected_ts_count, ts)) def test_wait_for_expiry_read_repeat(self): for _iter in range(self.iterations): self.maxttl = random.randint(20, 60) self.log.info("Test Iteration: {}".format(_iter)) self.test_expire_read_validate_meta() def test_expiry_full_compaction(self): self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(21600) self.log.info("Starting compaction for each bucket to add tombstones") self.run_compaction(compaction_iterations=1) self.sleep(300, "sleep after triggering compaction") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after compaction: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count after compaction is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(expected_ts_count, ts)) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.log.info("Starting compaction for each bucket") self.run_compaction(compaction_iterations=1) self.sleep(300, "sleep after triggering compaction, to drop tombstones") # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.log.info("Expected {}".format(self.vbuckets * (self.num_replicas+1))) self.assertTrue(self.vbuckets * (self.num_replicas+1) >= ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_drop_collection_expired_items(self): self.log.info("test_drop_collection_expired_items starts") tasks = dict() for collection in self.collections[::2]: self.generate_docs(doc_ops="expiry") task = self.loadgen_docs(scope=self.scope_name, collection=collection, _sync=False, doc_ops="expiry") tasks.update(task.items()) for task in tasks: self.task_manager.get_task_result(task) self.sleep(self.maxttl, "Wait for docs to expire") # Convert to tomb-stones tasks_info = dict() self.gen_delete = copy.deepcopy(self.gen_expiry) for collection in self.collections[::2]: temp_tasks_info = self.bucket_util._async_validate_docs( self.cluster, self.gen_delete, "delete", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout, scope=self.scope_name, collection=collection, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(temp_tasks_info.items()) for task in tasks_info: self.task_manager.get_task_result(task) for collection in self.collections[::2]: self.bucket_util.drop_collection(self.cluster.master, self.buckets[0], scope_name=self.scope_name, collection_name=collection) self.buckets[0].scopes[self.scope_name].collections.pop(collection) self.collections.remove(collection) self.sleep(180, "sleep after dropping collections") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("tombstone count is {}".format(ts)) self.assertEqual(ts, 0, "Tombstones found after collections(expired items) drop.") def test_drop_collection_during_tombstone_creation(self): scope_name, collections = self.scope_name, self.collections self.load_bucket() tasks = [] for collection in collections[::2]: self.generate_docs(doc_ops="expiry") tasks.append(self.loadgen_docs(scope=scope_name, collection=collection, _sync=False)) # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") for task in tasks: self.task_manager.get_task_result(task) for collection in collections[::2]: self.bucket_util.drop_collection(self.cluster.master, self.buckets[0], scope_name=scope_name, collection_name=collection) self.buckets[0].scopes[scope_name].collections.pop(collection) collections.remove(collection) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.assertEqual(ts, 0, "Tombstones found after collections(expired items) drop.") def test_failover_expired_items_in_vB(self): self.maxttl = 120 self.doc_ops = "expiry" self.expiry_perc = self.input.param("expiry_perc", 100) shell_conn = RemoteMachineShellConnection(self.cluster.nodes_in_cluster[-1]) cbstats = Cbstats(shell_conn) self.target_vbucket = cbstats.vbucket_list(self.bucket_util.buckets[0].name) self.generate_docs(target_vbucket=self.target_vbucket) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") self.task.async_failover(self.cluster.nodes_in_cluster, self.cluster.nodes_in_cluster[-1], graceful=True) self.nodes = self.rest.node_statuses() self.task.rebalance(self.cluster.nodes_in_cluster, to_add=[], to_remove=[self.cluster.nodes_in_cluster[-1]]) # Metadata Purge Interval self.meta_purge_interval = 60 self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.log.info("Starting compaction for each bucket") self.run_compaction() # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) def test_random_update_expire_same_docrange(self): pass def test_expiry_heavy_reads(self): pass def test_magma_purge_after_kv_purge(self): '''Need a way to trigger magma purging internally''' pass def test_magma_purge_before_kv_purge(self): '''Need a way to trigger magma purging internally''' pass def test_expire_data_key_tree(self): self.doc_size = 32 self.test_expiry() seqTree_update = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster)[-1] self.log.info("For upsert_size > 32 seqIndex usage-{}\ ".format(seqTree_update)) def test_random_key_expiry(self): self.key = "random_key" self.test_random_expiry() def test_expire_cold_data(self): '''Ensure that data expiring is from level 4''' pass def test_update_cold_data_with_ttl(self): '''Ensure that data expiring is from level 4''' pass def test_full_compaction_before_metadata_purge_interval(self): # if i am doing full compaction and there are expired items present # then such items will be converted into tombstones and on next compaction # those tombstones will be dropped if they satisfy above condition. self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") self.log.info("Starting compaction for each bucket") self.run_compaction() # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.assertTrue(1 >= ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format("<=1", ts)) def test_items_partially_greater_than_purge_interval(self): # Only tombstones greater than purge interval will be dropped. # remaining expired items should get converted to tombstones pass
def test_stop_process(self): """ 1. Starting loading docs into the default bucket 2. Stop the requested process, which will impact the memcached operations 3. Wait for load bucket task to complete 4. Validate the docs for durability """ error_to_simulate = self.input.param("simulate_error", None) def_bucket = self.bucket_util.buckets[0] target_node = self.getTargetNode() remote = RemoteMachineShellConnection(target_node) error_sim = CouchbaseError(self.log, remote) target_vbuckets = self.getVbucketNumbers(remote, def_bucket.name, self.target_node) if len(target_vbuckets) == 0: self.log.error("No target vbucket list generated to load data") remote.disconnect() return # Create doc_generator targeting only the active/replica vbuckets # present in the target_node gen_load = doc_generator(self.key, self.num_items, self.new_docs_to_add, key_size=self.key_size, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=target_vbuckets, vbuckets=self.cluster_util.vbuckets) if self.atomicity: task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, gen_load, "create", exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=True, sync=self.sync) else: task = self.task.async_load_gen_docs( self.cluster, def_bucket, gen_load, "create", exp=0, batch_size=1, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, skip_read_on_error=True) # Induce the error condition error_sim.create(error_to_simulate) self.sleep(20, "Wait before reverting the error condition") # Revert the simulated error condition and close the ssh session error_sim.revert(error_to_simulate) remote.disconnect() # Wait for doc loading task to complete self.task.jython_task_manager.get_task_result(task) if not self.atomicity: if len(task.fail.keys()) != 0: if self.target_node == "active" or self.num_replicas in [2, 3]: self.log_failure("Unwanted failures for keys: %s" % task.fail.keys()) validate_passed = \ self.durability_helper.validate_durability_exception( task.fail, SDKException.DurabilityAmbiguousException) if not validate_passed: self.log_failure("Unwanted exception seen during validation") # Create SDK connection for CRUD retries sdk_client = SDKClient([self.cluster.master], def_bucket) for doc_key, crud_result in task.fail.items(): result = sdk_client.crud("create", doc_key, crud_result["value"], replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout=self.sdk_timeout) if result["status"] is False: self.log_failure("Retry of doc_key %s failed: %s" % (doc_key, result["error"])) # Close the SDK connection sdk_client.close() # Update self.num_items self.num_items += self.new_docs_to_add if not self.atomicity: # Validate doc count self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.validate_test_failure()
def test_with_process_crash(self): """ Test to make sure durability will succeed even if a node goes down due to crash and has enough nodes to satisfy the durability 1. Select a node from the cluster to simulate the specified error 2. Perform CRUD on the target bucket with given timeout 3. Using cbstats to verify the operation succeeds 4. Validate all mutations are succeeded Note: self.sdk_timeout values is considered as 'seconds' """ if self.num_replicas < 2: self.assertTrue(False, msg="Required: num_replicas > 1") # Override num_of_nodes affected to 1 self.num_nodes_affected = 1 error_sim = dict() shell_conn = dict() cbstat_obj = dict() failover_info = dict() vb_info_info = dict() target_vbuckets = range(0, self.cluster_util.vbuckets) active_vbs_in_target_nodes = list() failover_info["init"] = dict() failover_info["afterCrud"] = dict() vb_info_info["init"] = dict() vb_info_info["afterCrud"] = dict() self.log.info("Selecting nodes to simulate error condition") target_nodes = self.getTargetNodes() self.log.info("Will simulate error condition on %s" % target_nodes) for node in target_nodes: shell_conn[node.ip] = RemoteMachineShellConnection(node) cbstat_obj[node.ip] = Cbstats(shell_conn[node.ip]) active_vbs_in_target_nodes += cbstat_obj[node.ip].vbucket_list( self.bucket.name, "active") vb_info_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno( self.bucket.name) failover_info["init"][node.ip] = \ cbstat_obj[node.ip].failover_stats(self.bucket.name) # Remove active vbuckets from doc_loading to avoid errors target_vbuckets = list(set(target_vbuckets) ^ set(active_vbs_in_target_nodes)) # Create required doc_generator for target_vbucket list tasks = list() gen_create = doc_generator(self.key, self.num_items, self.crud_batch_size, target_vbucket=target_vbuckets) gen_delete = doc_generator(self.key, 0, 50, target_vbucket=target_vbuckets) gen_update = doc_generator(self.key, self.num_items/2, 50, target_vbucket=target_vbuckets) # Perform CRUDs with induced error scenario is active self.log.info("Starting parallel doc_ops - Create/Read/Update/Delete") tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_create, "create", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, start_task=False)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_update, "update", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, start_task=False)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_update, "read", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, start_task=False)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_delete, "delete", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, start_task=False)) for task in tasks: self.task_manager.add_new_task(task) self.sleep(10, "Wait for doc loaders to start loading data") for node in target_nodes: # Create shell_connections shell_conn[node.ip] = RemoteMachineShellConnection(node) # Perform specified action error_sim[node.ip] = CouchbaseError(self.log, shell_conn[node.ip]) error_sim[node.ip].create(self.simulate_error, bucket_name=self.bucket.name) # Wait for document_loader tasks to complete for task in tasks: self.task.jython_task_manager.get_task_result(task) # Verify there is not failed docs in the task if len(task.fail.keys()) != 0: self.log_failure("Some CRUD failed during {0}: {1}" .format(task.op_type, task.fail)) # Update num_items value accordingly to the CRUD performed self.num_items += len(gen_create.doc_keys) - len(gen_delete.doc_keys) if self.simulate_error \ not in [DiskError.DISK_FULL, DiskError.DISK_FAILURE]: # Revert the induced error condition for node in target_nodes: error_sim[node.ip].revert(self.simulate_error, bucket_name=self.bucket.name) # Disconnect the shell connection shell_conn[node.ip].disconnect() self.sleep(10, "Wait for node recovery to complete") # In case of error with Ephemeral bucket, need to rebalance # to make sure data is redistributed properly if self.bucket_type == Bucket.Type.EPHEMERAL: retry_num = 0 while retry_num != 2: result = self.task.rebalance( self.servers[0:self.nodes_init], [], []) if result: break retry_num += 1 self.sleep(10, "Wait before retrying rebalance") self.assertTrue(result, "Rebalance failed") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) # Create a SDK client connection to retry operation client = SDKClient([self.cluster.master], self.bucket) # Retry failed docs (if any) for index, task in enumerate(tasks): if index == 0: op_type = "create" elif index == 1: op_type = "update" elif index == 2: op_type = "read" elif index == 3: op_type = "delete" op_failed = self.durability_helper.retry_with_no_error( client, task.fail, op_type) if op_failed: self.log_failure( "CRUD '{0}' failed on retry with no error condition" .format(op_type)) # Close the SDK connection client.close() # Fetch latest failover stats and validate the values are updated self.log.info("Validating failover and seqno cbstats") for node in target_nodes: vb_info_info["afterCrud"][node.ip] = \ cbstat_obj[node.ip].vbucket_seqno(self.bucket.name) failover_info["afterCrud"][node.ip] = \ cbstat_obj[node.ip].failover_stats(self.bucket.name) # Failover stat validation if self.simulate_error == CouchbaseError.KILL_MEMCACHED: val = failover_info["init"][node.ip] \ != failover_info["afterCrud"][node.ip] else: if self.simulate_error != CouchbaseError.STOP_MEMCACHED \ and self.bucket_type == Bucket.Type.EPHEMERAL: val = failover_info["init"][node.ip] \ != failover_info["afterCrud"][node.ip] else: val = failover_info["init"][node.ip] \ == failover_info["afterCrud"][node.ip] error_msg = "Failover stats mismatch after error condition:" \ " %s != %s" \ % (failover_info["init"][node.ip], failover_info["afterCrud"][node.ip]) self.assertTrue(val, msg=error_msg) # Seq_no validation (High level) val = vb_info_info["init"][node.ip] \ != vb_info_info["afterCrud"][node.ip] self.assertTrue(val, msg="vbucket seq_no not updated after CRUDs") # Verify doc count self.log.info("Validating doc count") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.validate_test_failure()
class basic_ops(BaseTestCase): def setUp(self): super(basic_ops, self).setUp() self.test_log = logging.getLogger("test") self.fail = self.input.param("fail", False) nodes_init = self.cluster.servers[1:self.nodes_init] if self.nodes_init != 1 else [] self.task.rebalance([self.cluster.master], nodes_init, []) self.cluster.nodes_in_cluster.extend([self.cluster.master] + nodes_init) self.bucket_util.add_rbac_user() if self.default_bucket: self.bucket_util.create_default_bucket(replica=self.num_replicas, compression_mode=self.compression_mode, ram_quota=100, bucket_type=self.bucket_type) time.sleep(10) self.def_bucket= self.bucket_util.get_all_buckets() self.client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket[0]) self.__durability_level() self.operation = self.input.param("operation", "afterAtrPending") # create load self.value = {'value':'value1'} self.content = self.client.translate_to_json_object(self.value) self.docs = [] self.keys = [] for i in range(self.num_items): key = "test_docs-" + str(i) doc = Tuples.of(key, self.content) self.keys.append(key) self.docs.append(doc) self.transaction_config = Transaction().createTransactionConfig(self.transaction_timeout, self.durability) self.log.info("==========Finished Basic_ops base setup========") def tearDown(self): self.client.close() super(basic_ops, self).tearDown() def __durability_level(self): if self.durability_level == Bucket.DurabilityLevel.MAJORITY: self.durability = 1 elif self.durability_level \ == Bucket.DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE: self.durability = 2 elif self.durability_level \ == Bucket.DurabilityLevel.PERSIST_TO_MAJORITY: self.durability = 3 elif self.durability_level == "ONLY_NONE": self.durability = 4 else: self.durability = 0 def set_exception(self, exception): raise BaseException(exception) def test_txnwithhooks(self): self.verify = self.input.param("verify", True) # transaction load if "Atr" in self.operation: exception = Transaction().MockRunTransaction(self.client.cluster, self.transaction_config, self.client.collection, self.docs, self.transaction_commit, self.operation, self.fail) else: if "Replace" in self.operation: exception = Transaction().MockRunTransaction(self.client.cluster, self.transaction_config, self.client.collection, self.docs, self.keys, [], self.transaction_commit, self.operation, self.keys[-1], self.fail) self.value = {'mutated':1, 'value':'value1'} self.content = self.client.translate_to_json_object(self.value) else: exception = Transaction().MockRunTransaction(self.client.cluster, self.transaction_config, self.client.collection, self.docs, [], [], self.transaction_commit, self.operation, self.keys[-1], self.fail) if "Remove" in self.operation: exception = Transaction().MockRunTransaction(self.client.cluster, self.transaction_config, self.client.collection, [], [], self.keys, self.transaction_commit, self.operation, self.keys[-1], self.fail) # verify the values for key in self.keys: result = self.client.read(key) if "Remove" in self.operation or self.transaction_commit == False or self.verify == False : if result['status']: actual_val = self.client.translate_to_json_object(result['value']) self.test_log.info("actual value for key {} is {}".format(key,actual_val)) msg = "Key should be deleted but present in the cluster {}".format(key) self.set_exception(msg) else: actual_val = self.client.translate_to_json_object(result['value']) if self.content != actual_val: self.test_log.info("actual value for key {} is {}".format(key,actual_val)) self.test_log.info("expected value for key {} is {}".format(key,self.content)) self.set_exception("actual and expected value does not match") if exception and self.fail != True: self.set_exception(exception)
def enable_error_scenario_and_test_durability(self): """ 1. Select nodes from the cluster to simulate the specified error 2. Perform CRUD on the target bucket with given timeout 3. Using cbstats to verify the operation succeeds 4. Validate all mutations met the durability condition """ error_sim = dict() shell_conn = dict() cbstat_obj = dict() failover_info = dict() vb_info_info = dict() target_vbuckets = range(0, self.vbuckets) active_vbs_in_target_nodes = list() failover_info["init"] = dict() failover_info["afterCrud"] = dict() vb_info_info["init"] = dict() vb_info_info["afterCrud"] = dict() disk_related_errors = [DiskError.DISK_FULL, DiskError.FAILOVER_DISK, "stop_persistence"] self.log.info("Selecting nodes to simulate error condition") target_nodes = self.getTargetNodes() self.log.info("Will simulate error condition on %s" % target_nodes) for node in target_nodes: shell_conn[node.ip] = RemoteMachineShellConnection(node) cbstat_obj[node.ip] = Cbstats(shell_conn[node.ip]) active_vbs = cbstat_obj[node.ip] .vbucket_list(self.bucket.name, "active") active_vbs_in_target_nodes += active_vbs vb_info_info["init"][node.ip] = cbstat_obj[node.ip].vbucket_seqno( self.bucket.name) failover_info["init"][node.ip] = \ cbstat_obj[node.ip].failover_stats(self.bucket.name) if self.simulate_error \ in [DiskError.DISK_FULL, DiskError.FAILOVER_DISK]: error_sim = DiskError(self.log, self.task_manager, self.cluster.master, target_nodes, 60, 0, False, 120, disk_location="/data") error_sim.create(action=self.simulate_error) else: for node in target_nodes: # Create shell_connections shell_conn[node.ip] = RemoteMachineShellConnection(node) # Perform specified action error_sim[node.ip] = CouchbaseError(self.log, shell_conn[node.ip]) error_sim[node.ip].create(self.simulate_error, bucket_name=self.bucket.name) if self.simulate_error not in disk_related_errors: # Remove active vbuckets from doc_loading to avoid errors target_vbuckets = list(set(target_vbuckets) ^ set(active_vbs_in_target_nodes)) # Perform CRUDs with induced error scenario is active tasks = list() gen_create = doc_generator(self.key, self.num_items, self.num_items+self.crud_batch_size, target_vbucket=target_vbuckets) gen_delete = doc_generator(self.key, 0, int(self.num_items/3), target_vbucket=target_vbuckets) gen_read = doc_generator(self.key, int(self.num_items/3), self.num_items, target_vbucket=target_vbuckets) gen_update = doc_generator(self.key, int(self.num_items/2), self.num_items, target_vbucket=target_vbuckets) self.log.info("Starting parallel doc_ops - Create/Read/Update/Delete") tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_create, "create", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_update, "update", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_read, "read", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries)) tasks.append(self.task.async_load_gen_docs( self.cluster, self.bucket, gen_delete, "delete", 0, batch_size=10, process_concurrency=1, replicate_to=self.replicate_to, persist_to=self.persist_to, durability=self.durability_level, timeout_secs=self.sdk_timeout, retries=self.sdk_retries)) # Update num_items value accordingly to the CRUD performed self.num_items += len(gen_create.doc_keys) - len(gen_delete.doc_keys) # Wait for document_loader tasks to complete for task in tasks: self.task.jython_task_manager.get_task_result(task) # Verify there is not failed docs in the task if len(task.fail.keys()) != 0: self.log_failure("Some CRUD failed for {0}".format(task.fail)) if self.simulate_error \ not in [DiskError.DISK_FULL, DiskError.FAILOVER_DISK]: # Revert the induced error condition for node in target_nodes: error_sim[node.ip].revert(self.simulate_error, bucket_name=self.bucket.name) # Disconnect the shell connection shell_conn[node.ip].disconnect() # Create a SDK client connection to retry operation client = SDKClient(RestConnection(self.cluster.master), self.bucket.name) # Retry failed docs (if any) for index, task in enumerate(tasks): if index == 0: op_type = "create" elif index == 1: op_type = "update" elif index == 2: op_type = "read" elif index == 3: op_type = "delete" op_failed = self.durability_helper.retry_with_no_error( client, task.fail, op_type) if op_failed: self.log_failure( "CRUD '{0}' failed on retry with no error condition" .format(op_type)) # Close the SDK connection client.close() # Fetch latest failover stats and validate the values are updated self.log.info("Validating failover and seqno cbstats") for node in target_nodes: vb_info_info["afterCrud"][node.ip] = \ cbstat_obj[node.ip].vbucket_seqno(self.bucket.name) failover_info["afterCrud"][node.ip] = \ cbstat_obj[node.ip].failover_stats(self.bucket.name) # Failover validation if self.simulate_error in disk_related_errors: val = failover_info["init"][node.ip] \ == failover_info["afterCrud"][node.ip] error_msg = "Failover stats got updated" else: val = failover_info["init"][node.ip] \ != failover_info["afterCrud"][node.ip] error_msg = "Failover stats not updated after error condition" self.assertTrue(val, msg=error_msg) # Seq_no validation (High level) val = vb_info_info["init"][node.ip] \ != vb_info_info["afterCrud"][node.ip] self.assertTrue(val, msg="vbucket seq_no not updated after CRUDs") # Verify doc count self.log.info("Validating doc count") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.validate_test_failure()