class BasicCrudTests(MagmaBaseTest): def setUp(self): super(BasicCrudTests, self).setUp() self.enable_disable_swap_space(self.servers) start = 0 end = self.num_items start_read = 0 end_read = self.num_items if self.rev_write: start = -int(self.num_items - 1) end = 1 if self.rev_read: start_read = -int(self.num_items - 1) end_read = 1 self.gen_create = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.result_task = self._load_all_buckets(self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size, dgm_batch=self.dgm_batch) if self.active_resident_threshold != 100: for task in self.result_task.keys(): self.num_items = task.doc_index self.log.info("Verifying num_items counts after doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.disk_usage = dict() if self.standard_buckets == 1 or self.standard_buckets == self.magma_buckets: for bucket in self.bucket_util.get_all_buckets(): disk_usage = self.get_disk_usage(bucket, self.servers) self.disk_usage[bucket.name] = disk_usage[0] self.log.info( "For bucket {} disk usage after initial creation is {}MB\ ".format(bucket.name, self.disk_usage[bucket.name])) self.gen_read = doc_generator( self.key, start_read, end_read, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.cluster_util.print_cluster_stats() self.bucket_util.print_bucket_stats() def tearDown(self): super(BasicCrudTests, self).tearDown() def test_expiry(self): result = True self.gen_create = doc_generator(self.key, 0, 10, doc_size=20, doc_type=self.doc_type, key_size=self.key_size) tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", 10, batch_size=10, process_concurrency=1, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, ) self.task.jython_task_manager.get_task_result(tasks_info.keys()[0]) self.sleep(20) self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) for i in range(10): key = (self.key + "-" + str(i).zfill(self.key_size - len(self.key))) try: getReplicaResult = self.client.collection.getAnyReplica( key, GetAnyReplicaOptions.getAnyReplicaOptions()) if getReplicaResult: result = False try: self.log.info( "Able to retreive: %s" % { "key": key, "value": getReplicaResult.contentAsObject(), "cas": getReplicaResult.cas() }) except Exception as e: print str(e) except DocumentUnretrievableException as e: pass if len(self.client.getFromAllReplica(key)) > 0: result = False self.client.close() self.assertTrue(result, "SDK is able to retrieve expired documents") def test_basic_create_read(self): """ Write and Read docs parallely , While reading we are using old doc generator (self.gen_create) using which we already created docs in magam_base for writing we are creating a new doc generator. Befor we start read, killing memcached to make sure, all reads happen from magma/storage """ self.log.info("Loading and Reading docs parallel") count = 0 init_items = self.num_items while count < self.test_itr: self.log.info("Create Iteration count == {}".format(count)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.doc_ops = "create:read" start = self.num_items end = self.num_items + init_items start_read = self.num_items end_read = self.num_items + init_items if self.rev_write: start = -int(self.num_items + init_items - 1) end = -int(self.num_items - 1) if self.rev_read: start_read = -int(self.num_items + init_items - 1) end_read = -int(self.num_items - 1) self.gen_create = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.gen_read = doc_generator( self.key, start_read, end_read, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) if self.doc_size <= 32: for bucket in self.bucket_util.get_all_buckets(): disk_usage = self.get_disk_usage(bucket, self.servers) self.assertIs( disk_usage[2] > disk_usage[3], True, "For Bucket {} , Disk Usage for seqIndex'\n' \ After new Creates count {}'\n' \ exceeds keyIndex disk'\n' \ usage".format(bucket.name, count + 1)) if self.standard_buckets > 1 and self.standard_buckets == self.magma_buckets: disk_usage = dict() for bucket in self.bucket_util.get_all_buckets(): usage = self.get_disk_usage(bucket, self.servers) disk_usage[bucket.name] = usage[0] self.assertTrue( all([ disk_usage[disk_usage.keys()[0]] == disk_usage[key] for key in disk_usage.keys() ]), '''Disk Usage for magma buckets is not equal for same number of docs ''') count += 1 self.log.info("====test_basic_create_read ends====") def test_update_multi(self): """ Update all the docs n times, and after each iteration check for space amplificationa and data validation """ count = 0 mutated = 1 update_doc_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("Count of docs to be updated is {}\ ".format(update_doc_count)) num_update = list() while update_doc_count > self.num_items: num_update.append(self.num_items) update_doc_count -= self.num_items if update_doc_count > 0: num_update.append(update_doc_count) while count < self.test_itr: self.log.info("Update Iteration count == {}".format(count)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) self.log.debug("List of docs to be updated {}\ ".format(num_update)) for itr in num_update: self.doc_ops = "update" start = 0 end = itr if self.rev_update: start = -int(itr - 1) end = 1 self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After update count {} disk usage is {}\ ".format(count + 1, _res)) usage_factor = ( (float(self.num_items + sum(num_update)) / self.num_items) + 0.5) self.log.debug("Disk usage factor is {}".format(usage_factor)) self.assertIs( _res > usage_factor * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After Update'\n' \ Count {} exceeds Actual'\n' \ disk usage {}MB by {}'\n' \ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]], usage_factor)) count += 1 data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.enable_disable_swap_space(self.servers, disable=False) self.log.info("====test_update_multi ends====") def test_multi_update_delete(self): """ Step 1: Kill memcached and Update all the docs update_itr times After each iteration check for space amplification and for last iteration of test_itr validate docs Step 2: Delete half the docs, check sapce amplification Step 3 Recreate check for space amplification. Repeat all above steps test_itr times Step 4 : Do data validation for newly create docs """ count = 0 mutated = 1 for i in range(self.test_itr): while count < self.update_itr: self.log.debug( "Iteration {}: Step 1 of test_multi_update_delete \ ".format(self.test_itr + 1)) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) self.doc_ops = "update" start = 0 end = self.num_items if self.rev_update: start = -int(self.num_items - 1) end = 1 self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After update count {} disk usage is {}MB\ ".format(count + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ Update Count {} exceeds Actual \ disk usage {}MB by 2.5\ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]])) count += 1 # Will check data validatio only in the last # iteration of test_tr to avoid multiple # data validation, that is why below if check if i + 1 == self.test_itr: data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.update_itr += self.update_itr self.log.debug("Iteration {}: Step 2 of test_multi_update_delete \ ".format(self.test_itr + 1)) start_del = 0 end_del = self.num_items // 2 if self.rev_del: start_del = -int(self.num_items // 2 - 1) end_del = 1 self.gen_delete = doc_generator( self.key, start_del, end_del, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.log.info("Deleting num_items//2 docs") self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After delete count {} disk usage is {}MB\ ".format(i + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ Delete count {} exceeds Actual \ disk usage {}MB by 2.5 \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.debug("Iteration{}: Step 3 of test_multi_update_delete \ ".format(self.test_itr + 1)) self.gen_create = copy.deepcopy(self.gen_delete) self.log.info("Recreating num_items//2 docs") self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("disk usage after new create \ is {}".format(_res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ new Creates count {} exceeds \ Actual disk usage {}MB by \ 2.5 times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.debug("Iteration{}: Step 4 of test_multi_update_delete \ ".format(self.test_itr + 1)) data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.log.info("====test_multiUpdate_delete ends====") def test_update_rev_update(self): count = 0 mutated = 1 for i in range(self.test_itr): while count < self.update_itr: for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) tasks_info = dict() data_validation = [] g_update = doc_generator( self.key, 0, self.num_items // 2, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, g_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(tem_tasks_info.items()) start = -(self.num_items // 2 - 1) end = 1 r_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 if self.next_half: mutated -= 2 start = -(self.num_items - 1) end = -(self.num_items // 2 - 1) r_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=mutated, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) mutated += 1 tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(tem_tasks_info.items()) for task in tasks_info: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tasks_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(tasks_info) if not self.next_half: tem_tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) for task in tem_tasks_info: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tem_tasks_info, self.cluster) self.bucket_util.log_doc_ops_task_failures(tem_tasks_info) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() if self.next_half: data_validation.extend([ self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], g_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout), self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) ]) else: data_validation.append( self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], r_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout)) for task in data_validation: self.task.jython_task_manager.get_task_result(task) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after update count {}\ is {}".format(count + 1, _res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ Update Count {} exceeds \ Actual disk usage {} by four \ times".format(_res, count, self.disk_usage[self.disk_usage.keys()[0]])) count += 1 self.update_itr += self.update_itr start_del = 0 end_del = self.num_items // 2 if self.rev_del: start_del = -int(self.num_items // 2 - 1) end_del = 1 self.gen_delete = doc_generator( self.key, start_del, end_del, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) self.log.info("Deleting num_items//2 docs") self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after delete is {}".format(_res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ Delete count {} exceeds Actual \ disk usage {} by four \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.gen_create = copy.deepcopy(self.gen_delete) self.log.info("Recreating num_items//2 docs") self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) d_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(d_validation) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] - disk_usage[1] self.log.info("disk usage after new create \ is {}".format(_res)) self.assertIs( _res > 4 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {} After \ new Creates count {} exceeds \ Actual disk usage {} by four \ times".format(_res, i + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.log.info("====test_update_rev_update ends====") def test_update_single_doc_n_times(self): """ Update same document n times, where n is number which gets derived from given fragmentation value and after updates check for space amplification and data validation """ count = 0 self.assertIs((self.fragmentation <= 0 or self.fragmentation >= 100), False, msg="Fragmentation value can't be <=0 or >=100") update_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("{} is the count with which doc will be updated \ ".format(update_count)) self.doc_ops = "update" self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) self.gen_update = doc_generator( self.key, 0, 1, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=count, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) key, val = self.gen_update.next() for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() self.assertTrue( self.bucket_util._wait_warmup_completed( [self.cluster_util.cluster.master], self.bucket_util.buckets[0], wait_time=self.wait_timeout * 10)) while count < (update_count + 1): self.log.debug("Update Iteration count == {}".format(count)) val.put("mutated", count + 1) self.client.upsert(key, val) count += 1 self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("After all updates disk usage is {}MB\ ".format(_res)) usage_factor = ( (float(self.num_items + update_count) / self.num_items) + 0.5) self.log.debug("Disk usage factor is {}".format(usage_factor)) self.assertIs( _res > usage_factor * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After all Updates'\n' \ exceeds Actual'\n' \ disk usage {}MB by {}'\n' \ times".format(_res, self.disk_usage[self.disk_usage.keys()[0]], usage_factor)) data_validation = self.task.async_validate_docs( self.cluster, self.bucket_util.buckets[0], self.gen_update, "update", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout) self.task.jython_task_manager.get_task_result(data_validation) self.enable_disable_swap_space(self.servers, disable=False) self.log.info("====test_update_single_doc_n_times====") def test_read_docs_using_multithreads(self): """ Read same docs together using multithreads. """ self.log.info("Reading docs parallelly using multi threading") tasks_info = dict() update_doc_count = int( math.ceil( float(self.fragmentation * self.num_items) / (100 - self.fragmentation))) self.log.info("Count of docs to be updated is {}\ ".format(update_doc_count)) num_update = list() while update_doc_count > self.num_items: num_update.append(self.num_items) update_doc_count -= self.num_items if update_doc_count > 0: num_update.append(update_doc_count) for itr in num_update: self.doc_ops = "update" start = 0 end = itr self.gen_update = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, mutate=0, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) update_task_info = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=False) tasks_info.update(update_task_info.items()) count = 0 self.doc_ops = "read" # if self.next_half is true then one thread will read # in ascending order and other in descending order if self.next_half: start = -int(self.num_items - 1) end = 1 g_read = doc_generator(self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) for node in self.cluster.nodes_in_cluster: shell = RemoteMachineShellConnection(node) shell.kill_memcached() shell.disconnect() while count < self.read_thread_count: read_task_info = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=False) tasks_info.update(read_task_info.items()) count += 1 if self.next_half and count < self.read_thread_count: read_tasks_info = self.bucket_util._async_validate_docs( self.cluster, g_read, "read", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(read_task_info.items()) count += 1 for task in tasks_info: self.task_manager.get_task_result(task) self.log.info("Waiting for ep-queues to get drained") self.bucket_util._wait_for_stats_all_buckets() self.log.info("test_read_docs_using_multithreads ends") def test_basic_create_delete(self): """ CREATE(n)-> DELETE(n)->DISK_USAGE_CHECK REPEAT ABove test_itr_times """ self.log.info("Cretaing and Deletes docs n times ") count = 0 start = 0 end = self.num_items self.gen_delete = doc_generator( self.key, start, end, doc_size=self.doc_size, doc_type=self.doc_type, target_vbucket=self.target_vbucket, vbuckets=self.cluster_util.vbuckets, key_size=self.key_size, randomize_doc_size=self.randomize_doc_size, randomize_value=self.randomize_value, mix_key_size=self.mix_key_size, deep_copy=self.deep_copy) while count < self.test_itr: self.doc_ops = "delete" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.log.info("Verifying doc counts after delete doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) disk_usage = self.get_disk_usage( self.bucket_util.get_all_buckets()[0], self.servers) _res = disk_usage[0] self.log.info("disk usage after delete count {} \ is {}MB".format(count + 1, _res)) self.assertIs( _res > 2.5 * self.disk_usage[self.disk_usage.keys()[0]], False, "Disk Usage {}MB After \ delete count {} exceeds \ Actual disk usage {}MB by \ 2.5 times".format(_res, count + 1, self.disk_usage[self.disk_usage.keys()[0]])) self.doc_ops = "create" _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) count += 1 self.log.info("====test_basic_create_delete ends====")
def verify_cas(self, ops, generator, scope, collection): """ Verify CAS value manipulation. For update we use the latest CAS value return by set() to do the mutation again to see if there is any exceptions. We should be able to mutate that item with the latest CAS value. For delete(), after it is called, we try to mutate that item with the cas value returned by delete(). We should see SDK Error. Otherwise the test should fail. For expire, We want to verify using the latest CAS value of that item can not mutate it because it is expired already. """ for bucket in self.bucket_util.buckets: client = SDKClient([self.cluster.master], bucket) client.select_collection(scope, collection) self.log.info("CAS test on collection %s: %s" % (scope, collection)) gen = generator while gen.has_next(): key, value = gen.next() vb_of_key = self.bucket_util.get_vbucket_num_for_key(key) active_node_ip = None for node_ip in self.shell_conn.keys(): if vb_of_key in self.vb_details[node_ip]["active"]: active_node_ip = node_ip break self.log.info("Performing %s on key %s" % (ops, key)) if ops in ["update", "touch"]: for x in range(self.mutate_times): old_cas = client.crud("read", key, timeout=10)["cas"] if ops == 'update': result = client.crud( "replace", key, value, durability=self.durability_level, cas=old_cas) else: prev_exp = 0 for exp in [0, 60, 0, 0]: result = client.touch( key, exp, durability=self.durability_level, timeout=self.sdk_timeout) if exp == prev_exp: if result["cas"] != old_cas: self.log_failure( "CAS updated for " "touch with same exp: %s" % result) else: if result["cas"] == old_cas: self.log_failure( "CAS not updated %s == %s" % (old_cas, result["cas"])) old_cas = result["cas"] prev_exp = exp if result["status"] is False: client.close() self.log_failure("Touch / replace with cas failed") return new_cas = result["cas"] if ops == 'update': if old_cas == new_cas: self.log_failure("CAS old (%s) == new (%s)" % (old_cas, new_cas)) if result["value"] != value: self.log_failure("Value mismatch. " "%s != %s" % (result["value"], value)) else: self.log.debug( "Mutate %s with CAS %s successfully! " "Current CAS: %s" % (key, old_cas, new_cas)) active_read = client.crud("read", key, timeout=self.sdk_timeout) active_cas = active_read["cas"] replica_cas = -1 cas_in_active_node = \ self.cb_stat[active_node_ip].vbucket_details( bucket.name)[str(vb_of_key)]["max_cas"] if str(cas_in_active_node) != str(new_cas): self.log_failure("CbStats CAS mismatch. %s != %s" % (cas_in_active_node, new_cas)) poll_count = 0 max_retry = 5 while poll_count < max_retry: replica_read = client.getFromAllReplica(key)[0] replica_cas = replica_read["cas"] if active_cas == replica_cas \ or self.durability_level: break poll_count = poll_count + 1 self.sleep(1, "Retry read CAS from replica..") if active_cas != replica_cas: self.log_failure("Replica cas mismatch. %s != %s" % (new_cas, replica_cas)) elif ops == "delete": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("delete", key, durability=self.durability_level, timeout=self.sdk_timeout) self.log.info("CAS after delete of key %s: %s" % (key, result["cas"])) result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("The item should already be deleted") if SDKException.DocumentNotFoundException \ not in result["error"]: self.log_failure("Invalid Exception: %s" % result) if result["cas"] != 0: self.log_failure("Delete returned invalid cas: %s, " "Expected 0" % result["cas"]) if result["cas"] == old_cas: self.log_failure("Deleted doc returned old cas: %s " % old_cas) elif ops == "expire": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("touch", key, exp=self.expire_time) if result["status"] is True: if result["cas"] == old_cas: self.log_failure("Touch failed to update CAS") else: self.log_failure("Touch operation failed") self.sleep(self.expire_time + 1, "Wait for item to expire") result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("Able to mutate %s with old cas: %s" % (key, old_cas)) if SDKException.DocumentNotFoundException \ not in result["error"]: self.log_failure("Invalid error after expiry: %s" % result)