class MagmaExpiryTests(MagmaBaseTest): def setUp(self): super(MagmaExpiryTests, self).setUp() self.gen_delete = None self.gen_create = None self.gen_update = None self.gen_expiry = None self.exp_pager_stime = self.input.param("exp_pager_stime", 10) self.iterations = self.input.param("iterations", 5) self.expiry_perc = self.input.param("expiry_perc", 100) self.items = self.num_items def load_bucket(self): tasks = dict() for collection in self.collections: self.generate_docs(doc_ops="create", target_vbucket=None) tasks.update(self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions, skip_read_on_error=False, scope=self.scope_name, collection=collection, monitor_stats=self.monitor_stats)) for task in tasks: self.task_manager.get_task_result(task) self.bucket_util.verify_doc_op_task_exceptions( tasks, self.cluster) self.bucket_util.log_doc_ops_task_failures(tasks) self.bucket_util._wait_for_stats_all_buckets(timeout=1200) def tearDown(self): super(MagmaExpiryTests, self).tearDown() def test_read_expired_replica(self): result = True self.gen_create = doc_generator( self.key, 0, 10, doc_size=20, doc_type=self.doc_type, key_size=self.key_size) tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", exp=10, batch_size=10, process_concurrency=1, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, ) self.task.jython_task_manager.get_task_result(tasks_info.keys()[0]) self.sleep(20) self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) for i in range(10): key = (self.key + "-" + str(i).zfill(self.key_size-len(self.key))) try: getReplicaResult = self.client.collection.getAnyReplica( key, GetAnyReplicaOptions.getAnyReplicaOptions()) if getReplicaResult: result = False try: self.log.info("Able to retreive: %s" % {"key": key, "value": getReplicaResult.contentAsObject(), "cas": getReplicaResult.cas()}) except Exception as e: print str(e) except DocumentUnretrievableException as e: pass if len(self.client.get_from_all_replicas(key)) > 0: result = False self.client.close() self.assertTrue(result, "SDK is able to retrieve expired documents") def test_expiry(self): ''' Test Focus: Expire items (for n iterations) and verify tombstone count after expiry pagers time and meta data purge interval Steps: --- Expired already created items/create expiry load --- Wait for docs to expire --- Check for tombstones count --- Verify doc count --- Set meta data purge age and verify tomb stone count --- Repeat above steps n times ''' self.log.info("test_expiry starts") self.expiry_start = 0 self.expiry_end = self.num_items self.doc_ops = "expiry" for it in range(self.iterations): self.log.info("Iteration {}".format(it)) self.expiry_perc = self.input.param("expiry_perc", 100) self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1)*(it+1) self.log.info("Iterations - {}, expected_ts_count - {}".format(it, expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) # self.bucket_util.set_metadata_purge_interval(str(self.meta_purge_interval), # buckets=self.buckets) # self.sleep(self.meta_purge_interval*60*60*2/0.04, "Wait for Metadata Purge Interval to drop \ # tomb-stones from storage") self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) #Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) self.assertTrue(self.vbuckets * (self.num_replicas+1)>=ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_create_expire_same_items(self): ''' Test Focus: Create and expire n items Steps: --- Create items == num_items (init_loading will be set to False) --- Check Disk Usage after creates --- Expire all the items --- Check for tombstones count --- Check Disk Usage --- Repeat above steps n times ''' self.log.info("test_create_expire_same_items starts") self.create_start = 0 self.create_end = self.num_items self.expiry_start = 0 self.expiry_end = self.num_items #self.create_perc = 100 #self.expiry_perc = 100 for _iter in range(self.iterations): self.maxttl = random.randint(5, 20) self.log.info("Test Iteration: {}".format(_iter)) # Create items which are expired self.generate_docs(doc_ops="create", create_start=self.create_start, create_end=self.create_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="create") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after creates {}".format(disk_usage)) size_before = disk_usage[0] self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="expiry") self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) # Space amplification check msg_stats = "Fragmentation value for {} stats exceeds\ the configured value" result = self.check_fragmentation_using_magma_stats(self.buckets[0], self.cluster.nodes_in_cluster) self.assertIs(result, True, msg_stats.format("magma")) result = self.check_fragmentation_using_bucket_stats( self.buckets[0], self.cluster.nodes_in_cluster) self.assertIs(result, True, msg_stats.format("KV")) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after expiry {}".format(disk_usage)) size_after = disk_usage[0] self.assertTrue(size_after < size_before * 0.8, "Data Size before(%s) and after expiry(%s)" .format(size_before, size_after)) # Metadata Purge Interva self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) self.sleep(60, "wait after compaction") disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) disk_usage_after_compaction = disk_usage[0] self.log.info("Iteration--{}, disk usage after compaction--{}". format(_iter, disk_usage[0])) self.assertTrue(disk_usage_after_compaction < 500, "Disk size after compaction exceeds 500MB") def test_expiry_no_wait_update(self): ''' Test Focus: Upsert and expire n items Steps: --- Upsert items == num_items (init_loading will be in magma base) --- Expire all the load -- Wait for docs to expire and conver to tomb stones --- Check Disk Usage after expiry --- Check for tombstones count --- Set meta data purge age and verify tomb stone count --- Check Disk Usage --- Repeat above steps n times ''' self.log.info(" test_expiry_no_wait_update starts") self.update_start = 0 self.update_end = self.num_items self.expiry_start = 0 self.expiry_end = self.num_items self.update_perc = 100 self.expiry_perc = 100 for _iter in range(self.iterations): self.log.info("Iteration--{}".format(_iter)) self.generate_docs(doc_ops="update", update_start=self.update_start, update_end=self.update_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="update") self.bucket_util._wait_for_stats_all_buckets() disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.debug("Disk usage after updates {}".format(disk_usage)) size_before = disk_usage[0] self.generate_docs(doc_ops="expiry", expiry_start=self.expiry_start, expiry_end=self.expiry_end) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="expiry") self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1)*(_iter+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after expiry {}".format(disk_usage)) size_after = disk_usage[0] self.assertTrue(size_after < self.disk_usage[self.disk_usage.keys()[0]] * 0.6, "Data Size before(%s) and after expiry(%s)" .format(self.disk_usage[self.disk_usage.keys()[0]], size_after)) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) disk_usage = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster) self.log.info("Disk usage after compaction {}".format(disk_usage)) size_after_compaction = disk_usage[0] self.log.info("disk usage after compaction {}".format(size_after_compaction)) self.sleep(300, "sleeping after test") # below assert is only applicable if we expire all the items self.assertTrue(size_after_compaction < 500, "size after compaction shouldn't be more than 500") def test_docs_expired_wait_for_magma_purge(self): pass def test_expiry_disk_full(self): self.expiry_perc = self.input.param("expiry_perc", 100) self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") def _get_disk_usage_in_MB(remote_client, path): disk_info = remote_client.get_disk_info(in_MB=True, path=path) disk_space = disk_info[1].split()[-3][:-1] return disk_space # Fill up the disk remote_client = RemoteMachineShellConnection(self.cluster.master) du = int(_get_disk_usage_in_MB(remote_client, self.cluster.master.data_path)) - 50 _file = os.path.join(self.cluster.master.data_path, "full_disk_") cmd = "dd if=/dev/zero of={0}{1} bs=1024M count=1" while int(du) > 0: cmd = cmd.format(_file, str(du) + "MB_" + str(time.time())) output, error = remote_client.execute_command(cmd, use_channel=True) remote_client.log_command_output(output, error) du -= 1024 if du < 1024: cmd = "dd if=/dev/zero of={0}{1} bs=" + str(du) + "M count=1" # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(items=0) # Metadata Purge Interval self.meta_purge_interval = 60 self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # free up the disk output, error = remote_client.execute_command("rm -rf full_disk*", use_channel=True) # Wait for expiry pager to insert tombstones again self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage after disk is available ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) # Metadata Purge Interval after disk space is available self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction() ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) def test_random_expiry(self): ''' Test Focus: Expire items (Items will have different ttl value) and verify tombstone count after expiry pagers time and meta data purge interval Steps: --- Expired already created items/create expiry load (Items will have different ttl values) --- Wait for docs to expire --- Check for tombstones count --- Verify doc count --- Set meta data purge age and verify tomb stone count --- Repeat above steps n times ''' self.random_exp = True self.doc_ops = "expiry" self.expiry_perc = self.input.param("expiry_perc", 100) self.generate_docs() _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}". format(expected_ts_count, ts)) self.log.info("Verifying doc counts after create doc_ops") self.bucket_util.verify_stats_all_buckets(self.num_items) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after persistent_metadata_purge_age: {}".format(ts)) # Check for tombs-tones removed self.run_compaction(compaction_iterations=1) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after bucket compaction: {}".format(ts)) self.assertTrue(self.vbuckets * (self.num_replicas+1)>=ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_expire_read_validate_meta(self): self.expiry_perc = self.input.param("expiry_perc", 100) self.doc_ops = "expiry" self.bucket_util._expiry_pager(216000) self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") self.sigkill_memcached() self.bucket_util._expiry_pager(216000) # Read all the docs to ensure they get converted to tombstones self.generate_docs(doc_ops="read", read_start=self.expiry_start, read_end=self.expiry_end) self.gen_delete = copy.deepcopy(self.gen_read) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True, doc_ops="delete") self.sleep(180, "wait after get ops") #data_validation = self.task.async_validate_docs( # self.cluster, self.bucket_util.buckets[0], # self.gen_read, "delete", 0, # batch_size=self.batch_size, # process_concurrency=self.process_concurrency, # pause_secs=5, timeout_secs=self.sdk_timeout) #self.task.jython_task_manager.get_task_result(data_validation) # All docs converted to tomb-stone # Check for tombstone count in Storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after exp_pager_stime: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(expected_ts_count, ts)) def test_wait_for_expiry_read_repeat(self): for _iter in range(self.iterations): self.maxttl = random.randint(20, 60) self.log.info("Test Iteration: {}".format(_iter)) self.test_expire_read_validate_meta() def test_expiry_full_compaction(self): self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") # exp_pager_stime self.bucket_util._expiry_pager(21600) self.log.info("Starting compaction for each bucket to add tombstones") self.run_compaction(compaction_iterations=1) self.sleep(300, "sleep after triggering compaction") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after compaction: {}".format(ts)) expected_ts_count = self.items*self.expiry_perc/100*(self.num_replicas+1) self.log.info("Expected ts count after compaction is {}".format(expected_ts_count)) self.assertEqual(expected_ts_count, ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(expected_ts_count, ts)) # Metadata Purge Interval self.meta_purge_interval = 180 self.meta_purge_interval_in_days = 180 / 86400.0 self.set_metadata_purge_interval( value=self.meta_purge_interval_in_days, buckets=self.buckets) self.sleep(180, "sleeping after setting metadata purge interval using diag/eval") self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.log.info("Starting compaction for each bucket") self.run_compaction(compaction_iterations=1) self.sleep(300, "sleep after triggering compaction, to drop tombstones") # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.log.info("Expected {}".format(self.vbuckets * (self.num_replicas+1))) self.assertTrue(self.vbuckets * (self.num_replicas+1) >= ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format(self.vbuckets * (self.num_replicas+1), ts)) def test_drop_collection_expired_items(self): self.log.info("test_drop_collection_expired_items starts") tasks = dict() for collection in self.collections[::2]: self.generate_docs(doc_ops="expiry") task = self.loadgen_docs(scope=self.scope_name, collection=collection, _sync=False, doc_ops="expiry") tasks.update(task.items()) for task in tasks: self.task_manager.get_task_result(task) self.sleep(self.maxttl, "Wait for docs to expire") # Convert to tomb-stones tasks_info = dict() self.gen_delete = copy.deepcopy(self.gen_expiry) for collection in self.collections[::2]: temp_tasks_info = self.bucket_util._async_validate_docs( self.cluster, self.gen_delete, "delete", 0, batch_size=self.batch_size, process_concurrency=self.process_concurrency, pause_secs=5, timeout_secs=self.sdk_timeout, scope=self.scope_name, collection=collection, retry_exceptions=self.retry_exceptions, ignore_exceptions=self.ignore_exceptions) tasks_info.update(temp_tasks_info.items()) for task in tasks_info: self.task_manager.get_task_result(task) for collection in self.collections[::2]: self.bucket_util.drop_collection(self.cluster.master, self.buckets[0], scope_name=self.scope_name, collection_name=collection) self.buckets[0].scopes[self.scope_name].collections.pop(collection) self.collections.remove(collection) self.sleep(180, "sleep after dropping collections") ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("tombstone count is {}".format(ts)) self.assertEqual(ts, 0, "Tombstones found after collections(expired items) drop.") def test_drop_collection_during_tombstone_creation(self): scope_name, collections = self.scope_name, self.collections self.load_bucket() tasks = [] for collection in collections[::2]: self.generate_docs(doc_ops="expiry") tasks.append(self.loadgen_docs(scope=scope_name, collection=collection, _sync=False)) # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") for task in tasks: self.task_manager.get_task_result(task) for collection in collections[::2]: self.bucket_util.drop_collection(self.cluster.master, self.buckets[0], scope_name=scope_name, collection_name=collection) self.buckets[0].scopes[scope_name].collections.pop(collection) collections.remove(collection) ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.assertEqual(ts, 0, "Tombstones found after collections(expired items) drop.") def test_failover_expired_items_in_vB(self): self.maxttl = 120 self.doc_ops = "expiry" self.expiry_perc = self.input.param("expiry_perc", 100) shell_conn = RemoteMachineShellConnection(self.cluster.nodes_in_cluster[-1]) cbstats = Cbstats(shell_conn) self.target_vbucket = cbstats.vbucket_list(self.bucket_util.buckets[0].name) self.generate_docs(target_vbucket=self.target_vbucket) _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() # exp_pager_stime self.bucket_util._expiry_pager(self.exp_pager_stime) self.sleep(self.exp_pager_stime, "Wait until exp_pager_stime for kv_purger\ to kickoff") self.sleep(self.exp_pager_stime*10, "Wait for KV purger to scan expired docs and add \ tombstones.") self.task.async_failover(self.cluster.nodes_in_cluster, self.cluster.nodes_in_cluster[-1], graceful=True) self.nodes = self.rest.node_statuses() self.task.rebalance(self.cluster.nodes_in_cluster, to_add=[], to_remove=[self.cluster.nodes_in_cluster[-1]]) # Metadata Purge Interval self.meta_purge_interval = 60 self.bucket_util.cbepctl_set_metadata_purge_interval( value=self.meta_purge_interval, buckets=self.buckets) self.sleep(self.meta_purge_interval*2, "Wait for Metadata Purge Interval to drop \ tomb-stones from storage") self.log.info("Starting compaction for each bucket") self.run_compaction() # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) def test_random_update_expire_same_docrange(self): pass def test_expiry_heavy_reads(self): pass def test_magma_purge_after_kv_purge(self): '''Need a way to trigger magma purging internally''' pass def test_magma_purge_before_kv_purge(self): '''Need a way to trigger magma purging internally''' pass def test_expire_data_key_tree(self): self.doc_size = 32 self.test_expiry() seqTree_update = self.get_disk_usage(self.buckets[0], self.cluster.nodes_in_cluster)[-1] self.log.info("For upsert_size > 32 seqIndex usage-{}\ ".format(seqTree_update)) def test_random_key_expiry(self): self.key = "random_key" self.test_random_expiry() def test_expire_cold_data(self): '''Ensure that data expiring is from level 4''' pass def test_update_cold_data_with_ttl(self): '''Ensure that data expiring is from level 4''' pass def test_full_compaction_before_metadata_purge_interval(self): # if i am doing full compaction and there are expired items present # then such items will be converted into tombstones and on next compaction # those tombstones will be dropped if they satisfy above condition. self.doc_ops = "expiry" self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.bucket_util._wait_for_stats_all_buckets() self.sleep(self.maxttl, "Wait for docs to expire") self.log.info("Starting compaction for each bucket") self.run_compaction() # All docs and tomb-stone should be dropped from the storage ts = self.get_tombstone_count_key(self.cluster.nodes_in_cluster) self.log.info("Tombstones after full compaction: {}".format(ts)) self.assertTrue(1 >= ts, "Incorrect tombstone count in storage,\ Expected: {}, Found: {}".format("<=1", ts)) def test_items_partially_greater_than_purge_interval(self): # Only tombstones greater than purge interval will be dropped. # remaining expired items should get converted to tombstones pass
def verify_cas(self, ops, generator): """ Verify CAS value manipulation. For update we use the latest CAS value return by set() to do the mutation again to see if there is any exceptions. We should be able to mutate that item with the latest CAS value. For delete(), after it is called, we try to mutate that item with the cas value returned by delete(). We should see SDK Error. Otherwise the test should fail. For expire, We want to verify using the latest CAS value of that item can not mutate it because it is expired already. """ for bucket in self.cluster.buckets: client = SDKClient([self.cluster.master], bucket) gen = generator while gen.has_next(): key, value = gen.next() vb_of_key = self.bucket_util.get_vbucket_num_for_key(key) active_node_ip = None for node_ip in self.shell_conn.keys(): if vb_of_key in self.vb_details[node_ip]["active"]: active_node_ip = node_ip break self.log.info("Performing %s on key %s" % (ops, key)) if ops in ["update", "touch"]: for x in range(self.mutate_times): old_cas = client.crud("read", key, timeout=10)["cas"] # value = {"val": "mysql-new-value-%s" % x} if ops == 'update': result = client.crud( "replace", key, value, durability=self.durability_level, cas=old_cas) else: prev_exp = 0 for exp in [0, 60, 0, 0]: result = client.touch( key, exp, durability=self.durability_level, timeout=self.sdk_timeout) if exp == prev_exp: if result["cas"] != old_cas: self.log_failure( "CAS updated for " "touch with same exp: %s" % result) else: if result["cas"] == old_cas: self.log_failure( "CAS not updated %s == %s" % (old_cas, result["cas"])) old_cas = result["cas"] prev_exp = exp if result["status"] is False: client.close() self.log_failure("Touch / replace with cas failed") return new_cas = result["cas"] if ops == 'update': if old_cas == new_cas: self.log_failure("CAS old (%s) == new (%s)" % (old_cas, new_cas)) if result["value"] != value: self.log_failure("Value mismatch. " "%s != %s" % (result["value"], value)) else: self.log.debug( "Mutate %s with CAS %s successfully! " "Current CAS: %s" % (key, old_cas, new_cas)) active_read = client.crud("read", key, timeout=self.sdk_timeout) active_cas = active_read["cas"] replica_cas = -1 cas_in_active_node = \ self.cb_stat[active_node_ip].vbucket_details( bucket.name)[str(vb_of_key)]["max_cas"] if str(cas_in_active_node) != str(new_cas): self.log_failure("CbStats CAS mismatch. %s != %s" % (cas_in_active_node, new_cas)) poll_count = 0 max_retry = 5 while poll_count < max_retry: replica_read = client.get_from_all_replicas(key)[0] replica_cas = replica_read["cas"] if active_cas == replica_cas \ or self.durability_level: break poll_count = poll_count + 1 self.sleep(1, "Retry read CAS from replica..") if active_cas != replica_cas: self.log_failure("Replica cas mismatch. %s != %s" % (new_cas, replica_cas)) elif ops == "delete": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("delete", key, durability=self.durability_level, timeout=self.sdk_timeout) self.log.info("CAS after delete of key %s: %s" % (key, result["cas"])) result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("The item should already be deleted") if SDKException.DocumentNotFoundException \ not in result["error"]: self.log_failure("Invalid Exception: %s" % result) if result["cas"] != 0: self.log_failure("Delete returned invalid cas: %s, " "Expected 0" % result["cas"]) if result["cas"] == old_cas: self.log_failure("Deleted doc returned old cas: %s " % old_cas) elif ops == "expire": old_cas = client.crud("read", key, timeout=10)["cas"] result = client.crud("touch", key, exp=self.expire_time) if result["status"] is True: if result["cas"] == old_cas: self.log_failure("Touch failed to update CAS") else: self.log_failure("Touch operation failed") self.sleep(self.expire_time + 1, "Wait for item to expire") result = client.crud("replace", key, "test", durability=self.durability_level, timeout=self.sdk_timeout, cas=old_cas) if result["status"] is True: self.log_failure("Able to mutate %s with old cas: %s" % (key, old_cas)) if SDKException.DocumentNotFoundException \ not in result["error"]: self.log_failure("Invalid error after expiry: %s" % result)
class MagmaExpiryTests(MagmaBaseTest): def setUp(self): super(MagmaExpiryTests, self).setUp() self.gen_delete = None self.gen_create = None self.gen_update = None self.gen_expiry = None def tearDown(self): super(MagmaExpiryTests, self).tearDown() def test_expiry(self): result = True self.gen_create = doc_generator(self.key, 0, 10, doc_size=20, doc_type=self.doc_type, key_size=self.key_size) tasks_info = self.bucket_util._async_load_all_buckets( self.cluster, self.gen_create, "create", 10, batch_size=10, process_concurrency=1, persist_to=self.persist_to, replicate_to=self.replicate_to, durability=self.durability_level, pause_secs=5, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, ) self.task.jython_task_manager.get_task_result(tasks_info.keys()[0]) self.sleep(20) self.client = SDKClient([self.cluster.master], self.bucket_util.buckets[0], scope=CbServer.default_scope, collection=CbServer.default_collection) for i in range(10): key = (self.key + "-" + str(i).zfill(self.key_size - len(self.key))) try: getReplicaResult = self.client.collection.getAnyReplica( key, GetAnyReplicaOptions.getAnyReplicaOptions()) if getReplicaResult: result = False try: self.log.info( "Able to retreive: %s" % { "key": key, "value": getReplicaResult.contentAsObject(), "cas": getReplicaResult.cas() }) except Exception as e: print str(e) except DocumentUnretrievableException as e: pass if len(self.client.get_from_all_replicas(key)) > 0: result = False self.client.close() self.assertTrue(result, "SDK is able to retrieve expired documents") def test_expiry_update(self): count = 0 self.gen_expiry = 100 self.log.info("Test Iteration count == {}".format(count)) self.generate_docs(doc_ops="expiry") _ = self.loadgen_docs(self.retry_exceptions, self.ignore_exceptions, _sync=True) self.sleep(self.maxttl, "Wait to docs to expire") self.log.info("Verifying doc counts after create doc_ops") self.bucket_util._wait_for_stats_all_buckets() self.bucket_util.verify_stats_all_buckets(self.num_items) self.generate_docs(doc_ops="read") count += 1