def test_basic_commit(self): ''' Test transaction commit, rollback, time ahead, time behind scenarios with replica, persist_to and replicate_to settings ''' # Atomicity.basic_ops.basic_ops.test_basic_commit self.drift_ahead = self.input.param("drift_ahead", False) self.drift_behind = self.input.param("drift_behind", False) gen_create = self.get_doc_generator(0, self.num_items) self.op_type = self.input.param("op_type", 'create') if self.drift_ahead: shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue(shell.change_system_time(3600), 'Failed to advance the clock') output, _ = shell.execute_command('date') self.log.info('Date after is set forward {0}'.format(output)) if self.drift_behind: shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue(shell.change_system_time(-3600), 'Failed to advance the clock') output, _ = shell.execute_command('date') self.log.info('Date after is set behind {0}'.format(output)) self.log.info("going to create a task") task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, gen_create, self.op_type, exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=self.transaction_timeout, commit=self.transaction_commit, durability=self.durability_level, sync=self.sync, defer=self.defer) self.log.info("going to execute the task") self.task.jython_task_manager.get_task_result(task) if self.op_type == "time_out": self.sleep(90, "sleep for 90 seconds so that the staged docs will be cleared") task = self.task.async_load_gen_docs_atomicity( self.cluster, self.bucket_util.buckets, gen_create, "create", exp=0, batch_size=10, process_concurrency=self.process_concurrency, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, update_count=self.update_count, transaction_timeout=200, commit=self.transaction_commit, durability=self.durability_level, sync=self.sync, defer=self.defer) self.task_manager.get_task_result(task)
def test_basic_commit(self): ''' Test transaction commit, rollback, time ahead, time behind scenarios with replica, persist_to and replicate_to settings ''' # Atomicity.basic_ops.basic_ops.test_basic_commit self.transaction_timeout = self.input.param("transaction_timeout", 100) self.transaction_commit = self.input.param("transaction_commit", True) self.drift_ahead = self.input.param("drift_ahead", False) self.drift_behind = self.input.param("drift_behind", False) gen_create = self.get_doc_generator(0, self.num_items) self.op_type = self.input.param("op_type", 'create') self.num_buckets = self.input.param("num_buckets", 0) self.default_bucket = self.input.param("default_bucket", True) self.standard_bucket = self.input.param("standard_bucket", 1) self.exp = self.input.param("expiry", 0) # print_ops_task = self.bucket_util.async_print_bucket_ops(def_bucket) if self.default_bucket: self.bucket_util.create_default_bucket(replica=self.num_replicas, compression_mode=self.compression_mode, ram_quota=100) if self.drift_ahead: shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue( shell.change_system_time(3600), 'Failed to advance the clock') output,error = shell.execute_command('date') self.log.info('Date after is set forward {0}'.format( output )) if self.drift_behind: shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue( shell.change_system_time(-3600), 'Failed to advance the clock') output,error = shell.execute_command('date') self.log.info('Date after is set behind {0}'.format( output )) if self.num_buckets: self.bucket_util.create_multiple_buckets(self.cluster.master, self.num_replicas, bucket_count=self.num_buckets, bucket_type='ephemeral') if self.standard_bucket: self.bucket_util.create_standard_buckets(self.cluster.master, self.standard_buckets, bucket_size=100) self.def_bucket= self.bucket_util.get_all_buckets() # client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket) # # Transaction().nonTxnRemoves(client.collection) print "going to create a task" task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket, gen_create, self.op_type , exp=self.exp, batch_size=10, process_concurrency=8, replicate_to=self.replicate_to, persist_to=self.persist_to, timeout_secs=self.sdk_timeout, retries=self.sdk_retries, transaction_timeout=self.transaction_timeout, commit=self.transaction_commit) print "going to execute the task" self.task.jython_task_manager.get_task_result(task)
def test_logical_clock_ticks(self): self.log.info('starting test_logical_clock_ticks') payload = "name={0}&roles=admin&password=password".format( self.buckets[0].name) self.rest.add_set_builtin_user(self.buckets[0].name, payload) sdk_client = SDKClient(scheme='couchbase', hosts=[self.servers[0].ip], bucket=self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client( self.servers[0], self.buckets[0]) shell = RemoteMachineShellConnection(self.servers[0]) # do a bunch of mutations to set the max cas gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') base_total_logical_clock_ticks = 0 for i in range(self.vbuckets): #print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'] base_total_logical_clock_ticks = base_total_logical_clock_ticks + int( vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The base total logical clock ticks is {0}'.format( base_total_logical_clock_ticks)) # move the system clock back so the logical counter part of HLC is used and the logical clock ticks # stat is incremented self.assertTrue( shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), 'Failed to advance the clock') # do more mutations NUMBER_OF_MUTATIONS = 10000 gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=NUMBER_OF_MUTATIONS) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') time.sleep(30) total_logical_clock_ticks = 0 for i in range(self.vbuckets): total_logical_clock_ticks = total_logical_clock_ticks + int( vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The total logical clock ticks is {0}'.format( total_logical_clock_ticks)) self.assertTrue( total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS, 'Expected clock tick {0} actual {1}'.format( NUMBER_OF_MUTATIONS, total_logical_clock_ticks - base_total_logical_clock_ticks))
def test_logical_clock_ticks(self): self.log.info("starting test_logical_clock_ticks") sdk_client = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket=self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0]) shell = RemoteMachineShellConnection(self.servers[0]) # do a bunch of mutations to set the max cas gen_load = BlobGenerator("key-for-cas-test-logical-ticks", "value-for-cas-test-", self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats("vbucket-details") base_total_logical_clock_ticks = 0 for i in range(self.vbuckets): # print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'] base_total_logical_clock_ticks = base_total_logical_clock_ticks + int( vbucket_stats["vb_" + str(i) + ":logical_clock_ticks"] ) self.log.info("The base total logical clock ticks is {0}".format(base_total_logical_clock_ticks)) # move the system clock back so the logical counter part of HLC is used and the logical clock ticks # stat is incremented self.assertTrue(shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to advance the clock") # do more mutations NUMBER_OF_MUTATIONS = 10000 gen_load = BlobGenerator( "key-for-cas-test-logical-ticks", "value-for-cas-test-", self.value_size, end=NUMBER_OF_MUTATIONS ) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats("vbucket-details") total_logical_clock_ticks = 0 for i in range(self.vbuckets): total_logical_clock_ticks = total_logical_clock_ticks + int( vbucket_stats["vb_" + str(i) + ":logical_clock_ticks"] ) self.log.info("The total logical clock ticks is {0}".format(total_logical_clock_ticks)) self.assertTrue( total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS, "Expected clock tick {0} actual {1}".format( NUMBER_OF_MUTATIONS, total_logical_clock_ticks - base_total_logical_clock_ticks ), )
def test_logical_clock_ticks(self): self.log.info('starting test_logical_clock_ticks') payload = "name={0}&roles=admin&password=password".format( self.buckets[0].name) self.rest.add_set_builtin_user(self.buckets[0].name, payload) sdk_client = SDKClient(scheme='couchbase',hosts = [self.servers[0].ip], bucket = self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0]) shell = RemoteMachineShellConnection(self.servers[0]) # do a bunch of mutations to set the max cas gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') base_total_logical_clock_ticks = 0 for i in range(self.vbuckets): #print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'] base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The base total logical clock ticks is {0}'.format( base_total_logical_clock_ticks)) # move the system clock back so the logical counter part of HLC is used and the logical clock ticks # stat is incremented self.assertTrue( shell.change_system_time( -LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to advance the clock') # do more mutations NUMBER_OF_MUTATIONS = 10000 gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=NUMBER_OF_MUTATIONS) self._load_all_buckets(self.master, gen_load, "create", 0) vbucket_stats = mc_client.stats('vbucket-details') time.sleep(30) total_logical_clock_ticks = 0 for i in range(self.vbuckets): total_logical_clock_ticks = total_logical_clock_ticks + int(vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']) self.log.info('The total logical clock ticks is {0}'.format( total_logical_clock_ticks)) self.assertTrue( total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS, 'Expected clock tick {0} actual {1}'.format(NUMBER_OF_MUTATIONS, total_logical_clock_ticks- base_total_logical_clock_ticks ))
def test_poisoned_cas(self): self.log.info("starting test_poisoned_cas") """ - set the clock ahead - do lots of sets and get some CASs - do a set and get the CAS (flag, CAS, value) and save it - set the clock back - verify the CAS is still big on new sets - reset the CAS - do the vbucket max cas and verify - do a new mutation and verify the CAS is smaller """ sdk_client = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket=self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0]) shell = RemoteMachineShellConnection(self.servers[0]) # move the system clock ahead to poison the CAS shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue(shell.change_system_time(LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to advance the clock") output, error = shell.execute_command("date") self.log.info("Date after is set forward {0}".format(output)) rc = sdk_client.set("key1", "val1") rc = mc_client.get("key1") poisoned_cas = rc[1] self.log.info("The poisoned CAS is {0}".format(poisoned_cas)) # do lots of mutations to set the max CAS for all vbuckets gen_load = BlobGenerator("key-for-cas-test", "value-for-cas-test-", self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) # move the clock back again and verify the CAS stays large self.assertTrue(shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to change the clock") output, error = shell.execute_command("date") self.log.info("Date after is set backwards {0}".format(output)) use_mc_bin_client = self.input.param("use_mc_bin_client", False) if use_mc_bin_client: rc = mc_client.set("key2", 0, 0, "val2") second_poisoned_cas = rc[1] else: rc = sdk_client.set("key2", "val2") second_poisoned_cas = rc.cas self.log.info("The second_poisoned CAS is {0}".format(second_poisoned_cas)) self.assertTrue( second_poisoned_cas > poisoned_cas, "Second poisoned CAS {0} is not larger than the first poisoned cas".format( second_poisoned_cas, poisoned_cas ), ) # reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not # changed then the CAS will immediately continue with the clock. I see two scenarios: # 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used # 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use # do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000 for i in range(self.vbuckets): output, error = shell.execute_cbepctl( self.buckets[0], "", "set_vbucket_param", "max_cas ", str(i) + " " + str(earlier_max_cas) ) if len(error) > 0: self.fail("Failed to set the max cas") # verify the max CAS for i in range(self.vbuckets): max_cas = int(mc_client.stats("vbucket-details")["vb_" + str(i) + ":max_cas"]) self.assertTrue( max_cas == earlier_max_cas, "Max CAS not properly set for vbucket {0} set as {1} and observed {2}".format( i, earlier_max_cas, max_cas ), ) self.log.info("Per cbstats the max cas for bucket {0} is {1}".format(i, max_cas)) rc1 = sdk_client.set("key-after-resetting cas", "val1") rc2 = mc_client.get("key-after-resetting cas") set_cas_after_reset_max_cas = rc2[1] self.log.info("The later CAS is {0}".format(set_cas_after_reset_max_cas)) self.assertTrue( set_cas_after_reset_max_cas < poisoned_cas, "For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format( "key-after-resetting cas", set_cas_after_reset_max_cas, poisoned_cas ), ) # do a bunch of sets and verify the CAS is small - this is really only one set, need to do more gen_load = BlobGenerator( "key-for-cas-test-after-cas-is-reset", "value-for-cas-test-", self.value_size, end=1000 ) self._load_all_buckets(self.master, gen_load, "create", 0) gen_load.reset() while gen_load.has_next(): key, value = gen_load.next() try: rc = mc_client.get(key) # rc = sdk_client.get(key) cas = rc[1] self.assertTrue( cas < poisoned_cas, "For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format( key, cas, poisoned_cas ), ) except: self.log.info("get error with {0}".format(key)) rc = sdk_client.set("key3", "val1") better_cas = rc.cas self.log.info("The better CAS is {0}".format(better_cas)) self.assertTrue(better_cas < poisoned_cas, "The CAS was not improved") # set the clock way ahead - remote_util_OS.py (new) # do a bunch of mutations - not really needed # do the fix command - cbepctl, the existing way (remote util) # do some mutations, verify they conform to the new CAS - build on the CAS code, # where to iterate over the keys and get the CAS? """
def test_poisoned_cas(self): """ @note: - set the clock ahead - do lots of sets and get some CASs - do a set and get the CAS (flag, CAS, value) and save it - set the clock back - verify the CAS is still big on new sets - reset the CAS - do the vbucket max cas and verify - do a new mutation and verify the CAS is smaller """ #creating a user 'default' for the bucket self.log.info('starting test_poisoned_cas') payload = "name={0}&roles=admin&password=password".format( self.buckets[0].name) self.rest.add_set_builtin_user(self.buckets[0].name, payload) sdk_client = SDKClient(scheme='couchbase', hosts = [self.servers[0].ip], bucket = self.buckets[0].name) mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0]) # move the system clock ahead to poison the CAS shell = RemoteMachineShellConnection(self.servers[0]) self.assertTrue( shell.change_system_time( LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to advance the clock') output, error = shell.execute_command('date') self.log.info('Date after is set forward {0}'.format( output )) rc = sdk_client.set('key1', 'val1') rc = mc_client.get('key1' ) poisoned_cas = rc[1] self.log.info('The poisoned CAS is {0}'.format(poisoned_cas)) # do lots of mutations to set the max CAS for all vbuckets gen_load = BlobGenerator('key-for-cas-test', 'value-for-cas-test-', self.value_size, end=10000) self._load_all_buckets(self.master, gen_load, "create", 0) # move the clock back again and verify the CAS stays large self.assertTrue( shell.change_system_time( -LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to change the clock') output, error = shell.execute_command('date') self.log.info('Date after is set backwards {0}'.format( output)) use_mc_bin_client = self.input.param("use_mc_bin_client", True) if use_mc_bin_client: rc = mc_client.set('key2', 0, 0, 'val2') second_poisoned_cas = rc[1] else: rc = sdk_client.set('key2', 'val2') second_poisoned_cas = rc.cas self.log.info('The second_poisoned CAS is {0}'.format(second_poisoned_cas)) self.assertTrue( second_poisoned_cas > poisoned_cas, 'Second poisoned CAS {0} is not larger than the first poisoned cas'.format(second_poisoned_cas, poisoned_cas)) # reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not # changed then the CAS will immediately continue with the clock. I see two scenarios: # 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used # 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use # do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000 for i in range(self.vbuckets): output, error = shell.execute_cbepctl(self.buckets[0], "", "set_vbucket_param", "max_cas ", str(i) + ' ' + str(earlier_max_cas) ) if len(error) > 0: self.fail('Failed to set the max cas') # verify the max CAS for i in range(self.vbuckets): max_cas = int( mc_client.stats('vbucket-details')['vb_' + str(i) + ':max_cas'] ) self.assertTrue(max_cas == earlier_max_cas, 'Max CAS not properly set for vbucket {0} set as {1} and observed {2}'.format(i, earlier_max_cas, max_cas ) ) self.log.info('Per cbstats the max cas for bucket {0} is {1}'.format(i, max_cas) ) rc1 = sdk_client.set('key-after-resetting cas', 'val1') rc2 = mc_client.get('key-after-resetting cas' ) set_cas_after_reset_max_cas = rc2[1] self.log.info('The later CAS is {0}'.format(set_cas_after_reset_max_cas)) self.assertTrue( set_cas_after_reset_max_cas < poisoned_cas, 'For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format('key-after-resetting cas', set_cas_after_reset_max_cas, poisoned_cas)) # do a bunch of sets and verify the CAS is small - this is really only one set, need to do more gen_load = BlobGenerator('key-for-cas-test-after-cas-is-reset', 'value-for-cas-test-', self.value_size, end=1000) self._load_all_buckets(self.master, gen_load, "create", 0) gen_load.reset() while gen_load.has_next(): key, value = next(gen_load) try: rc = mc_client.get( key ) #rc = sdk_client.get(key) cas = rc[1] self.assertTrue( cas < poisoned_cas, 'For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format(key, cas, poisoned_cas)) except: self.log.info('get error with {0}'.format(key)) rc = sdk_client.set('key3', 'val1') better_cas = rc.cas self.log.info('The better CAS is {0}'.format(better_cas)) self.assertTrue( better_cas < poisoned_cas, 'The CAS was not improved') # set the clock way ahead - remote_util_OS.py (new) # do a bunch of mutations - not really needed # do the fix command - cbepctl, the existing way (remote util) # do some mutations, verify they conform to the new CAS - build on the CAS code, # where to iterate over the keys and get the CAS? """