def test_basic_commit(self):
'''
Test transaction commit, rollback, time ahead,
time behind scenarios with replica, persist_to and
replicate_to settings
'''
# Atomicity.basic_ops.basic_ops.test_basic_commit
self.drift_ahead = self.input.param("drift_ahead", False)
self.drift_behind = self.input.param("drift_behind", False)
gen_create = self.get_doc_generator(0, self.num_items)
self.op_type = self.input.param("op_type", 'create')
if self.drift_ahead:
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue(shell.change_system_time(3600),
'Failed to advance the clock')
output, _ = shell.execute_command('date')
self.log.info('Date after is set forward {0}'.format(output))
if self.drift_behind:
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue(shell.change_system_time(-3600),
'Failed to advance the clock')
output, _ = shell.execute_command('date')
self.log.info('Date after is set behind {0}'.format(output))
self.log.info("going to create a task")
task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.bucket_util.buckets,
gen_create, self.op_type, exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries, update_count=self.update_count,
transaction_timeout=self.transaction_timeout,
commit=self.transaction_commit, durability=self.durability_level,
sync=self.sync, defer=self.defer)
self.log.info("going to execute the task")
self.task.jython_task_manager.get_task_result(task)
if self.op_type == "time_out":
self.sleep(90, "sleep for 90 seconds so that the staged docs will be cleared")
task = self.task.async_load_gen_docs_atomicity(
self.cluster, self.bucket_util.buckets,
gen_create, "create", exp=0,
batch_size=10,
process_concurrency=self.process_concurrency,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries, update_count=self.update_count,
transaction_timeout=200,
commit=self.transaction_commit,
durability=self.durability_level,
sync=self.sync, defer=self.defer)
self.task_manager.get_task_result(task)
def test_basic_commit(self):
''' Test transaction commit, rollback, time ahead, time behind scenarios with replica, persist_to and replicate_to settings '''
# Atomicity.basic_ops.basic_ops.test_basic_commit
self.transaction_timeout = self.input.param("transaction_timeout", 100)
self.transaction_commit = self.input.param("transaction_commit", True)
self.drift_ahead = self.input.param("drift_ahead", False)
self.drift_behind = self.input.param("drift_behind", False)
gen_create = self.get_doc_generator(0, self.num_items)
self.op_type = self.input.param("op_type", 'create')
self.num_buckets = self.input.param("num_buckets", 0)
self.default_bucket = self.input.param("default_bucket", True)
self.standard_bucket = self.input.param("standard_bucket", 1)
self.exp = self.input.param("expiry", 0)
# print_ops_task = self.bucket_util.async_print_bucket_ops(def_bucket)
if self.default_bucket:
self.bucket_util.create_default_bucket(replica=self.num_replicas,
compression_mode=self.compression_mode, ram_quota=100)
if self.drift_ahead:
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue( shell.change_system_time(3600), 'Failed to advance the clock')
output,error = shell.execute_command('date')
self.log.info('Date after is set forward {0}'.format( output ))
if self.drift_behind:
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue( shell.change_system_time(-3600), 'Failed to advance the clock')
output,error = shell.execute_command('date')
self.log.info('Date after is set behind {0}'.format( output ))
if self.num_buckets:
self.bucket_util.create_multiple_buckets(self.cluster.master, self.num_replicas, bucket_count=self.num_buckets, bucket_type='ephemeral')
if self.standard_bucket:
self.bucket_util.create_standard_buckets(self.cluster.master, self.standard_buckets, bucket_size=100)
self.def_bucket= self.bucket_util.get_all_buckets()
# client = VBucketAwareMemcached(RestConnection(self.cluster.master), self.def_bucket)
#
# Transaction().nonTxnRemoves(client.collection)
print "going to create a task"
task = self.task.async_load_gen_docs_atomicity(self.cluster, self.def_bucket,
gen_create, self.op_type , exp=self.exp,
batch_size=10,
process_concurrency=8,
replicate_to=self.replicate_to,
persist_to=self.persist_to, timeout_secs=self.sdk_timeout,
retries=self.sdk_retries, transaction_timeout=self.transaction_timeout, commit=self.transaction_commit)
print "going to execute the task"
self.task.jython_task_manager.get_task_result(task)
def test_logical_clock_ticks(self):
self.log.info('starting test_logical_clock_ticks')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.servers[0].ip],
bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(
self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# do a bunch of mutations to set the max cas
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
base_total_logical_clock_ticks = 0
for i in range(self.vbuckets):
#print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']
base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The base total logical clock ticks is {0}'.format(
base_total_logical_clock_ticks))
# move the system clock back so the logical counter part of HLC is used and the logical clock ticks
# stat is incremented
self.assertTrue(
shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS),
'Failed to advance the clock')
# do more mutations
NUMBER_OF_MUTATIONS = 10000
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=NUMBER_OF_MUTATIONS)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
time.sleep(30)
total_logical_clock_ticks = 0
for i in range(self.vbuckets):
total_logical_clock_ticks = total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The total logical clock ticks is {0}'.format(
total_logical_clock_ticks))
self.assertTrue(
total_logical_clock_ticks -
base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS,
'Expected clock tick {0} actual {1}'.format(
NUMBER_OF_MUTATIONS,
total_logical_clock_ticks - base_total_logical_clock_ticks))
def test_logical_clock_ticks(self):
self.log.info("starting test_logical_clock_ticks")
sdk_client = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# do a bunch of mutations to set the max cas
gen_load = BlobGenerator("key-for-cas-test-logical-ticks", "value-for-cas-test-", self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats("vbucket-details")
base_total_logical_clock_ticks = 0
for i in range(self.vbuckets):
# print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']
base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(
vbucket_stats["vb_" + str(i) + ":logical_clock_ticks"]
)
self.log.info("The base total logical clock ticks is {0}".format(base_total_logical_clock_ticks))
# move the system clock back so the logical counter part of HLC is used and the logical clock ticks
# stat is incremented
self.assertTrue(shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to advance the clock")
# do more mutations
NUMBER_OF_MUTATIONS = 10000
gen_load = BlobGenerator(
"key-for-cas-test-logical-ticks", "value-for-cas-test-", self.value_size, end=NUMBER_OF_MUTATIONS
)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats("vbucket-details")
total_logical_clock_ticks = 0
for i in range(self.vbuckets):
total_logical_clock_ticks = total_logical_clock_ticks + int(
vbucket_stats["vb_" + str(i) + ":logical_clock_ticks"]
)
self.log.info("The total logical clock ticks is {0}".format(total_logical_clock_ticks))
self.assertTrue(
total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS,
"Expected clock tick {0} actual {1}".format(
NUMBER_OF_MUTATIONS, total_logical_clock_ticks - base_total_logical_clock_ticks
),
)
def test_logical_clock_ticks(self):
self.log.info('starting test_logical_clock_ticks')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase',hosts = [self.servers[0].ip], bucket = self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# do a bunch of mutations to set the max cas
gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
base_total_logical_clock_ticks = 0
for i in range(self.vbuckets):
#print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']
base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The base total logical clock ticks is {0}'.format( base_total_logical_clock_ticks))
# move the system clock back so the logical counter part of HLC is used and the logical clock ticks
# stat is incremented
self.assertTrue( shell.change_system_time( -LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to advance the clock')
# do more mutations
NUMBER_OF_MUTATIONS = 10000
gen_load = BlobGenerator('key-for-cas-test-logical-ticks', 'value-for-cas-test-', self.value_size, end=NUMBER_OF_MUTATIONS)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
time.sleep(30)
total_logical_clock_ticks = 0
for i in range(self.vbuckets):
total_logical_clock_ticks = total_logical_clock_ticks + int(vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The total logical clock ticks is {0}'.format( total_logical_clock_ticks))
self.assertTrue( total_logical_clock_ticks - base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS,
'Expected clock tick {0} actual {1}'.format(NUMBER_OF_MUTATIONS,
total_logical_clock_ticks- base_total_logical_clock_ticks ))
def test_poisoned_cas(self):
self.log.info("starting test_poisoned_cas")
"""
- set the clock ahead
- do lots of sets and get some CASs
- do a set and get the CAS (flag, CAS, value) and save it
- set the clock back
- verify the CAS is still big on new sets
- reset the CAS
- do the vbucket max cas and verify
- do a new mutation and verify the CAS is smaller
"""
sdk_client = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# move the system clock ahead to poison the CAS
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue(shell.change_system_time(LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to advance the clock")
output, error = shell.execute_command("date")
self.log.info("Date after is set forward {0}".format(output))
rc = sdk_client.set("key1", "val1")
rc = mc_client.get("key1")
poisoned_cas = rc[1]
self.log.info("The poisoned CAS is {0}".format(poisoned_cas))
# do lots of mutations to set the max CAS for all vbuckets
gen_load = BlobGenerator("key-for-cas-test", "value-for-cas-test-", self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
# move the clock back again and verify the CAS stays large
self.assertTrue(shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to change the clock")
output, error = shell.execute_command("date")
self.log.info("Date after is set backwards {0}".format(output))
use_mc_bin_client = self.input.param("use_mc_bin_client", False)
if use_mc_bin_client:
rc = mc_client.set("key2", 0, 0, "val2")
second_poisoned_cas = rc[1]
else:
rc = sdk_client.set("key2", "val2")
second_poisoned_cas = rc.cas
self.log.info("The second_poisoned CAS is {0}".format(second_poisoned_cas))
self.assertTrue(
second_poisoned_cas > poisoned_cas,
"Second poisoned CAS {0} is not larger than the first poisoned cas".format(
second_poisoned_cas, poisoned_cas
),
)
# reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not
# changed then the CAS will immediately continue with the clock. I see two scenarios:
# 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used
# 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use
# do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds
earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000
for i in range(self.vbuckets):
output, error = shell.execute_cbepctl(
self.buckets[0], "", "set_vbucket_param", "max_cas ", str(i) + " " + str(earlier_max_cas)
)
if len(error) > 0:
self.fail("Failed to set the max cas")
# verify the max CAS
for i in range(self.vbuckets):
max_cas = int(mc_client.stats("vbucket-details")["vb_" + str(i) + ":max_cas"])
self.assertTrue(
max_cas == earlier_max_cas,
"Max CAS not properly set for vbucket {0} set as {1} and observed {2}".format(
i, earlier_max_cas, max_cas
),
)
self.log.info("Per cbstats the max cas for bucket {0} is {1}".format(i, max_cas))
rc1 = sdk_client.set("key-after-resetting cas", "val1")
rc2 = mc_client.get("key-after-resetting cas")
set_cas_after_reset_max_cas = rc2[1]
self.log.info("The later CAS is {0}".format(set_cas_after_reset_max_cas))
self.assertTrue(
set_cas_after_reset_max_cas < poisoned_cas,
"For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format(
"key-after-resetting cas", set_cas_after_reset_max_cas, poisoned_cas
),
)
# do a bunch of sets and verify the CAS is small - this is really only one set, need to do more
gen_load = BlobGenerator(
"key-for-cas-test-after-cas-is-reset", "value-for-cas-test-", self.value_size, end=1000
)
self._load_all_buckets(self.master, gen_load, "create", 0)
gen_load.reset()
while gen_load.has_next():
key, value = gen_load.next()
try:
rc = mc_client.get(key)
# rc = sdk_client.get(key)
cas = rc[1]
self.assertTrue(
cas < poisoned_cas,
"For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format(
key, cas, poisoned_cas
),
)
except:
self.log.info("get error with {0}".format(key))
rc = sdk_client.set("key3", "val1")
better_cas = rc.cas
self.log.info("The better CAS is {0}".format(better_cas))
self.assertTrue(better_cas < poisoned_cas, "The CAS was not improved")
# set the clock way ahead - remote_util_OS.py (new)
# do a bunch of mutations - not really needed
# do the fix command - cbepctl, the existing way (remote util)
# do some mutations, verify they conform to the new CAS - build on the CAS code,
# where to iterate over the keys and get the CAS?
"""
def test_poisoned_cas(self):
"""
@note: - set the clock ahead
- do lots of sets and get some CASs
- do a set and get the CAS (flag, CAS, value) and save it
- set the clock back
- verify the CAS is still big on new sets
- reset the CAS
- do the vbucket max cas and verify
- do a new mutation and verify the CAS is smaller
"""
#creating a user 'default' for the bucket
self.log.info('starting test_poisoned_cas')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase', hosts = [self.servers[0].ip], bucket = self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
# move the system clock ahead to poison the CAS
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue( shell.change_system_time( LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to advance the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set forward {0}'.format( output ))
rc = sdk_client.set('key1', 'val1')
rc = mc_client.get('key1' )
poisoned_cas = rc[1]
self.log.info('The poisoned CAS is {0}'.format(poisoned_cas))
# do lots of mutations to set the max CAS for all vbuckets
gen_load = BlobGenerator('key-for-cas-test', 'value-for-cas-test-', self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
# move the clock back again and verify the CAS stays large
self.assertTrue( shell.change_system_time( -LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to change the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set backwards {0}'.format( output))
use_mc_bin_client = self.input.param("use_mc_bin_client", True)
if use_mc_bin_client:
rc = mc_client.set('key2', 0, 0, 'val2')
second_poisoned_cas = rc[1]
else:
rc = sdk_client.set('key2', 'val2')
second_poisoned_cas = rc.cas
self.log.info('The second_poisoned CAS is {0}'.format(second_poisoned_cas))
self.assertTrue( second_poisoned_cas > poisoned_cas,
'Second poisoned CAS {0} is not larger than the first poisoned cas'.format(second_poisoned_cas, poisoned_cas))
# reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not
# changed then the CAS will immediately continue with the clock. I see two scenarios:
# 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used
# 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use
# do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds
earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000
for i in range(self.vbuckets):
output, error = shell.execute_cbepctl(self.buckets[0], "", "set_vbucket_param",
"max_cas ", str(i) + ' ' + str(earlier_max_cas) )
if len(error) > 0:
self.fail('Failed to set the max cas')
# verify the max CAS
for i in range(self.vbuckets):
max_cas = int( mc_client.stats('vbucket-details')['vb_' + str(i) + ':max_cas'] )
self.assertTrue(max_cas == earlier_max_cas,
'Max CAS not properly set for vbucket {0} set as {1} and observed {2}'.format(i, earlier_max_cas, max_cas ) )
self.log.info('Per cbstats the max cas for bucket {0} is {1}'.format(i, max_cas) )
rc1 = sdk_client.set('key-after-resetting cas', 'val1')
rc2 = mc_client.get('key-after-resetting cas' )
set_cas_after_reset_max_cas = rc2[1]
self.log.info('The later CAS is {0}'.format(set_cas_after_reset_max_cas))
self.assertTrue( set_cas_after_reset_max_cas < poisoned_cas,
'For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format('key-after-resetting cas', set_cas_after_reset_max_cas, poisoned_cas))
# do a bunch of sets and verify the CAS is small - this is really only one set, need to do more
gen_load = BlobGenerator('key-for-cas-test-after-cas-is-reset', 'value-for-cas-test-', self.value_size, end=1000)
self._load_all_buckets(self.master, gen_load, "create", 0)
gen_load.reset()
while gen_load.has_next():
key, value = next(gen_load)
try:
rc = mc_client.get( key )
#rc = sdk_client.get(key)
cas = rc[1]
self.assertTrue( cas < poisoned_cas, 'For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format(key, cas, poisoned_cas))
except:
self.log.info('get error with {0}'.format(key))
rc = sdk_client.set('key3', 'val1')
better_cas = rc.cas
self.log.info('The better CAS is {0}'.format(better_cas))
self.assertTrue( better_cas < poisoned_cas, 'The CAS was not improved')
# set the clock way ahead - remote_util_OS.py (new)
# do a bunch of mutations - not really needed
# do the fix command - cbepctl, the existing way (remote util)
# do some mutations, verify they conform to the new CAS - build on the CAS code,
# where to iterate over the keys and get the CAS?
"""
