class CCCP(BaseTestCase):
def setUp(self):
super(CCCP, self).setUp()
self.map_fn = 'function (doc){emit([doc.join_yr, doc.join_mo],doc.name);}'
self.ddoc_name = "cccp_ddoc"
self.view_name = "cccp_view"
self.default_view = View(self.view_name, self.map_fn, None, False)
self.ops = self.input.param("ops", None)
self.clients = {}
try:
for bucket in self.buckets:
self.clients[bucket.name] =\
MemcachedClientHelper.direct_client(self.master, bucket.name)
except:
self.tearDown()
def tearDown(self):
super(CCCP, self).tearDown()
def test_get_config_client(self):
tasks = self.run_ops()
for task in tasks:
if self.ops != 'failover':
task.result()
for bucket in self.buckets:
_, _, config = self.clients[bucket.name].get_config()
self.verify_config(json.loads(config), bucket)
def test_get_config_rest(self):
tasks = self.run_ops()
for task in tasks:
if not task:
self.fail("no task to run")
task.result()
for bucket in self.buckets:
config = RestConnection(self.master).get_bucket_CCCP(bucket)
self.verify_config(config, bucket)
def test_set_config(self):
tasks = self.run_ops()
config_expected = 'abcabc'
for task in tasks:
task.result()
for bucket in self.buckets:
self.clients[bucket.name].set_config(config_expected)
_, _, config = self.clients[bucket.name].get_config()
self.assertEquals(config_expected, config, "Expected config: %s, actual %s" %(
config_expected, config))
self.log.info("Config was set correctly. Bucket %s" % bucket.name)
def test_not_my_vbucket_config(self):
self.gen_load = BlobGenerator('cccp', 'cccp-', self.value_size, end=self.num_items)
self._load_all_buckets(self.master, self.gen_load, "create", 0)
self.cluster.rebalance(self.servers[:self.nodes_init],
self.servers[self.nodes_init:self.nodes_init + 1], [])
self.nodes_init = self.nodes_init + 1
not_my_vbucket = False
for bucket in self.buckets:
while self.gen_load.has_next() and not not_my_vbucket:
key, _ = self.gen_load.next()
try:
self.clients[bucket.name].get(key)
except Exception, ex:
self.log.info("Config in exception is correct. Bucket %s, key %s"\
% (bucket.name, key))
config = str(ex)[str(ex).find("Not my vbucket':") \
+ 16 : str(ex).find("for vbucket")]
config = json.loads(config)
self.verify_config(config, bucket)
""" from watson, only the first error contains bucket details """
not_my_vbucket = True
class CCCP(BaseTestCase):
def setUp(self):
super(CCCP, self).setUp()
self.map_fn = 'function (doc){emit([doc.join_yr, doc.join_mo],doc.name);}'
self.ddoc_name = "cccp_ddoc"
self.view_name = "cccp_view"
self.default_view = View(self.view_name, self.map_fn, None, False)
self.ops = self.input.param("ops", None)
self.clients = {}
try:
for bucket in self.buckets:
self.clients[bucket.name] =\
MemcachedClientHelper.direct_client(self.master, bucket.name)
except:
self.tearDown()
def tearDown(self):
super(CCCP, self).tearDown()
def test_get_config_client(self):
tasks = self.run_ops()
for task in tasks:
if self.ops != 'failover':
task.result()
for bucket in self.buckets:
_, _, config = self.clients[bucket.name].get_config()
self.verify_config(json.loads(config), bucket)
def test_get_config_rest(self):
tasks = self.run_ops()
for task in tasks:
if not task:
self.fail("no task to run")
if self.ops == 'failover':
if not task:
self.fail("Ops failover failed ")
else:
task.result()
for bucket in self.buckets:
config = RestConnection(self.master).get_bucket_CCCP(bucket)
self.verify_config(config, bucket)
def test_set_config(self):
""" Negative test for setting bucket config. """
tasks = self.run_ops()
config_expected = 'abcabc'
for task in tasks:
task.result()
for bucket in self.buckets:
try:
self.clients[bucket.name].set_config(config_expected)
_, _, config = self.clients[bucket.name].get_config()
if econfig_expected == config:
self.fail("It should not allow to set this format config ")
except Exception as e:
if e and not "Memcached error #4 'Invalid'" in str(e):
self.fail(
"ns server should not allow to set this config format")
def test_not_my_vbucket_config(self):
self.gen_load = BlobGenerator('cccp',
'cccp-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, self.gen_load, "create", 0)
self.cluster.rebalance(
self.servers[:self.nodes_init],
self.servers[self.nodes_init:self.nodes_init + 1], [])
self.nodes_init = self.nodes_init + 1
not_my_vbucket = False
for bucket in self.buckets:
while self.gen_load.has_next() and not not_my_vbucket:
key, _ = self.gen_load.next()
try:
self.clients[bucket.name].get(key)
except Exception, ex:
self.log.info("Config in exception is correct. Bucket %s, key %s"\
% (bucket.name, key))
config = str(ex)[str(ex).find("Not my vbucket':") \
+ 16 : str(ex).find("for vbucket")]
config = json.loads(config)
self.verify_config(config, bucket)
""" from watson, only the first error contains bucket details """
not_my_vbucket = True
def test_poisoned_cas(self):
self.log.info("starting test_poisoned_cas")
"""
- set the clock ahead
- do lots of sets and get some CASs
- do a set and get the CAS (flag, CAS, value) and save it
- set the clock back
- verify the CAS is still big on new sets
- reset the CAS
- do the vbucket max cas and verify
- do a new mutation and verify the CAS is smaller
"""
sdk_client = SDKClient(scheme="couchbase", hosts=[self.servers[0].ip], bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# move the system clock ahead to poison the CAS
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue(shell.change_system_time(LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to advance the clock")
output, error = shell.execute_command("date")
self.log.info("Date after is set forward {0}".format(output))
rc = sdk_client.set("key1", "val1")
rc = mc_client.get("key1")
poisoned_cas = rc[1]
self.log.info("The poisoned CAS is {0}".format(poisoned_cas))
# do lots of mutations to set the max CAS for all vbuckets
gen_load = BlobGenerator("key-for-cas-test", "value-for-cas-test-", self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
# move the clock back again and verify the CAS stays large
self.assertTrue(shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS), "Failed to change the clock")
output, error = shell.execute_command("date")
self.log.info("Date after is set backwards {0}".format(output))
use_mc_bin_client = self.input.param("use_mc_bin_client", False)
if use_mc_bin_client:
rc = mc_client.set("key2", 0, 0, "val2")
second_poisoned_cas = rc[1]
else:
rc = sdk_client.set("key2", "val2")
second_poisoned_cas = rc.cas
self.log.info("The second_poisoned CAS is {0}".format(second_poisoned_cas))
self.assertTrue(
second_poisoned_cas > poisoned_cas,
"Second poisoned CAS {0} is not larger than the first poisoned cas".format(
second_poisoned_cas, poisoned_cas
),
)
# reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not
# changed then the CAS will immediately continue with the clock. I see two scenarios:
# 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used
# 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use
# do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds
earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000
for i in range(self.vbuckets):
output, error = shell.execute_cbepctl(
self.buckets[0], "", "set_vbucket_param", "max_cas ", str(i) + " " + str(earlier_max_cas)
)
if len(error) > 0:
self.fail("Failed to set the max cas")
# verify the max CAS
for i in range(self.vbuckets):
max_cas = int(mc_client.stats("vbucket-details")["vb_" + str(i) + ":max_cas"])
self.assertTrue(
max_cas == earlier_max_cas,
"Max CAS not properly set for vbucket {0} set as {1} and observed {2}".format(
i, earlier_max_cas, max_cas
),
)
self.log.info("Per cbstats the max cas for bucket {0} is {1}".format(i, max_cas))
rc1 = sdk_client.set("key-after-resetting cas", "val1")
rc2 = mc_client.get("key-after-resetting cas")
set_cas_after_reset_max_cas = rc2[1]
self.log.info("The later CAS is {0}".format(set_cas_after_reset_max_cas))
self.assertTrue(
set_cas_after_reset_max_cas < poisoned_cas,
"For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format(
"key-after-resetting cas", set_cas_after_reset_max_cas, poisoned_cas
),
)
# do a bunch of sets and verify the CAS is small - this is really only one set, need to do more
gen_load = BlobGenerator(
"key-for-cas-test-after-cas-is-reset", "value-for-cas-test-", self.value_size, end=1000
)
self._load_all_buckets(self.master, gen_load, "create", 0)
gen_load.reset()
while gen_load.has_next():
key, value = gen_load.next()
try:
rc = mc_client.get(key)
# rc = sdk_client.get(key)
cas = rc[1]
self.assertTrue(
cas < poisoned_cas,
"For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}".format(
key, cas, poisoned_cas
),
)
except:
self.log.info("get error with {0}".format(key))
rc = sdk_client.set("key3", "val1")
better_cas = rc.cas
self.log.info("The better CAS is {0}".format(better_cas))
self.assertTrue(better_cas < poisoned_cas, "The CAS was not improved")
# set the clock way ahead - remote_util_OS.py (new)
# do a bunch of mutations - not really needed
# do the fix command - cbepctl, the existing way (remote util)
# do some mutations, verify they conform to the new CAS - build on the CAS code,
# where to iterate over the keys and get the CAS?
"""
def test_poisoned_cas(self):
"""
@note: - set the clock ahead
- do lots of sets and get some CASs
- do a set and get the CAS (flag, CAS, value) and save it
- set the clock back
- verify the CAS is still big on new sets
- reset the CAS
- do the vbucket max cas and verify
- do a new mutation and verify the CAS is smaller
"""
#creating a user 'default' for the bucket
self.log.info('starting test_poisoned_cas')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase', hosts = [self.servers[0].ip], bucket = self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
# move the system clock ahead to poison the CAS
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue( shell.change_system_time( LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to advance the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set forward {0}'.format( output ))
rc = sdk_client.set('key1', 'val1')
rc = mc_client.get('key1' )
poisoned_cas = rc[1]
self.log.info('The poisoned CAS is {0}'.format(poisoned_cas))
# do lots of mutations to set the max CAS for all vbuckets
gen_load = BlobGenerator('key-for-cas-test', 'value-for-cas-test-', self.value_size, end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
# move the clock back again and verify the CAS stays large
self.assertTrue( shell.change_system_time( -LWWStatsTests.ONE_HOUR_IN_SECONDS ), 'Failed to change the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set backwards {0}'.format( output))
use_mc_bin_client = self.input.param("use_mc_bin_client", True)
if use_mc_bin_client:
rc = mc_client.set('key2', 0, 0, 'val2')
second_poisoned_cas = rc[1]
else:
rc = sdk_client.set('key2', 'val2')
second_poisoned_cas = rc.cas
self.log.info('The second_poisoned CAS is {0}'.format(second_poisoned_cas))
self.assertTrue( second_poisoned_cas > poisoned_cas,
'Second poisoned CAS {0} is not larger than the first poisoned cas'.format(second_poisoned_cas, poisoned_cas))
# reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not
# changed then the CAS will immediately continue with the clock. I see two scenarios:
# 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used
# 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use
# do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds
earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000
for i in range(self.vbuckets):
output, error = shell.execute_cbepctl(self.buckets[0], "", "set_vbucket_param",
"max_cas ", str(i) + ' ' + str(earlier_max_cas) )
if len(error) > 0:
self.fail('Failed to set the max cas')
# verify the max CAS
for i in range(self.vbuckets):
max_cas = int( mc_client.stats('vbucket-details')['vb_' + str(i) + ':max_cas'] )
self.assertTrue(max_cas == earlier_max_cas,
'Max CAS not properly set for vbucket {0} set as {1} and observed {2}'.format(i, earlier_max_cas, max_cas ) )
self.log.info('Per cbstats the max cas for bucket {0} is {1}'.format(i, max_cas) )
rc1 = sdk_client.set('key-after-resetting cas', 'val1')
rc2 = mc_client.get('key-after-resetting cas' )
set_cas_after_reset_max_cas = rc2[1]
self.log.info('The later CAS is {0}'.format(set_cas_after_reset_max_cas))
self.assertTrue( set_cas_after_reset_max_cas < poisoned_cas,
'For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format('key-after-resetting cas', set_cas_after_reset_max_cas, poisoned_cas))
# do a bunch of sets and verify the CAS is small - this is really only one set, need to do more
gen_load = BlobGenerator('key-for-cas-test-after-cas-is-reset', 'value-for-cas-test-', self.value_size, end=1000)
self._load_all_buckets(self.master, gen_load, "create", 0)
gen_load.reset()
while gen_load.has_next():
key, value = next(gen_load)
try:
rc = mc_client.get( key )
#rc = sdk_client.get(key)
cas = rc[1]
self.assertTrue( cas < poisoned_cas, 'For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.format(key, cas, poisoned_cas))
except:
self.log.info('get error with {0}'.format(key))
rc = sdk_client.set('key3', 'val1')
better_cas = rc.cas
self.log.info('The better CAS is {0}'.format(better_cas))
self.assertTrue( better_cas < poisoned_cas, 'The CAS was not improved')
# set the clock way ahead - remote_util_OS.py (new)
# do a bunch of mutations - not really needed
# do the fix command - cbepctl, the existing way (remote util)
# do some mutations, verify they conform to the new CAS - build on the CAS code,
# where to iterate over the keys and get the CAS?
"""
class CCCP(BaseTestCase):
def setUp(self):
super(CCCP, self).setUp()
self.map_fn = 'function (doc){emit([doc.join_yr, doc.join_mo],doc.name);}'
self.ddoc_name = "cccp_ddoc"
self.view_name = "cccp_view"
self.default_view = View(self.view_name, self.map_fn, None, False)
self.ops = self.input.param("ops", None)
self.clients = {}
try:
for bucket in self.buckets:
self.clients[bucket.name] =\
MemcachedClientHelper.direct_client(self.master, bucket.name)
except:
self.tearDown()
def tearDown(self):
super(CCCP, self).tearDown()
def test_get_config_client(self):
tasks = self.run_ops()
for task in tasks:
if self.ops != 'failover':
task.result()
for bucket in self.buckets:
_, _, config = self.clients[bucket.name].get_config()
self.verify_config(json.loads(config), bucket)
def test_get_config_rest(self):
tasks = self.run_ops()
for task in tasks:
if not task:
self.fail("no task to run")
if self.ops == 'failover':
if not task:
self.fail("Ops failover failed ")
else:
task.result()
for bucket in self.buckets:
config = RestConnection(self.master).get_bucket_CCCP(bucket)
self.verify_config(config, bucket)
def test_set_config(self):
""" Negative test for setting bucket config. """
tasks = self.run_ops()
config_expected = 'abcabc'
for task in tasks:
task.result()
for bucket in self.buckets:
try:
self.clients[bucket.name].set_config(config_expected)
_, _, config = self.clients[bucket.name].get_config()
if econfig_expected == config:
self.fail("It should not allow to set this format config ")
except Exception as e:
if e and not "Memcached error #4 'Invalid'" in str(e):
self.fail(
"ns server should not allow to set this config format")
def test_not_my_vbucket_config(self):
self.gen_load = BlobGenerator('cccp',
'cccp-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, self.gen_load, "create", 0)
self.cluster.rebalance(
self.servers[:self.nodes_init],
self.servers[self.nodes_init:self.nodes_init + 1], [])
self.nodes_init = self.nodes_init + 1
not_my_vbucket = False
for bucket in self.buckets:
while self.gen_load.has_next() and not not_my_vbucket:
key, _ = next(self.gen_load)
try:
self.clients[bucket.name].get(key)
except Exception as ex:
self.log.info("Config in exception is correct. Bucket %s, key %s"\
% (bucket.name, key))
config = str(ex)[str(ex).find("Not my vbucket':") \
+ 16 : str(ex).find("for vbucket")]
if not config.endswith("}"):
config += "}"
try:
config = json.loads(config)
except Exception as e:
if "Expecting object" in str(e):
config += "}"
config = json.loads(config)
self.verify_config(config, bucket)
""" from watson, only the first error contains bucket details """
not_my_vbucket = True
def verify_config(self, config_json, bucket):
expected_params = [
"nodeLocator", "rev", "uuid", "bucketCapabilitiesVer",
"bucketCapabilities"
]
print("-->config_json:{}".format(config_json))
for param in expected_params:
self.assertTrue(param in config_json, "No %s in config" % param)
self.assertTrue(
"name" in config_json and config_json["name"] == bucket.name,
"No bucket name in config")
if self.cb_version and self.cb_version[:5] in COUCHBASE_FROM_VERSION_4:
self.assertTrue(
len(config_json["nodesExt"]) == self.nodes_init,
"Number of nodes expected %s, actual %s" %
(self.nodes_init, len(config_json["nodesExt"])))
else:
self.assertTrue(
len(config_json["nodes"]) == self.nodes_init,
"Number of nodes expected %s, actual %s" %
(self.nodes_init, len(config_json["nodes"])))
for node in config_json["nodes"]:
self.assertTrue("couchApiBase" in node and "hostname" in node,
"No hostname name in config")
if self.cb_version and self.cb_version[:
5] in COUCHBASE_FROM_MAD_HATTER:
""" moxi port is removed from Mad-Hatter 6.5.0 """
self.assertTrue(node["ports"]["direct"] == 11210,
"ports are incorrect: %s" % node)
else:
self.assertTrue(node["ports"]["proxy"] == 11211 and \
node["ports"]["direct"] == 11210,
"ports are incorrect: %s" % node)
self.assertTrue(config_json["ddocs"]["uri"] == \
("/pools/default/buckets/%s/ddocs" % bucket.name),
"Ddocs uri is incorrect: %s "
% "/pools/default/buckets/default/ddocs")
self.assertTrue(
config_json["vBucketServerMap"]["numReplicas"] ==
self.num_replicas, "Num replicas is incorrect: %s " %
config_json["vBucketServerMap"]["numReplicas"])
for param in ["hashAlgorithm", "serverList", "vBucketMap"]:
self.assertTrue(param in config_json["vBucketServerMap"],
"%s in vBucketServerMap" % param)
self.log.info("Bucket %s .Config was checked" % bucket.name)
def run_ops(self):
tasks = []
if not self.ops:
return tasks
if self.ops == 'rebalance_in':
tasks.append(
self.cluster.async_rebalance(
self.servers[:self.nodes_init],
self.servers[self.nodes_init:self.nodes_init +
self.nodes_in], []))
self.nodes_init += self.nodes_in
elif self.ops == 'rebalance_out':
tasks.append(
self.cluster.async_rebalance(
self.servers[:self.nodes_init], [],
self.servers[(self.nodes_init -
self.nodes_out):self.nodes_init]))
self.nodes_init -= self.nodes_out
elif self.ops == 'failover':
tasks.append(
self.cluster.failover(
self.servers[:self.nodes_init],
self.servers[(self.nodes_init -
self.nodes_out):self.nodes_init]))
self.sleep(20)
self.nodes_init -= self.nodes_out
if self.ops == 'create_views':
views_num = 10
views = self.make_default_views(self.view_name,
views_num,
different_map=True)
tasks.extend(
self.async_create_views(self.master, self.ddoc_name, views))
if self.ops == 'restart':
servers_to_choose = [
serv for serv in self.servers if self.master.ip != serv.ip
]
self.assertTrue(servers_to_choose,
"There is only one node in cluster")
shell = RemoteMachineShellConnection(servers_to_choose[0])
try:
shell.stop_couchbase()
shell.start_couchbase()
finally:
shell.disconnect()
self.sleep(5, "Server %s is starting..." % servers_to_choose[0].ip)
return tasks
class CCCP(BaseTestCase):
def setUp(self):
super(CCCP, self).setUp()
self.map_fn = 'function (doc){emit([doc.join_yr, doc.join_mo],doc.name);}'
self.ddoc_name = "cccp_ddoc"
self.view_name = "cccp_view"
self.default_view = View(self.view_name, self.map_fn, None, False)
self.ops = self.input.param("ops", None)
self.clients = {}
try:
for bucket in self.buckets:
self.clients[bucket.name] =\
MemcachedClientHelper.direct_client(self.master, bucket.name)
except:
self.tearDown()
def tearDown(self):
super(CCCP, self).tearDown()
def test_get_config_client(self):
tasks = self.run_ops()
for bucket in self.buckets:
_, _, config = self.clients[bucket.name].get_config()
self.verify_config(json.loads(config), bucket)
for task in tasks:
task.result()
def test_get_config_rest(self):
tasks = self.run_ops()
for bucket in self.buckets:
config = RestConnection(self.master).get_bucket_CCCP(bucket)
self.verify_config(config, bucket)
for task in tasks:
task.result()
def test_set_config(self):
tasks = self.run_ops()
config_expected = 'abcabc'
for bucket in self.buckets:
self.clients[bucket.name].set_config(config_expected)
_, _, config = self.clients[bucket.name].get_config()
self.assertEquals(
config_expected, config,
"Expected config: %s, actual %s" % (config_expected, config))
self.log.info("Config was set correctly. Bucket %s" % bucket.name)
for task in tasks:
task.result()
def test_not_my_vbucket_config(self):
self.gen_load = BlobGenerator('cccp',
'cccp-',
self.value_size,
end=self.num_items)
self._load_all_buckets(self.master, self.gen_load, "create", 0)
self.cluster.rebalance(
self.servers[:self.nodes_init],
self.servers[self.nodes_init:self.nodes_init + 1], [])
self.nodes_init = self.nodes_init + 1
for bucket in self.buckets:
while self.gen_load.has_next():
key, _ = self.gen_load.next()
try:
self.clients[bucket.name].get(key)
except Exception, ex:
self.log.info(
"Config in exception is correct. Bucket %s, key %s" %
(bucket.name, key))
config = str(ex)[str(ex).find("Not my vbucket':") +
16:str(ex).find("for vbucket")]
config = json.loads(config)
self.verify_config(config, bucket)
