def create_user(self):
rest = RestConnection(self.host)
response = rest.add_set_builtin_user(self.user_id,self.payload)
return response
def create_user(self):
rest = RestConnection(self.host)
response = rest.add_set_builtin_user(self.user_id, self.payload)
return response
def setUp(self):
super(OnPremBaseTest, self).setUp()
# Framework specific parameters (Extension from cb_basetest)
self.skip_cluster_reset = self.input.param("skip_cluster_reset", False)
self.skip_setup_cleanup = self.input.param("skip_setup_cleanup", False)
# End of framework parameters
# Cluster level info settings
self.log_info = self.input.param("log_info", None)
self.log_location = self.input.param("log_location", None)
self.stat_info = self.input.param("stat_info", None)
self.port = self.input.param("port", None)
self.port_info = self.input.param("port_info", None)
self.servers = self.input.servers
self.num_servers = self.input.param("servers", len(self.servers))
self.vbuckets = self.input.param("vbuckets", CbServer.total_vbuckets)
self.gsi_type = self.input.param("gsi_type", 'plasma')
# Memory quota settings
# Max memory quota to utilize per node
self.quota_percent = self.input.param("quota_percent", 100)
# Services' RAM quota to set on cluster
self.kv_mem_quota_percent = self.input.param("kv_quota_percent", None)
self.index_mem_quota_percent = \
self.input.param("index_quota_percent", None)
self.fts_mem_quota_percent = \
self.input.param("fts_quota_percent", None)
self.cbas_mem_quota_percent = \
self.input.param("cbas_quota_percent", None)
self.eventing_mem_quota_percent = \
self.input.param("eventing_quota_percent", None)
# CBAS setting
self.jre_path = self.input.param("jre_path", None)
self.enable_dp = self.input.param("enable_dp", False)
# End of cluster info parameters
# Bucket specific params
# Note: Over riding bucket_eviction_policy from CouchbaseBaseTest
self.bucket_eviction_policy = \
self.input.param("bucket_eviction_policy",
Bucket.EvictionPolicy.VALUE_ONLY)
self.bucket_replica_index = self.input.param("bucket_replica_index", 1)
if self.bucket_storage == Bucket.StorageBackend.magma:
self.bucket_eviction_policy = Bucket.EvictionPolicy.FULL_EVICTION
# End of bucket parameters
self.services_in = self.input.param("services_in", None)
self.forceEject = self.input.param("forceEject", False)
self.wait_timeout = self.input.param("wait_timeout", 120)
self.verify_unacked_bytes = \
self.input.param("verify_unacked_bytes", False)
self.disabled_consistent_view = \
self.input.param("disabled_consistent_view", None)
self.rebalanceIndexWaitingDisabled = \
self.input.param("rebalanceIndexWaitingDisabled", None)
self.rebalanceIndexPausingDisabled = \
self.input.param("rebalanceIndexPausingDisabled", None)
self.maxParallelIndexers = \
self.input.param("maxParallelIndexers", None)
self.maxParallelReplicaIndexers = \
self.input.param("maxParallelReplicaIndexers", None)
self.use_https = self.input.param("use_https", False)
self.enforce_tls = self.input.param("enforce_tls", False)
self.ipv4_only = self.input.param("ipv4_only", False)
self.ipv6_only = self.input.param("ipv6_only", False)
self.multiple_ca = self.input.param("multiple_ca", False)
if self.use_https:
CbServer.use_https = True
trust_all_certs()
self.node_utils.cleanup_pcaps(self.servers)
self.collect_pcaps = self.input.param("collect_pcaps", False)
if self.collect_pcaps:
self.node_utils.start_collect_pcaps(self.servers)
'''
Be careful while using this flag.
This is only and only for stand-alone tests.
During bugs reproductions, when a crash is seen
stop_server_on_crash will stop the server
so that we can collect data/logs/dumps at the right time
'''
self.stop_server_on_crash = self.input.param("stop_server_on_crash",
False)
self.collect_data = self.input.param("collect_data", False)
self.validate_system_event_logs = \
self.input.param("validate_sys_event_logs", False)
self.nonroot = False
self.crash_warning = self.input.param("crash_warning", False)
# Populate memcached_port in case of cluster_run
cluster_run_base_port = ClusterRun.port
if int(self.input.servers[0].port) == ClusterRun.port:
for server in self.input.servers:
server.port = cluster_run_base_port
cluster_run_base_port += 1
# If not defined in node.ini under 'memcached_port' section
if server.memcached_port is CbServer.memcached_port:
server.memcached_port = \
ClusterRun.memcached_port \
+ (2 * (int(server.port) - ClusterRun.port))
self.log_setup_status(self.__class__.__name__, "started")
cluster_name_format = "C%s"
default_cluster_index = counter_index = 1
if len(self.input.clusters) > 1:
# Multi cluster setup
for _, nodes in self.input.clusters.iteritems():
cluster_name = cluster_name_format % counter_index
tem_cluster = CBCluster(name=cluster_name,
servers=nodes,
vbuckets=self.vbuckets)
self.cb_clusters[cluster_name] = tem_cluster
counter_index += 1
else:
# Single cluster
cluster_name = cluster_name_format % counter_index
self.cb_clusters[cluster_name] = CBCluster(name=cluster_name,
servers=self.servers,
vbuckets=self.vbuckets)
# Initialize self.cluster with first available cluster as default
self.cluster = self.cb_clusters[cluster_name_format %
default_cluster_index]
self.cluster_util = ClusterUtils(self.task_manager)
self.bucket_util = BucketUtils(self.cluster_util, self.task)
CbServer.enterprise_edition = \
self.cluster_util.is_enterprise_edition(self.cluster)
if CbServer.enterprise_edition:
self.cluster.edition = "enterprise"
else:
self.cluster.edition = "community"
if self.standard_buckets > 10:
self.bucket_util.change_max_buckets(self.cluster.master,
self.standard_buckets)
for cluster_name, cluster in self.cb_clusters.items():
# Append initial master node to the nodes_in_cluster list
cluster.nodes_in_cluster.append(cluster.master)
shell = RemoteMachineShellConnection(cluster.master)
self.os_info = shell.extract_remote_info().type.lower()
if self.os_info != 'windows':
if cluster.master.ssh_username != "root":
self.nonroot = True
shell.disconnect()
break
shell.disconnect()
self.log_setup_status("OnPremBaseTest", "started")
try:
# Construct dict of mem. quota percent / mb per service
mem_quota_percent = dict()
# Construct dict of mem. quota percent per service
if self.kv_mem_quota_percent:
mem_quota_percent[CbServer.Services.KV] = \
self.kv_mem_quota_percent
if self.index_mem_quota_percent:
mem_quota_percent[CbServer.Services.INDEX] = \
self.index_mem_quota_percent
if self.cbas_mem_quota_percent:
mem_quota_percent[CbServer.Services.CBAS] = \
self.cbas_mem_quota_percent
if self.fts_mem_quota_percent:
mem_quota_percent[CbServer.Services.FTS] = \
self.fts_mem_quota_percent
if self.eventing_mem_quota_percent:
mem_quota_percent[CbServer.Services.EVENTING] = \
self.eventing_mem_quota_percent
if not mem_quota_percent:
mem_quota_percent = None
if self.skip_setup_cleanup:
# Update current server/service map and buckets for the cluster
for _, cluster in self.cb_clusters.items():
self.cluster_util.update_cluster_nodes_service_list(
cluster)
cluster.buckets = self.bucket_util.get_all_buckets(cluster)
return
else:
for cluster_name, cluster in self.cb_clusters.items():
self.log.info("Delete all buckets and rebalance out "
"other nodes from '%s'" % cluster_name)
self.cluster_util.cluster_cleanup(cluster,
self.bucket_util)
reload(Cb_constants)
# avoid clean up if the previous test has been tear down
if self.case_number == 1 or self.case_number > 1000:
if self.case_number > 1000:
self.log.warn("TearDown for prev test failed. Will retry")
self.case_number -= 1000
self.tearDownEverything(reset_cluster_env_vars=False)
for cluster_name, cluster in self.cb_clusters.items():
if not self.skip_cluster_reset:
self.initialize_cluster(
cluster_name,
cluster,
services=None,
services_mem_quota_percent=mem_quota_percent)
# Update initial service map for the master node
self.cluster_util.update_cluster_nodes_service_list(cluster)
# Set this unconditionally
RestConnection(cluster.master).set_internalSetting(
"magmaMinMemoryQuota", 256)
# Enable dp_version since we need collections enabled
if self.enable_dp:
tasks = []
for server in self.cluster.servers:
task = self.node_utils.async_enable_dp(server)
tasks.append(task)
for task in tasks:
self.task_manager.get_task_result(task)
# Enforce tls on nodes of all clusters
if self.use_https and self.enforce_tls:
for _, cluster in self.cb_clusters.items():
tasks = []
for node in cluster.servers:
task = self.node_utils.async_enable_tls(node)
tasks.append(task)
for task in tasks:
self.task_manager.get_task_result(task)
self.log.info(
"Validating if services obey tls only on servers {0}".
format(cluster.servers))
status = self.cluster_util.check_if_services_obey_tls(
cluster.servers)
if not status:
self.fail("Services did not honor enforce tls")
# Enforce IPv4 or IPv6 or both
if self.ipv4_only or self.ipv6_only:
for _, cluster in self.cb_clusters.items():
status, msg = self.cluster_util.enable_disable_ip_address_family_type(
cluster, True, self.ipv4_only, self.ipv6_only)
if not status:
self.fail(msg)
self.standard = self.input.param("standard", "pkcs8")
self.passphrase_type = self.input.param("passphrase_type",
"script")
self.encryption_type = self.input.param("encryption_type",
"aes256")
if self.multiple_ca:
for _, cluster in self.cb_clusters.items():
cluster.x509 = x509main(
host=cluster.master,
standard=self.standard,
encryption_type=self.encryption_type,
passphrase_type=self.passphrase_type)
self.generate_and_upload_cert(cluster.servers,
cluster.x509,
upload_root_certs=True,
upload_node_certs=True,
upload_client_certs=True)
payload = "name=cbadminbucket&roles=admin&password=password"
rest = RestConnection(cluster.master)
rest.add_set_builtin_user("cbadminbucket", payload)
for cluster_name, cluster in self.cb_clusters.items():
self.modify_cluster_settings(cluster)
# Track test start time only if we need system log validation
if self.validate_system_event_logs:
self.system_events.set_test_start_time()
self.log_setup_status("OnPremBaseTest", "finished")
self.__log("started")
except Exception as e:
traceback.print_exc()
self.task.shutdown(force=True)
self.fail(e)
finally:
# Track test start time only if we need system log validation
if self.validate_system_event_logs:
self.system_events.set_test_start_time()
self.log_setup_status("OnPremBaseTest", "finished")
class LimitTest(ClusterSetup):
def setUp(self):
super(LimitTest, self).setUp()
self.no_of_users = self.input.param("users", 3)
# The resource we're testing
self.resource_name = self.input.param(
"resource_name", "ns_server_num_concurrent_requests")
# The threshold at which the limit is configured at
self.resource_limit = self.input.param("resource_limit", 100)
# Extra resources - a list of key:value pairs separated by dashes
self.extra_resources = self.input.param("extra_resources", "")
self.extra_resources = dict(
item.split(":") for item in self.extra_resources.split("-")
if item)
# The absolute error that is allowed in validation
self.error = self.input.param("error", 5)
# The difference between throughput changes
self.throughput_difference = self.input.param("throughput_difference",
50)
# The retry timeout
self.retry_timeout = self.input.param("retry_timeout", 5)
# A multiplier for the throughput producer (e.g. use 1 << 20 for
# mebibytes)
self.units = self.input.param("units", 1 << 20)
# A rest connection
self.rest = RestConnection(self.cluster.master)
# The name of the bucket
self.bucket_name = "default"
# Create bucket
self.fragmentationPercentage = self.input.param(
"fragmentationPercentage", False)
bucket_params = Bucket({
"name":
self.bucket_name,
"fragmentationPercentage":
self.fragmentationPercentage
})
self.bucket_util.create_bucket(self.cluster, bucket_params)
# Create users
self.users = self.create_users()
# Some hints for controlling the user resource tasks
self.hints = {}
# A streaming uri for http long polling
self.hints["streaminguri"] = RestConnection(
self.cluster.master).get_path("pools/default/buckets/default",
"streamingUri")
# A factory-esque object that produces objects for the given resource
self.resource_producer = UserResourceProducer(self.resource_name,
self.hints)
# Create throughput tasks from test params
self.tasks = self.create_tasks()
def create_users(self):
""" Creates users """
users = [User("user{}".format(i)) for i in range(self.no_of_users)]
for user in users:
self.rest.add_set_builtin_user(user.username, user.params())
return users
def create_tasks(self):
""" Creates resource tasks """
tasks = []
for user in self.users:
for node in self.cluster.servers[:self.nodes_init]:
tasks.append(
self.resource_producer.get_resource_task(
user, copy_node_for_user(user, node)))
return tasks
def run_compaction(self):
if self.fragmentationPercentage:
self.bucket_util._run_compaction(self.cluster, number_of_times=2)
def remove_user_and_task(self, user):
""" Removes a user and their associated task """
self.users.remove(user)
self.tasks = [task for task in self.tasks if task.user != user]
self.rest.delete_builtin_user(user.username)
def insert_user_and_task(self, user):
""" Inserts a user and their associated task """
self.users.append(user)
self.rest.add_set_builtin_user(user.username, user.params())
for node in self.cluster.servers[:self.nodes_init]:
self.tasks.append(
self.resource_producer.get_resource_task(
user, copy_node_for_user(user, node)))
def get_stats(self, throughput, above=True):
nodes_in_server = self.cluster_util.get_nodes_in_cluster(self.cluster)
self.resource_producer.get_resource_stat_monitor(
self.users, nodes_in_server, throughput, above)
def set_throughput_to_zero(self):
""" Set the throughput for all tasks to 0 """
for task in self.tasks:
task.set_throughput(0)
def tearDown(self):
super(LimitTest, self).tearDown()
def enforce(self):
""" Make effects take place """
self.rest.enforce_limits()
def set_limits_for_all_users(self, resource_name, resource_limit):
""" Sets limits for all users and makes them take effect """
# Configure limits from the test params
limit_config = LimitConfig()
limit_config.set_limit(resource_name, resource_limit)
# Configure extra limits
for resource_name, resource_limit in self.extra_resources.items():
limit_config.set_limit(resource_name, int(resource_limit))
# Set user limits
for user in self.users:
self.rest.add_set_builtin_user(
user.username, user.params(limit_config.get_user_config()))
# A pattern for matching UUID-4
pattern = re.compile(
r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
)
# Ensure the user limits were set
if limit_config.get_user_config():
for user in self.users:
content = self.rest.get_builtin_user(user.username)
# Ensure the user's uuid is a UUID-4
self.assertTrue(pattern.match(content['uuid']),
"The user's uuid is does match a UUID-4")
self.assertEqual(content['limits'],
limit_config.get_user_config())
# Set scope limits
for user in self.users:
self.rest.set_scope_limit(self.bucket_name, user.scope.name,
limit_config.get_scope_config())
# Ensure the scope limits were set
if limit_config.get_scope_config():
for user in self.users:
_, content = BucketHelper(
self.cluster.master).list_collections(self.bucket_name)
scope = next(scope for scope in json.loads(content)['scopes']
if scope['name'] == user.scope.name)
self.assertEqual(scope['limits'],
limit_config.get_scope_config())
self.enforce()
def test_below_threshold(self):
""" A test in which the throughput is below the threshold and operations succeed. """
throughput_difference = self.throughput_difference
self.set_limits_for_all_users(self.resource_name, self.resource_limit)
# Check that a throughput below the limit succeeds
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units -
throughput_difference)
for task in self.tasks:
self.get_throughput = task.get_throughput() + self.error
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(),
task.get_throughput(), self.error)))
self.get_stats(self.get_throughput, False)
self.set_throughput_to_zero()
def check(self, lhs, rhs, error):
self.log.info(
"Expected:{} Actual:{} Actual Difference:{} Expected Difference:{}"
.format(lhs, rhs, abs(lhs - rhs), error))
return abs(lhs - rhs) <= error
def check_error(self, actual_error, expected_error):
self.log.info("Expected: {} Actual: {}".format(expected_error,
actual_error[:20]))
return actual_error == expected_error
def test_above_threshold(self):
""" A test in which the throughput is above the threshold and
operations exceeding the threshold fail. """
throughput_difference = self.throughput_difference
self.set_limits_for_all_users(self.resource_name, self.resource_limit)
# Check that a throughput above the threshold is constrained by the
# resource limit
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units +
throughput_difference)
for task in self.tasks:
self.get_throughput = self.resource_limit * self.units
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(), self.
get_throughput, self.error)))
# Once above threshold, ensure the expected error message is thrown
if self.tasks and self.tasks[0].expected_error():
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check_error(self.tasks[0].error(), self.tasks[
0].expected_error())))
self.run_compaction()
# set a higher limit to validate stats
self.set_limits_for_all_users(self.resource_name,
self.resource_limit * 2)
self.get_stats(self.get_throughput)
self.set_throughput_to_zero()
def test_above_to_below_threshold(self):
""" A test in which the throughput is initially above the threshold and
operations fail, the throughput is decreased to below the threshold and
operations succeed. """
throughput_difference = self.throughput_difference
self.set_limits_for_all_users(self.resource_name, self.resource_limit)
# Check that a throughput above the threshold is constrained by the
# resource limit
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units +
throughput_difference)
for task in self.tasks:
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(
), self.resource_limit * self.units, self.error)))
self.sleep(5)
# Check that a throughput below the limit succeeds
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units -
throughput_difference)
for task in self.tasks:
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(),
task.get_throughput(), self.error)))
self.set_throughput_to_zero()
def test_below_to_above_threshold(self):
""" A test in which the throughput is initially below the threshold and
operations succeed, the throughput is increased to above the threshold and
operations fail. """
throughput_difference = self.throughput_difference
self.set_limits_for_all_users(self.resource_name, self.resource_limit)
# Check that a throughput below the limit succeeds
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units -
throughput_difference)
for task in self.tasks:
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(),
task.get_throughput(), self.error)))
self.sleep(5)
# Check that a throughput above the threshold is constrained by the
# resource limit
for task in self.tasks:
task.set_throughput(self.resource_limit * self.units +
throughput_difference)
for task in self.tasks:
self.assertTrue(
retry_with_timeout(
self.retry_timeout,
lambda: self.check(task.get_throughput_success(
), self.resource_limit * self.units, self.error)))
self.set_throughput_to_zero()
def test_user_churn(self):
""" A test in which initial users are deleted to make space for new
users. Limits are applied to new users. A throughput of above the
threshold is applied and it is expected for the throughput to remain at
or below the limit threshold. """
for i in range(self.input.param("iterations", 5)):
# Test that throughput remains at or below the threshold
self.test_above_threshold()
# Remove the user from the system
self.remove_user_and_task(self.users[-1])
# Add a new user to the system
self.insert_user_and_task(
User("user{}".format(len(self.users) + i)))
def test_cluster_ops(self):
""" A test in which the limits are configured, a cluser operation
happens and throughput is produced above the threshold and operations
exceeding the threshold fail.
"""
# The number of times the cluster operation happens
cycles = self.input.param("cycles", 3)
# The type of cluster operation
strategy = self.input.param("strategy", "rebalance")
self.assertGreaterEqual(self.nodes_init, 3,
"Requires at least 3 nodes")
if strategy not in {'rebalance', 'graceful-failover', 'hard-failover'}:
self.fail(
"Strategy {} is not an allowed strategy".format(strategy))
# Apply cluster operation
apply_rebalance(self.task,
self.cluster.servers,
cycles=cycles,
strategy=strategy)
# Ensure limits still apply after cluster operation
self.test_above_threshold()
class LWWStatsTests(BaseTestCase):
# The stats related epctl vbucket commands actually apply to the whole bucket but we need a vbucket parameter,
# (which is apparently ignore)
DUMMY_VBUCKET = ' 123' # the leading space is needed
DEFAULT_THRESHOLD = 5000000
ONE_HOUR_IN_SECONDS = 3600
def setUp(self):
super(LWWStatsTests, self).setUp()
self.rest = RestConnection(self.servers[0])
def tearDown(self):
super(LWWStatsTests, self).tearDown()
def test_time_sync_threshold_setting(self):
'''
@summary: This method checks for the change in drift threshold
settings. We change the drift_ahead_threshold value using REST
and then verify it against retreived value
'''
self.log.info('starting test_time_sync_threshold_setting')
# bucket is created with lww in base test case using the LWW parameter
# get the stats
client = MemcachedClientHelper.direct_client(self.servers[0],
self.buckets[0])
ahead_threshold = int(
client.stats()["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD,
'Ahead threshold mismatch expected: {0} '\
'actual {1}'.format(LWWStatsTests.DEFAULT_THRESHOLD,
ahead_threshold))
# change the setting and verify it is per the new setting - this may or may not be supported
cmd = "curl -X POST -u Administrator:password -d "\
"'driftAheadThresholdMs={0}' http://{1}:8091/pools/default/"\
"buckets/default".format(str(LWWStatsTests.DEFAULT_THRESHOLD/2000),#Rest API accepts value in milli-seconds
self.servers[0].ip)
self.log.info("Executing command: %s" % cmd)
try:
os.system(cmd)
except Exception as err:
self.fail('Exception occurred: %s' % str(err))
time.sleep(10)
cl_stats = client.stats()
ahead_threshold = int(cl_stats["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(
ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD / 2,
'Ahead threshold mismatch expected: {0} actual {1}'.format(
LWWStatsTests.DEFAULT_THRESHOLD / 2, ahead_threshold))
# generally need to fill out a matrix here behind/ahead - big and small
def test_time_sync_threshold_setting_rest_call(self):
self.log.info('starting test_time_sync_threshold_setting_rest_call')
# bucket is created with lww in base test case using the LWW parameter
client = MemcachedClientHelper.direct_client(self.servers[0],
self.buckets[0])
rest = RestConnection(self.master)
self.assertTrue(
rest.set_cas_drift_threshold(self.buckets[0], 100000, 200000),
'Unable to set the CAS drift threshold')
time.sleep(15) # take a few seconds for the stats to settle in
stats = client.stats()
self.assertTrue(
int(stats['ep_hlc_drift_ahead_threshold_us']) == 100000 * 1000,
'Ahead threshold incorrect. Expected {0} actual {1}'.format(
100000 * 1000, stats['ep_hlc_drift_ahead_threshold_us']))
self.assertTrue(
int(stats['ep_hlc_drift_behind_threshold_us']) == 200000 * 1000,
'Ahead threshold incorrect. Expected {0} actual {1}'.format(
200000 * 1000, stats['ep_hlc_drift_behind_threshold_us']))
# generally need to fill out a matrix here behind/ahead - big and small
def test_poisoned_cas(self):
"""
@note: - set the clock ahead
- do lots of sets and get some CASs
- do a set and get the CAS (flag, CAS, value) and save it
- set the clock back
- verify the CAS is still big on new sets
- reset the CAS
- do the vbucket max cas and verify
- do a new mutation and verify the CAS is smaller
"""
#creating a user 'default' for the bucket
self.log.info('starting test_poisoned_cas')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.servers[0].ip],
bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(
self.servers[0], self.buckets[0])
# move the system clock ahead to poison the CAS
shell = RemoteMachineShellConnection(self.servers[0])
self.assertTrue(
shell.change_system_time(LWWStatsTests.ONE_HOUR_IN_SECONDS),
'Failed to advance the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set forward {0}'.format(output))
rc = sdk_client.set('key1', 'val1')
rc = mc_client.get('key1')
poisoned_cas = rc[1]
self.log.info('The poisoned CAS is {0}'.format(poisoned_cas))
# do lots of mutations to set the max CAS for all vbuckets
gen_load = BlobGenerator('key-for-cas-test',
'value-for-cas-test-',
self.value_size,
end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
# move the clock back again and verify the CAS stays large
self.assertTrue(
shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS),
'Failed to change the clock')
output, error = shell.execute_command('date')
self.log.info('Date after is set backwards {0}'.format(output))
use_mc_bin_client = self.input.param("use_mc_bin_client", True)
if use_mc_bin_client:
rc = mc_client.set('key2', 0, 0, 'val2')
second_poisoned_cas = rc[1]
else:
rc = sdk_client.set('key2', 'val2')
second_poisoned_cas = rc.cas
self.log.info(
'The second_poisoned CAS is {0}'.format(second_poisoned_cas))
self.assertTrue(
second_poisoned_cas > poisoned_cas,
'Second poisoned CAS {0} is not larger than the first poisoned cas'
.format(second_poisoned_cas, poisoned_cas))
# reset the CAS for all vbuckets. This needs to be done in conjunction with a clock change. If the clock is not
# changed then the CAS will immediately continue with the clock. I see two scenarios:
# 1. Set the clock back 1 hours and the CAS back 30 minutes, the CAS should be used
# 2. Set the clock back 1 hour, set the CAS back 2 hours, the clock should be use
# do case 1, set the CAS back 30 minutes. Calculation below assumes the CAS is in nanoseconds
earlier_max_cas = poisoned_cas - 30 * 60 * 1000000000
for i in range(self.vbuckets):
output, error = shell.execute_cbepctl(
self.buckets[0], "", "set_vbucket_param", "max_cas ",
str(i) + ' ' + str(earlier_max_cas))
if len(error) > 0:
self.fail('Failed to set the max cas')
# verify the max CAS
for i in range(self.vbuckets):
max_cas = int(
mc_client.stats('vbucket-details')['vb_' + str(i) +
':max_cas'])
self.assertTrue(
max_cas == earlier_max_cas,
'Max CAS not properly set for vbucket {0} set as {1} and observed {2}'
.format(i, earlier_max_cas, max_cas))
self.log.info(
'Per cbstats the max cas for bucket {0} is {1}'.format(
i, max_cas))
rc1 = sdk_client.set('key-after-resetting cas', 'val1')
rc2 = mc_client.get('key-after-resetting cas')
set_cas_after_reset_max_cas = rc2[1]
self.log.info(
'The later CAS is {0}'.format(set_cas_after_reset_max_cas))
self.assertTrue(
set_cas_after_reset_max_cas < poisoned_cas,
'For {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'.
format('key-after-resetting cas', set_cas_after_reset_max_cas,
poisoned_cas))
# do a bunch of sets and verify the CAS is small - this is really only one set, need to do more
gen_load = BlobGenerator('key-for-cas-test-after-cas-is-reset',
'value-for-cas-test-',
self.value_size,
end=1000)
self._load_all_buckets(self.master, gen_load, "create", 0)
gen_load.reset()
while gen_load.has_next():
key, value = next(gen_load)
try:
rc = mc_client.get(key)
#rc = sdk_client.get(key)
cas = rc[1]
self.assertTrue(
cas < poisoned_cas,
'For key {0} CAS has not decreased. Current CAS {1} poisoned CAS {2}'
.format(key, cas, poisoned_cas))
except:
self.log.info('get error with {0}'.format(key))
rc = sdk_client.set('key3', 'val1')
better_cas = rc.cas
self.log.info('The better CAS is {0}'.format(better_cas))
self.assertTrue(better_cas < poisoned_cas, 'The CAS was not improved')
# set the clock way ahead - remote_util_OS.py (new)
# do a bunch of mutations - not really needed
# do the fix command - cbepctl, the existing way (remote util)
# do some mutations, verify they conform to the new CAS - build on the CAS code,
# where to iterate over the keys and get the CAS?
"""
use the SDK client
while gen.has_next():
key, value = gen.next()
get the cas for these
also do the vbucket stats
"""
# also can be checked in the vbucket stats somewhere
# revert the clock
def test_drift_stats(self):
'''
@note: An exercise in filling out the matrix with the right amount of code,
we want to test (ahead,behind) and (setwithmeta, deleteWithmeta)
and (active,replica).
So for now let's do the set/del in sequences
'''
self.log.info('starting test_drift_stats')
#Creating a user with the bucket name having admin access
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
check_ahead_threshold = self.input.param("check_ahead_threshold", True)
self.log.info(
'Checking the ahead threshold? {0}'.format(check_ahead_threshold))
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.servers[0].ip],
bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(
self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# get the current time
rc = sdk_client.set('key1', 'val1')
current_time_cas = rc.cas
test_key = 'test-set-with-metaxxxx'
vbId = (((zlib.crc32(test_key.encode())) >> 16)
& 0x7fff) & (self.vbuckets - 1)
#import pdb;pdb.set_trace()
# verifying the case where we are within the threshold, do a set and del, neither should trigger
#mc_active.setWithMeta(key, '123456789', 0, 0, 123, cas)
rc = mc_client.setWithMeta(test_key, 'test-value', 0, 0, 1,
current_time_cas)
#rc = mc_client.setWithMetaLWW(test_key, 'test-value', 0, 0, current_time_cas)
#rc = mc_client.delWithMetaLWW(test_key, 0, 0, current_time_cas+1)
vbucket_stats = mc_client.stats('vbucket-details')
ahead_exceeded = int(vbucket_stats['vb_' + str(vbId) +
':drift_ahead_threshold_exceeded'])
self.assertTrue(
ahead_exceeded == 0,
'Ahead exceeded expected is 0 but is {0}'.format(ahead_exceeded))
behind_exceeded = int(
vbucket_stats['vb_' + str(vbId) +
':drift_behind_threshold_exceeded'])
self.assertTrue(
behind_exceeded == 0,
'Behind exceeded expected is 0 but is {0}'.format(behind_exceeded))
# out of curiousity, log the total counts
self.log.info(
'Total stats: total abs drift {0} and total abs drift count {1}'.
format(vbucket_stats['vb_' + str(vbId) + ':total_abs_drift'],
vbucket_stats['vb_' + str(vbId) +
':total_abs_drift_count']))
# do the ahead set with meta case - verify: ahead threshold exceeded, total_abs_drift count and abs_drift
if check_ahead_threshold:
stat_descriptor = 'ahead'
cas = current_time_cas + 5000 * LWWStatsTests.DEFAULT_THRESHOLD
else:
stat_descriptor = 'behind'
cas = current_time_cas - (5000 * LWWStatsTests.DEFAULT_THRESHOLD)
rc = mc_client.setWithMeta(test_key, 'test-value', 0, 0, 0, cas)
#rc = mc_client.delWithMetaLWW(test_key, 0, 0, cas+1)
# verify the vbucket stats
vbucket_stats = mc_client.stats('vbucket-details')
drift_counter_stat = 'vb_' + str(
vbId) + ':drift_' + stat_descriptor + '_threshold_exceeded'
threshold_exceeded = int(
mc_client.stats('vbucket-details')[drift_counter_stat])
# MB-21450 self.assertTrue( ahead_exceeded == 2, '{0} exceeded expected is 1 but is {1}'.
# format( stat_descriptor, threshold_exceeded))
self.log.info(
'Total stats: total abs drift {0} and total abs drift count {1}'.
format(vbucket_stats['vb_' + str(vbId) + ':total_abs_drift'],
vbucket_stats['vb_' + str(vbId) +
':total_abs_drift_count']))
# and verify the bucket stats: ep_active_hlc_drift_count, ep_clock_cas_drift_threshold_exceeded,
# ep_active_hlc_drift
bucket_stats = mc_client.stats()
ep_active_hlc_drift_count = int(
bucket_stats['ep_active_hlc_drift_count'])
ep_clock_cas_drift_threshold_exceeded = int(
bucket_stats['ep_clock_cas_drift_threshold_exceeded'])
ep_active_hlc_drift = int(bucket_stats['ep_active_hlc_drift'])
# Drift count appears to be the number of mutations
self.assertTrue(
ep_active_hlc_drift_count > 0,
'ep_active_hlc_drift_count is 0, expected a positive value')
# drift itself is the sum of the absolute values of all drifts, so check that it is greater than 0
self.assertTrue(ep_active_hlc_drift > 0,
'ep_active_hlc_drift is 0, expected a positive value')
# the actual drift count is a little more granular
expected_drift_threshold_exceed_count = 1
self.assertTrue(
expected_drift_threshold_exceed_count ==
ep_clock_cas_drift_threshold_exceeded,
'ep_clock_cas_drift_threshold_exceeded is incorrect. Expected {0}, actual {1}'
.format(expected_drift_threshold_exceed_count,
ep_clock_cas_drift_threshold_exceeded))
def test_logical_clock_ticks(self):
self.log.info('starting test_logical_clock_ticks')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.servers[0].ip],
bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(
self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# do a bunch of mutations to set the max cas
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
base_total_logical_clock_ticks = 0
for i in range(self.vbuckets):
#print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']
base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The base total logical clock ticks is {0}'.format(
base_total_logical_clock_ticks))
# move the system clock back so the logical counter part of HLC is used and the logical clock ticks
# stat is incremented
self.assertTrue(
shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS),
'Failed to advance the clock')
# do more mutations
NUMBER_OF_MUTATIONS = 10000
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=NUMBER_OF_MUTATIONS)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
time.sleep(30)
total_logical_clock_ticks = 0
for i in range(self.vbuckets):
total_logical_clock_ticks = total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The total logical clock ticks is {0}'.format(
total_logical_clock_ticks))
self.assertTrue(
total_logical_clock_ticks -
base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS,
'Expected clock tick {0} actual {1}'.format(
NUMBER_OF_MUTATIONS,
total_logical_clock_ticks - base_total_logical_clock_ticks))