def iostats(self, nodes, frequency, verbosity=False):
shells = []
for node in nodes:
try:
bucket = RestConnection(node).get_buckets()[0].name
MemcachedClientHelper.direct_client(node, bucket)
shells.append(RemoteMachineShellConnection(node))
except:
pass
self._task["iostats"] = []
print "started capturing io stats"
while not self._aborted():
time.sleep(frequency)
print "collecting io_stats"
for shell in shells:
kB_read, kB_wrtn = self._extract_io_info(shell)
if kB_read and kB_wrtn:
self._task["iostats"].append({"time": time.time(),
"ip": shell.ip,
"read": kB_read,
"write": kB_wrtn})
print "finished capturing io stats"
def system_stats(self, nodes, pnames, frequency, verbosity=False):
shells = []
for node in nodes:
try:
bucket = RestConnection(node).get_buckets()[0].name
MemcachedClientHelper.direct_client(node, bucket)
shells.append(RemoteMachineShellConnection(node))
except:
pass
d = {"snapshots": []}
# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]
start_time = str(self._task["time"])
while not self._aborted():
time.sleep(frequency)
current_time = time.time()
i = 0
for shell in shells:
node = nodes[i]
unique_id = node.ip+'-'+start_time
for pname in pnames:
obj = RemoteMachineHelper(shell).is_process_running(pname)
if obj and obj.pid:
value = self._extract_proc_info(shell, obj.pid)
value["name"] = pname
value["id"] = obj.pid
value["unique_id"] = unique_id
value["time"] = current_time
value["ip"] = node.ip
d["snapshots"].append(value)
i += 1
self._task["systemstats"] = d["snapshots"]
print " finished system_stats"
def test_checkpointing_with_full_rollback(self):
bucket = self.src_cluster.get_buckets()[0]
nodes = self.src_cluster.get_nodes()
# Stop Persistence on Node A & Node B
for node in nodes:
mem_client = MemcachedClientHelper.direct_client(node, bucket)
mem_client.stop_persistence()
self.src_cluster.pause_all_replications()
gen = BlobGenerator("C1-", "C1-", self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.src_cluster.resume_all_replications()
self.sleep(self._checkpoint_interval * 2)
self.get_and_validate_latest_checkpoint()
# Perform mutations on the bucket
self.async_perform_update_delete()
self.sleep(self._wait_timeout)
# Kill memcached on Node A so that Node B becomes master
shell = RemoteMachineShellConnection(self.src_cluster.get_master_node())
shell.kill_memcached()
# Start persistence on Node B
mem_client = MemcachedClientHelper.direct_client(nodes[1], bucket)
mem_client.start_persistence()
# Failover Node B
failover_task = self.src_cluster.async_failover()
failover_task.result()
# Wait for Failover & rollback to complete
self.sleep(self._wait_timeout * 5)
goxdcr_log = NodeHelper.get_goxdcr_log_dir(self._input.servers[0]) \
+ '/goxdcr.log*'
count1 = NodeHelper.check_goxdcr_log(
nodes[0],
"Received rollback from DCP stream",
goxdcr_log)
self.assertGreater(count1, 0, "full rollback not received from DCP as expected")
self.log.info("full rollback received from DCP as expected")
count2 = NodeHelper.check_goxdcr_log(
nodes[0],
"Rolled back startSeqno to 0",
goxdcr_log)
self.assertGreater(count2, 0, "startSeqno not rolled back to 0 as expected")
self.log.info("startSeqno rolled back to 0 as expected")
shell.disconnect()
def _test_backup_add_restore_bucket_with_expiration_key(self, replica):
bucket = "default"
rest = RestConnection(self.master)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi, replicaNumber=replica)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
client = MemcachedClientHelper.direct_client(self.master, bucket)
expiry = 60
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(5000)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, key)
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
client.close()
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
node = RestConnection(self.master).get_nodes_self()
output, error = self.shell.execute_command(self.perm_command)
self.shell.log_command_output(output, error)
backupHelper = BackupHelper(self.master, self)
backupHelper.backup(bucket, node, self.remote_tmp_folder)
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket, self)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
backupHelper.restore(self.remote_tmp_folder)
time.sleep(60)
client = MemcachedClientHelper.direct_client(self.master, bucket)
self.log.info('verifying that all those keys have expired...')
for key in keys:
try:
client.get(key=key)
msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
self.fail(msg.format(expiry, key, expiry))
except mc_bin_client.MemcachedError as error:
self.assertEquals(error.status, 1,
msg="expected error code {0} but saw error code {1}".format(1, error.status))
client.close()
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def _kill_nodes(self, nodes, servers, bucket_name):
self.reboot = self.input.param("reboot", True)
if not self.reboot:
for node in nodes:
_node = {
"ip": node.ip,
"port": node.port,
"username": self.servers[0].rest_username,
"password": self.servers[0].rest_password,
}
node_rest = RestConnection(_node)
_mc = MemcachedClientHelper.direct_client(_node, bucket_name)
self.log.info("restarted the node %s:%s" % (node.ip, node.port))
pid = _mc.stats()["pid"]
command = 'os:cmd("kill -9 {0} ")'.format(pid)
self.log.info(command)
killed = node_rest.diag_eval(command)
self.log.info("killed ?? {0} ".format(killed))
_mc.close()
else:
for server in servers:
shell = RemoteMachineShellConnection(server)
command = "reboot"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
time.sleep(self.wait_timeout * 8)
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
def test_time_sync_threshold_setting(self):
'''
@summary: This method checks for the change in drift threshold
settings. We change the drift_ahead_threshold value using REST
and then verify it against retreived value
'''
self.log.info('starting test_time_sync_threshold_setting')
# bucket is created with lww in base test case using the LWW parameter
# get the stats
client = MemcachedClientHelper.direct_client(self.servers[0],
self.buckets[0])
ahead_threshold = int(
client.stats()["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD,
'Ahead threshold mismatch expected: {0} '\
'actual {1}'.format(LWWStatsTests.DEFAULT_THRESHOLD,
ahead_threshold))
# change the setting and verify it is per the new setting - this may or may not be supported
cmd = "curl -X POST -u Administrator:password -d "\
"'driftAheadThresholdMs={0}' http://{1}:8091/pools/default/"\
"buckets/default".format(str(LWWStatsTests.DEFAULT_THRESHOLD/2000),#Rest API accepts value in milli-seconds
self.servers[0].ip)
self.log.info("Executing command: %s" % cmd)
try:
os.system(cmd)
except Exception as err:
self.fail('Exception occurred: %s' % str(err))
time.sleep(10)
cl_stats = client.stats()
ahead_threshold = int(cl_stats["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(
ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD / 2,
'Ahead threshold mismatch expected: {0} actual {1}'.format(
LWWStatsTests.DEFAULT_THRESHOLD / 2, ahead_threshold))
def getr_dgm_test(self):
resident_ratio = self.input.param("resident_ratio", 50)
gens = []
delta_items = 200000
self.num_items = 0
mc = MemcachedClientHelper.direct_client(self.master, self.default_bucket_name)
self.log.info("LOAD PHASE")
end_time = time.time() + self.wait_timeout * 30
while (int(mc.stats()["vb_active_perc_mem_resident"]) == 0 or\
int(mc.stats()["vb_active_perc_mem_resident"]) > resident_ratio) and\
time.time() < end_time:
self.log.info("Resident ratio is %s" % mc.stats()["vb_active_perc_mem_resident"])
gen = DocumentGenerator('test_docs', '{{"age": {0}}}', xrange(5),
start=self.num_items, end=(self.num_items + delta_items))
gens.append(copy.deepcopy(gen))
self._load_all_buckets(self.master, gen, 'create', self.expiration, kv_store=1,
flag=self.flags, only_store_hash=False, batch_size=1)
self.num_items += delta_items
self.log.info("Resident ratio is %s" % mc.stats()["vb_active_perc_mem_resident"])
self.assertTrue(int(mc.stats()["vb_active_perc_mem_resident"]) < resident_ratio,
"Resident ratio is not reached")
self.verify_cluster_stats(self.servers[:self.nodes_init], only_store_hash=False,
batch_size=1)
self.log.info("Currently loaded items: %s" % self.num_items)
self.log.info("READ REPLICA PHASE")
self.verify_cluster_stats(self.servers[:self.nodes_init], only_store_hash=False,
replica_to_read=self.replica_to_read, batch_size=1)
def block_for_replication(self, key, cas=0, num=1, timeout=0, persist=False):
"""
observe a key until it has been replicated to @param num of servers
@param persist : block until item has been persisted to disk
"""
vbucketid = self.client._get_vBucket_id(key)
repl_servers = self._get_server_str(vbucketid, repl=True)
persisted = 0
self.log.info("VbucketId:%s on replicated servers:%s" % (vbucketid, repl_servers))
while len(repl_servers) >= num > 0:
for server in repl_servers:
node = self._get_node(server)
self.log.info("Replicated Server:- %s" % (server))
newclient = MemcachedClientHelper.direct_client(node, self.default_bucket_name)
t_start = datetime.now()
while persisted == 0:
opaque, rep_time, persist_time, persisted, cas = newclient.observe(key)
t_end = datetime.now()
self.log.info("######key:-%s and Server:- %s#########" % (key, server))
self.log.info("Persisted:- %s" % (persisted))
self.log.info("Time taken to persist:- %s" % (t_end - t_start))
num = num + 1
if num == 0:
break
return True
def _restart_memcache(self, bucket_name):
rest = RestConnection(self.master)
nodes = rest.node_statuses()
self._kill_nodes(nodes, bucket_name)
start = time.time()
memcached_restarted = False
for server in self.servers[:self.nodes_init]:
mc = None
while time.time() - start < 60:
try:
mc = MemcachedClientHelper.direct_client(server, bucket_name)
stats = mc.stats()
new_uptime = int(stats["uptime"])
self.log.info("warmutime%s:%s" % (new_uptime, self.pre_warmup_stats["%s:%s" % (server.ip, server.port)]["uptime"]))
if new_uptime < self.pre_warmup_stats["%s:%s" % (server.ip, server.port)]["uptime"]:
self.log.info("memcached restarted...")
memcached_restarted = True
break;
except Exception:
self.log.error("unable to connect to %s:%s" % (server.ip, server.port))
if mc:
mc.close()
time.sleep(1)
if not memcached_restarted:
self.fail("memcached did not start %s:%s" % (server.ip, server.port))
def setUp(self):
self.log = logger.Logger.get_logger()
self.master = TestInputSingleton.input.servers[0]
ClusterOperationHelper.cleanup_cluster([self.master])
BucketOperationHelper.delete_all_buckets_or_assert([self.master], self)
self._bucket_name = 'default'
serverInfo = self.master
rest = RestConnection(serverInfo)
info = rest.get_nodes_self()
self._bucket_port = info.moxi
rest.init_cluster(username=serverInfo.rest_username,
password=serverInfo.rest_password)
rest.init_cluster_memoryQuota(memoryQuota=info.mcdMemoryReserved)
bucket_ram = info.memoryQuota * 2 / 3
# Add built-in user
testuser = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'password': '******'
}]
RbacBase().create_user_source(testuser, 'builtin', self.master)
# Assign user to role
role_list = [{
'id': 'cbadminbucket',
'name': 'cbadminbucket',
'roles': 'admin'
}]
RbacBase().add_user_role(role_list, RestConnection(self.master),
'builtin')
rest.create_bucket(bucket=self._bucket_name,
ramQuotaMB=bucket_ram,
proxyPort=info.memcached)
msg = 'create_bucket succeeded but bucket "default" does not exist'
if (testconstants.TESTRUNNER_CLIENT in os.environ.keys()
) and os.environ[
testconstants.TESTRUNNER_CLIENT] == testconstants.PYTHON_SDK:
self.client = SDKSmartClient(
serverInfo,
self._bucket_name,
compression=TestInputSingleton.input.param(
"sdk_compression", True))
else:
self.client = MemcachedClientHelper.direct_client(
serverInfo, self._bucket_name)
self.assertTrue(BucketOperationHelper.wait_for_bucket_creation(
self._bucket_name, rest),
msg=msg)
ready = BucketOperationHelper.wait_for_memcached(
serverInfo, self._bucket_name)
self.assertTrue(ready, "wait_for_memcached failed")
self._log_start()
def get_stats_memc(self, bucket_name, stat_name="", key=None):
"""
Fetches stats using cbstat and greps for specific line.
Uses command:
cbstats localhost:port 'stat_name' | grep 'field_to_grep'
Note: Function calling this API should take care of validating
the outputs and handling the errors/warnings from execution.
Arguments:
:bucket_name - Name of the bucket to get the stats
:stat_name - Any valid stat_command accepted by cbstats
:field_to_grep - Target stat name string to grep.
Default=None, means fetch all data
Returns:
:output - Output for the cbstats command
:error - Buffer containing warnings/errors from the execution
"""
# result = dict()
if stat_name == "all":
stat_name = ""
client = MemcachedClientHelper.direct_client(
self.server, Bucket({"name": bucket_name}), 30, self.username,
self.password)
output = client.stats(stat_name)
client.close()
return output if key is None else output[key]
def wait_warmup_completed(self, warmupnodes, bucket_names=["default"]):
if isinstance(bucket_names, str):
bucket_names = [bucket_names]
for server in warmupnodes:
for bucket in bucket_names:
mc = MemcachedClientHelper.direct_client(server, bucket)
start = time.time()
while time.time() - start < 150:
if mc.stats()["ep_warmup_thread"] == "complete":
self._log.info("Warmed up: %s items " %
(mc.stats()["curr_items_tot"]))
time.sleep(10)
break
elif mc.stats()["ep_warmup_thread"] == "running":
self._log.info(
"Still warming up .. curr_items_tot : %s" %
(mc.stats()["curr_items_tot"]))
continue
else:
self._log.info(
"Value of ep_warmup_thread does not exist, exiting from this server"
)
break
if mc.stats()["ep_warmup_thread"] == "running":
self._log.info(
"ERROR: ep_warmup_thread's status not complete")
mc.close
def test_time_sync_threshold_setting(self):
'''
@summary: This method checks for the change in drift threshold
settings. We change the drift_ahead_threshold value using REST
and then verify it against retreived value
'''
self.log.info('starting test_time_sync_threshold_setting')
# bucket is created with lww in base test case using the LWW parameter
# get the stats
client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
ahead_threshold = int(client.stats()["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD,
'Ahead threshold mismatch expected: {0} '\
'actual {1}'.format(LWWStatsTests.DEFAULT_THRESHOLD,
ahead_threshold))
# change the setting and verify it is per the new setting - this may or may not be supported
cmd = "curl -X POST -u Administrator:password -d "\
"'driftAheadThresholdMs={0}' http://{1}:8091/pools/default/"\
"buckets/default".format(str(LWWStatsTests.DEFAULT_THRESHOLD/2000),#Rest API accepts value in milli-seconds
self.servers[0].ip)
self.log.info("Executing command: %s"%cmd)
try:
os.system(cmd)
except Exception,err:
self.fail('Exception occurred: %s'%str(err))
def memory_quota_default_bucket(self):
resident_ratio = self.input.param("resident_ratio", 50)
delta_items = 200000
mc = MemcachedClientHelper.direct_client(self.master,
self.default_bucket_name)
self.log.info("LOAD PHASE")
end_time = time.time() + self.wait_timeout * 30
while (int(mc.stats()["vb_active_perc_mem_resident"]) == 0 or \
int(mc.stats()["vb_active_perc_mem_resident"]) > resident_ratio) and \
time.time() < end_time:
self.log.info("Resident ratio is %s" %
mc.stats()["vb_active_perc_mem_resident"])
gen = DocumentGenerator('test_docs',
'{{"age": {0}}}',
xrange(5),
start=self.num_items,
end=(self.num_items + delta_items))
self._load_all_buckets(self.master, gen, 'create', 0)
self.num_items += delta_items
self.log.info("Resident ratio is %s" %
mc.stats()["vb_active_perc_mem_resident"])
memory_mb = int(mc.stats("memory")["mem_used"]) / (1024 * 1024)
self.log.info("Memory Used is %s" % memory_mb)
self.assertTrue(
memory_mb <= self.quota,
"Memory Used %s should be within %s" % (memory_mb, self.quota))
def _load_memcached_bucket(self, server, gen_load, bucket_name):
num_tries = 0
while num_tries < 6:
try:
num_tries += 1
client = MemcachedClientHelper.direct_client(
server, bucket_name)
break
except Exception as ex:
if num_tries < 5:
self.log.info(
"unable to create memcached client due to {0}. Try again"
.format(ex))
else:
self.log.error(
"unable to create memcached client due to {0}.".format(
ex))
while gen_load.has_next():
key, value = gen_load.next()
for v in xrange(1024):
try:
client.set(key, 0, 0, value, v)
break
except:
pass
client.close()
def _kill_nodes(self, nodes, servers, bucket_name):
self.reboot = self.input.param("reboot", True)
if not self.reboot:
for node in nodes:
_node = {
"ip": node.ip,
"port": node.port,
"username": self.servers[0].rest_username,
"password": self.servers[0].rest_password
}
node_rest = RestConnection(_node)
_mc = MemcachedClientHelper.direct_client(_node, bucket_name)
self.log.info("restarted the node %s:%s" %
(node.ip, node.port))
pid = _mc.stats()["pid"]
command = "os:cmd(\"kill -9 {0} \")".format(pid)
self.log.info(command)
killed = node_rest.diag_eval(command)
self.log.info("killed ?? {0} ".format(killed))
_mc.close()
else:
for server in servers:
shell = RemoteMachineShellConnection(server)
command = "reboot"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
time.sleep(self.wait_timeout * 8)
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
def collect_vbucket_num_stats(self,servers, buckets):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
Returns:
Failover stats as follows:
if not collecting per node :: {bucket : [{key:value}]}
if collecting per node :: {bucket : {node:[{key:value}]}}
"""
active_bucketMap = {}
replica_bucketMap = {}
for bucket in buckets:
active_map_data = {}
replica_map_data = {}
for server in servers:
client = MemcachedClientHelper.direct_client(server, bucket)
stats = client.stats('')
for key in stats.keys():
if key == 'vb_active_num':
active_map_data[server.ip] = int(stats[key])
if key == 'vb_replica_num':
replica_map_data[server.ip] = int(stats[key])
active_bucketMap[bucket.name] = active_map_data
replica_bucketMap[bucket.name] = replica_map_data
return active_bucketMap,replica_bucketMap
def run(self):
client = MemcachedClientHelper.direct_client(server, bucket)
for i in range(num_items):
key = "key-{0}".format(i)
value = "value-{0}".format(str(uuid.uuid4())[:7])
client.set(key, 0, 0, value, 0)
log.info("Loaded {0} key".format(num_items))
def run_test(self):
ep_threshold = self.input.param("ep_threshold", "ep_mem_low_wat")
active_resident_threshold = int(self.input.param("active_resident_threshold", 10))
mc = MemcachedClientHelper.direct_client(self.servers[0], self.bucket_name)
stats = mc.stats()
threshold = int(self.input.param("threshold", stats[ep_threshold]))
threshold_reached = False
self.num_items = self.input.param("items", 10000)
self._load_doc_data_all_buckets("create")
# load items till reached threshold or mem-ratio is less than resident ratio threshold
while not threshold_reached:
mem_used = int(mc.stats()["mem_used"])
if mem_used < threshold or int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
self.log.info(
"mem_used and vb_active_perc_mem_resident_ratio reached at %s/%s and %s "
% (mem_used, threshold, mc.stats()["vb_active_perc_mem_resident"])
)
items = self.num_items
self.num_items += self.input.param("items", 10000)
self._load_doc_data_all_buckets("create", items)
else:
threshold_reached = True
self.log.info("DGM state achieved!!!!")
# wait for draining of data before restart and warm up
for bucket in self.buckets:
RebalanceHelper.wait_for_persistence(self.nodes_server[0], bucket)
while 1:
# read_data_task = self.cluster.async_verify_data(self.master, self.buckets[0], self.buckets[0].kvs[1])
read_data_task = Thread(target=self._run_get)
read_data_task.start()
# 5 threads to run stats all and reset asynchronously
start = time.time()
while (time.time() - start) < 300:
stats_all_thread = []
stats_reset_thread = []
for i in xrange(self.threads_to_run):
stat_str = ""
stats_all_thread.append(Thread(target=self._get_stats, args=[stat_str]))
stats_all_thread[i].start()
stat_str = "reset"
stats_reset_thread.append(Thread(target=self._get_stats, args=[stat_str]))
stats_reset_thread[i].start()
for i in xrange(self.threads_to_run):
stats_all_thread[i].join()
stats_reset_thread[i].join()
del stats_all_thread
del stats_reset_thread
# read_data_task.result()
read_data_task.join()
def _stats_befor_warmup(self, bucket_name):
self.pre_warmup_stats[bucket_name] = {}
self.stats_monitor = self.input.param("stats_monitor", "")
self.warmup_stats_monitor = self.input.param("warmup_stats_monitor", "")
if self.stats_monitor is not "":
self.stats_monitor = self.stats_monitor.split(";")
if self.warmup_stats_monitor is not "":
self.warmup_stats_monitor = self.warmup_stats_monitor.split(";")
for server in self.servers:
mc_conn = MemcachedClientHelper.direct_client(server, bucket_name, self.timeout)
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)] = {}
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)]["uptime"] = mc_conn.stats("")[
"uptime"
]
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)]["curr_items_tot"] = mc_conn.stats(
""
)["curr_items_tot"]
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)]["curr_items"] = mc_conn.stats("")[
"curr_items"
]
for stat_to_monitor in self.stats_monitor:
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)][stat_to_monitor] = mc_conn.stats(
""
)[stat_to_monitor]
if self.without_access_log:
for stat_to_monitor in self.warmup_stats_monitor:
self.pre_warmup_stats[bucket_name]["%s:%s" % (server.ip, server.port)][
stat_to_monitor
] = mc_conn.stats("warmup")[stat_to_monitor]
mc_conn.close()
def test_time_sync_threshold_setting(self):
self.log.info("starting test_time_sync_threshold_setting")
# bucket is created with lww in base test case using the LWW parameter
# get the stats
client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
ahead_threshold = int(client.stats()["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(
ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD,
"Ahead threshold mismatch expected: {0} actual {1}".format(
LWWStatsTests.DEFAULT_THRESHOLD, ahead_threshold
),
)
# change the setting and verify it is per the new setting - this may or may not be supported
shell = RemoteMachineShellConnection(self.servers[0])
output, error = shell.execute_cbepctl(
self.buckets[0],
"",
"set vbucket_param",
"hlc_drift_ahead_threshold_us ",
str(LWWStatsTests.DEFAULT_THRESHOLD / 2) + LWWStatsTests.DUMMY_VBUCKET,
)
if len(error) > 0:
self.fail("Failed to set the drift counter threshold, please check the logs.")
ahead_threshold = int(client.stats()["ep_hlc_drift_ahead_threshold_us"])
self.assertTrue(
ahead_threshold == LWWStatsTests.DEFAULT_THRESHOLD / 2,
"Ahead threshold mismatch expected: {0} actual {1}".format(
LWWStatsTests.DEFAULT_THRESHOLD / 2, ahead_threshold
),
)
def test_expired_keys(self):
serverInfo = self.master
client = MemcachedClientHelper.direct_client(serverInfo, self._bucket_name)
expirations = [2, 5, 10]
for expiry in expirations:
testuuid = uuid.uuid4()
keys = ["key_%s_%d" % (testuuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, key)
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
delay = expiry + 5
msg = "sleeping for {0} seconds to wait for those items with expiry set to {1} to expire"
self.log.info(msg.format(delay, expiry))
time.sleep(delay)
self.log.info('verifying that all those keys have expired...')
for key in keys:
try:
client.get(key=key)
msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
self.fail(msg.format(expiry, key, delay))
except mc_bin_client.MemcachedError as error:
self.assertEquals(error.status, 1,
msg="expected error code {0} but saw error code {1}".format(1, error.status))
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def _verify_direct_client_stats(self, bucket, command, output):
mc_conn = MemcachedClientHelper.direct_client(self.master,
bucket.name, self.timeout)
for line in output:
stats = line.rsplit(":", 1)
collect_stats = ""
commands = ["hash", "tapagg"]
if command in commands:
output, error = self.shell.execute_cbstats(bucket, command)
d = []
if len(output) > 0:
d = dict(s.strip().rsplit(':', 1) for s in output)
collect_stats = d[stats[0].strip()].strip()
else:
raise Exception("Command does not throw out error message \
but cbstats gives no output. \
Please check the output manually")
else:
collect_stats = mc_conn.stats(command)[stats[0].strip()]
self.log.info("CbStats###### for {0}:::{1}=={2}" \
.format(stats[0].strip(), collect_stats, stats[1].strip()))
if stats[1].strip() == collect_stats:
continue
else:
if stats[0].find('tcmalloc') != -1 or stats[0].find('bytes') != -1 or\
stats[0].find('mem_used') != -1:
self.log.warn("Stat didn't match, but it can be changed, not a bug")
continue
raise Exception("Command does not throw out error message \
but cbstats does not match.")
def get_vbucket_stats(self,
bucket_name,
stat_name,
vbucket_num,
field_to_grep=None):
"""
Fetches failovers stats for specified vbucket
and greps for specific stat.
Uses command:
cbstats localhost:port failovers '[vbucket_num]' | \
grep '[field_to_grep]'
Note: Function calling this API should take care of validating
the outputs and handling the errors/warnings from execution.
Arguments:
:bucket_name - Name of the bucket to get the stats
:stat_name - Any valid stat_command accepted by cbstats
:vbucket_num - Target vbucket number to fetch the stats
:field_to_grep - Target stat name string to grep.
Default=None, means to fetch all stats related to
the selected vbucket stat
Returns:
:output - Output for the cbstats command
:error - Buffer containing warnings/errors from the execution
"""
client = MemcachedClientHelper.direct_client(self.server,
Bucket(bucket_name), 30,
self.username,
self.password)
output = client.stats("{} {}".format(stat_name, vbucket_num))
client.close()
return output
def test_logical_clock_ticks(self):
self.log.info('starting test_logical_clock_ticks')
payload = "name={0}&roles=admin&password=password".format(
self.buckets[0].name)
self.rest.add_set_builtin_user(self.buckets[0].name, payload)
sdk_client = SDKClient(scheme='couchbase',
hosts=[self.servers[0].ip],
bucket=self.buckets[0].name)
mc_client = MemcachedClientHelper.direct_client(
self.servers[0], self.buckets[0])
shell = RemoteMachineShellConnection(self.servers[0])
# do a bunch of mutations to set the max cas
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=10000)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
base_total_logical_clock_ticks = 0
for i in range(self.vbuckets):
#print vbucket_stats['vb_' + str(i) + ':logical_clock_ticks']
base_total_logical_clock_ticks = base_total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The base total logical clock ticks is {0}'.format(
base_total_logical_clock_ticks))
# move the system clock back so the logical counter part of HLC is used and the logical clock ticks
# stat is incremented
self.assertTrue(
shell.change_system_time(-LWWStatsTests.ONE_HOUR_IN_SECONDS),
'Failed to advance the clock')
# do more mutations
NUMBER_OF_MUTATIONS = 10000
gen_load = BlobGenerator('key-for-cas-test-logical-ticks',
'value-for-cas-test-',
self.value_size,
end=NUMBER_OF_MUTATIONS)
self._load_all_buckets(self.master, gen_load, "create", 0)
vbucket_stats = mc_client.stats('vbucket-details')
time.sleep(30)
total_logical_clock_ticks = 0
for i in range(self.vbuckets):
total_logical_clock_ticks = total_logical_clock_ticks + int(
vbucket_stats['vb_' + str(i) + ':logical_clock_ticks'])
self.log.info('The total logical clock ticks is {0}'.format(
total_logical_clock_ticks))
self.assertTrue(
total_logical_clock_ticks -
base_total_logical_clock_ticks == NUMBER_OF_MUTATIONS,
'Expected clock tick {0} actual {1}'.format(
NUMBER_OF_MUTATIONS,
total_logical_clock_ticks - base_total_logical_clock_ticks))
def test_time_sync_threshold_setting_rest_call(self):
self.log.info("starting test_time_sync_threshold_setting_rest_call")
# bucket is created with lww in base test case using the LWW parameter
client = MemcachedClientHelper.direct_client(self.servers[0], self.buckets[0])
rest = RestConnection(self.master)
self.assertTrue(
rest.set_cas_drift_threshold(self.buckets[0], 100000, 200000), "Unable to set the CAS drift threshold"
)
time.sleep(15) # take a few seconds for the stats to settle in
stats = client.stats()
self.assertTrue(
int(stats["ep_hlc_drift_ahead_threshold_us"]) == 100000 * 1000,
"Ahead threshold incorrect. Expected {0} actual {1}".format(
100000 * 1000, stats["ep_hlc_drift_ahead_threshold_us"]
),
)
self.assertTrue(
int(stats["ep_hlc_drift_behind_threshold_us"]) == 200000 * 1000,
"Ahead threshold incorrect. Expected {0} actual {1}".format(
200000 * 1000, stats["ep_hlc_drift_behind_threshold_us"]
),
)
def collect_vbucket_num_stats(self,servers, buckets):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
Returns:
Failover stats as follows:
if not collecting per node :: {bucket : [{key:value}]}
if collecting per node :: {bucket : {node:[{key:value}]}}
"""
active_bucketMap = {}
replica_bucketMap = {}
for bucket in buckets:
active_map_data = {}
replica_map_data = {}
for server in servers:
client = MemcachedClientHelper.direct_client(server, bucket)
stats = client.stats('')
for key in stats.keys():
if key == 'vb_active_num':
active_map_data[server.ip] = int(stats[key])
if key == 'vb_replica_num':
replica_map_data[server.ip] = int(stats[key])
active_bucketMap[bucket.name] = active_map_data
replica_bucketMap[bucket.name] = replica_map_data
return active_bucketMap,replica_bucketMap
def collect_dcp_stats(self, buckets, servers, stat_names = [],
extra_key_condition = "replication"):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
stat_names: stats we are searching to compare
Returns:
map of bucket informing map[bucket][vbucket id][stat name]
example:: unacked_bytes in dcp
"""
bucketMap = {}
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
stats = MemcachedClientHelper.direct_client(server, bucket).stats('dcp')
for key in stats.keys():
for stat_name in stat_names:
if stat_name in key and extra_key_condition in key:
value = int(stats[key])
tokens = key.split(":")
vb_no = int(tokens[len(tokens) - 1].split("_")[1])
if vb_no not in dataMap:
dataMap[vb_no] = {}
dataMap[vb_no][stat_name] = value
bucketMap[bucket.name] = dataMap
return bucketMap
def collect_dcp_stats(self, buckets, servers, stat_names = [], extra_key_condition = "replication"):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
stat_names: stats we are searching to compare
Returns:
map of bucket informing map[bucket][vbucket id][stat name]
example:: unacked_bytes in dcp
"""
bucketMap = {}
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
stats = MemcachedClientHelper.direct_client(server, bucket).stats('dcp')
for key in stats.keys():
for stat_name in stat_names:
if stat_name in key and extra_key_condition in key:
value = int(stats[key])
tokens = key.split(":")
vb_no = int(tokens[len(tokens) - 1].split("_")[1])
if vb_no not in dataMap:
dataMap[vb_no] = {}
dataMap[vb_no][stat_name] = value
bucketMap[bucket.name] = dataMap
return bucketMap
def _test_backup_add_restore_bucket_with_expiration_key(self, replica):
bucket = "default"
rest = RestConnection(self.master)
info = rest.get_nodes_self()
size = int(info.memoryQuota * 2.0 / 3.0)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi, replicaNumber=replica)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
client = MemcachedClientHelper.direct_client(self.master, bucket)
expiry = 60
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(5000)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, key)
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
client.close()
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
ready = RebalanceHelper.wait_for_persistence(self.master, bucket, bucket_type=self.bucket_type)
self.assertTrue(ready, "not all items persisted. see logs")
node = RestConnection(self.master).get_nodes_self()
output, error = self.shell.execute_command(self.perm_command)
self.shell.log_command_output(output, error)
backupHelper = BackupHelper(self.master, self)
backupHelper.backup(bucket, node, self.remote_tmp_folder)
BucketOperationHelper.delete_bucket_or_assert(self.master, bucket, self)
rest.create_bucket(bucket, ramQuotaMB=size, proxyPort=info.moxi)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
backupHelper.restore(self.remote_tmp_folder)
time.sleep(60)
client = MemcachedClientHelper.direct_client(self.master, bucket)
self.log.info('verifying that all those keys have expired...')
for key in keys:
try:
client.get(key=key)
msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
self.fail(msg.format(expiry, key, expiry))
except mc_bin_client.MemcachedError as error:
self.assertEqual(error.status, 1,
msg="expected error code {0} but saw error code {1}".format(1, error.status))
client.close()
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def run_test(self):
ep_threshold = self.input.param("ep_threshold", "ep_mem_low_wat")
active_resident_threshold = int(self.input.param("active_resident_threshold", 10))
mc = MemcachedClientHelper.direct_client(self.servers[0], self.bucket_name)
stats = mc.stats()
threshold = int(self.input.param('threshold', stats[ep_threshold]))
threshold_reached = False
self.num_items = self.input.param("items", 10000)
self._load_doc_data_all_buckets('create')
# load items till reached threshold or mem-ratio is less than resident ratio threshold
while not threshold_reached :
mem_used = int(mc.stats()["mem_used"])
if mem_used < threshold or int(mc.stats()["vb_active_perc_mem_resident"]) >= active_resident_threshold:
self.log.info("mem_used and vb_active_perc_mem_resident_ratio reached at %s/%s and %s " % (mem_used, threshold, mc.stats()["vb_active_perc_mem_resident"]))
items = self.num_items
self.num_items += self.input.param("items", 10000)
self._load_doc_data_all_buckets('create', items)
else:
threshold_reached = True
self.log.info("DGM state achieved!!!!")
# wait for draining of data before restart and warm up
for bucket in self.buckets:
RebalanceHelper.wait_for_persistence(self.nodes_server[0], bucket, bucket_type=self.bucket_type)
while True:
# read_data_task = self.cluster.async_verify_data(self.master, self.buckets[0], self.buckets[0].kvs[1])
read_data_task = Thread(target=self._run_get)
read_data_task.start()
#5 threads to run stats all and reset asynchronously
start = time.time()
while (time.time() - start) < 300:
stats_all_thread = []
stats_reset_thread = []
for i in range(self.threads_to_run):
stat_str = ''
stats_all_thread.append(Thread(target=self._get_stats, args=[stat_str]))
stats_all_thread[i].start()
stat_str = 'reset'
stats_reset_thread.append(Thread(target=self._get_stats, args=[stat_str]))
stats_reset_thread[i].start()
for i in range(self.threads_to_run):
stats_all_thread[i].join()
stats_reset_thread[i].join()
del stats_all_thread
del stats_reset_thread
# read_data_task.result()
read_data_task.join()
def test_expired_keys_tap(self):
server = self.master
# callback to track deletes
queue = Queue(maxsize=10000)
listener = TapListener(queue, server, "CMD_TAP_DELETE")
client = MemcachedClientHelper.direct_client(server, self._bucket_name)
expirations = [15]
for expiry in expirations:
testuuid = uuid.uuid4()
keys = ["key_%s_%d" % (testuuid, i) for i in range(1000)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, key)
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(
msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(
len(keys), expiry))
delay = expiry + 15
msg = "sleeping for {0} seconds to wait for those items with expiry set to {1} to expire"
self.log.info(msg.format(delay, expiry))
time.sleep(delay)
self.log.info('verifying that all those keys have expired...')
for key in keys:
try:
client.get(key=key)
msg = "expiry was set to {0} but key: {1} did not expire after waiting for {2}+ seconds"
self.fail(msg.format(expiry, key, delay))
except mc_bin_client.MemcachedError as error:
self.assertEquals(
error.status,
1,
msg="expected error code {0} but saw error code {1}".
format(1, error.status))
self.log.info(
"verified that those keys inserted with expiry set to {0} have expired"
.format(expiry))
listener.start()
try:
was_empty = 0
deletes_seen = 0
while was_empty < 2:
try:
queue.get(False, 5)
deletes_seen += 1
except Empty:
print "exception thrown"
print "how many deletes_seen ? {0}".format(
deletes_seen)
was_empty += 1
self.assertEquals(deletes_seen, 0, msg="some some deletes")
self.log.info("seen {0} CMD_TAP_DELETE".format(deletes_seen))
finally:
listener.aborted = True
def wait_for_mc_stats_all_nodes(master, bucket, stat_key, stat_value,
timeout_in_seconds=120, verbose=True):
log = logging.getLogger("infra")
log.info("waiting for bucket {0} stat : {1} to match {2} on {3}"
.format(bucket, stat_key, stat_value, master.ip))
time_to_timeout = 0
previous_stat_value = -1
curr_stat_value = -1
verified = False
all_stats = {}
while not verified:
rest = RestConnection(master)
nodes = rest.node_statuses()
for node in nodes:
_server = {"ip": node.ip, "port": node.port,
"username": master.rest_username,
"password": master.rest_password}
# Failed over node is part of node_statuses but since
# its failed over memcached connections to this node will fail
node_self = RestConnection(_server).get_nodes_self()
if node_self.clusterMembership == 'active':
mc = MemcachedClientHelper.direct_client(_server, bucket)
n_stats = mc.stats("")
mc.close()
all_stats[node.id] = n_stats
actual_stat_value = -1
for k in all_stats:
if all_stats[k] and stat_key in all_stats[k]:
if actual_stat_value == -1:
log.info(all_stats[k][stat_key])
actual_stat_value = int(all_stats[k][stat_key])
else:
actual_stat_value += int(all_stats[k][stat_key])
if actual_stat_value == stat_value:
log.info("{0} : {1}".format(stat_key, actual_stat_value))
verified = True
break
else:
if verbose:
log.info("{0} : {1}".format(stat_key, actual_stat_value))
curr_stat_value = actual_stat_value
# values are changing so clear any timeout
if curr_stat_value != previous_stat_value:
time_to_timeout = 0
else:
if time_to_timeout == 0:
time_to_timeout = time.time() + timeout_in_seconds
if time_to_timeout < time.time():
log.info("no change in {0} stat after {1} seconds (value = {2})".format(stat_key, timeout_in_seconds, curr_stat_value))
break
previous_stat_value = curr_stat_value
if not verbose:
time.sleep(0.1)
else:
time.sleep(2)
return verified
def set_get_test(self, value_size, number_of_items):
fixed_value = MemcachedClientHelper.create_value("S", value_size)
specs = [
("default", 0),
("set-get-bucket-replica-1", 1),
("set-get-bucket-replica-2", 2),
("set-get-bucket-replica-3", 3),
]
serverInfo = self.master
rest = RestConnection(serverInfo)
bucket_ram = int(rest.get_nodes_self().memoryQuota / 4)
mcport = rest.get_nodes_self().memcached
for name, replica in specs:
rest.create_bucket(name, bucket_ram, "sasl", "password", replica, mcport)
bucket_data = {}
buckets = RestConnection(serverInfo).get_buckets()
for bucket in buckets:
bucket_data[bucket.name] = {}
ready = BucketOperationHelper.wait_for_memcached(serverInfo, bucket.name)
self.test.assertTrue(ready, "wait_for_memcached failed")
client = MemcachedClientHelper.direct_client(serverInfo, bucket.name)
inserted = []
rejected = []
while len(inserted) <= number_of_items and len(rejected) <= number_of_items:
try:
key = str(uuid.uuid4())
client.set(key, 0, 0, fixed_value)
inserted.append(key)
except mc_bin_client.MemcachedError:
pass
retry = 0
remaining_items = []
remaining_items.extend(inserted)
msg = "memcachedError : {0} - unable to get a pre-inserted key : {1}"
while retry < 10 and len(remaining_items) > 0:
verified_keys = []
for key in remaining_items:
try:
flag, keyx, value = client.get(key=key)
if not value == fixed_value:
self.test.fail("value mismatch for key {0}".format(key))
verified_keys.append(key)
except mc_bin_client.MemcachedError as error:
self.log.error(msg.format(error.status, key))
retry += 1
[remaining_items.remove(x) for x in verified_keys]
print_count = 0
for key in remaining_items:
if print_count > 100:
break
print_count += 1
self.log.error("unable to verify key : {0}".format(key))
if remaining_items:
self.test.fail("unable to verify {0} keys".format(len(remaining_items)))
def _get_stats(self, stat_str='all'):
# for server in self.nodes_server:
server = self.servers[0]
mc_conn = MemcachedClientHelper.direct_client(server, self.bucket_name, self.timeout)
stat_result = mc_conn.stats(stat_str)
# self.log.info("Getting stats {0} : {1}".format(stat_str, stat_result))
self.log.info("Getting stats {0}".format(stat_str))
mc_conn.close()
def _test_delete_key_and_backup_and_restore_body(self):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master, name=bucket, test_case=self)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
client.delete(keys[0])
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
#let's create a unique folder in the remote location
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node, self.remote_tmp_folder)
shell.disconnect()
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder)
time.sleep(10)
self.log.info('verifying that all those keys...')
missing_keys = []
verify_keys = []
for key in keys:
vBucketId = crc32.crc32_hash(key) & 1023 # or & 0x3FF
client.vbucketId = vBucketId
if key == keys[0]:
missing_keys.append(key)
else:
verify_keys.append(key)
self.assertTrue(BucketOperationHelper.keys_dont_exist(self.master, missing_keys, self),
"Keys are not empty")
self.assertTrue(BucketOperationHelper.verify_data(self.master, verify_keys, False, False, 11210, self),
"Missing keys")
def _test_delete_key_and_backup_and_restore_body(self):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master, name=bucket, test_case=self)
ready = BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.assertTrue(ready, "wait_for_memcached failed")
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
client.delete(keys[0])
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
#let's create a unique folder in the remote location
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node, self.remote_tmp_folder)
shell.disconnect()
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder)
time.sleep(10)
self.log.info('verifying that all those keys...')
missing_keys = []
verify_keys = []
for key in keys:
vBucketId = crc32.crc32_hash(key) & 1023 # or & 0x3FF
client.vbucketId = vBucketId
if key == keys[0]:
missing_keys.append(key)
else:
verify_keys.append(key)
self.assertTrue(BucketOperationHelper.keys_dont_exist(self.master, missing_keys, self),
"Keys are not empty")
self.assertTrue(BucketOperationHelper.verify_data(self.master, verify_keys, False, False, 11210, self),
"Missing keys")
def _parallel_read(self):
rest = RestConnection(self.master)
buckets = rest.get_buckets()
while not self.reader_shutdown:
for bucket in buckets:
name = bucket.name.encode("ascii", "ignore")
mc = MemcachedClientHelper.direct_client(self.master, name)
for key in self.bucket_data[name]["inserted_keys"]:
mc.get(key)
def _get_stats(self, stat_str='all'):
# for server in self.nodes_server:
server = self.servers[0]
mc_conn = MemcachedClientHelper.direct_client(server, self.bucket_name, self.timeout)
stat_result = mc_conn.stats(stat_str)
# self.log.info("Getting stats {0} : {1}".format(stat_str, stat_result))
self.log.info("Getting stats {0}".format(stat_str))
mc_conn.close()
def _parallel_read(self):
rest = RestConnection(self.master)
buckets = rest.get_buckets()
while not self.reader_shutdown:
for bucket in buckets:
name = bucket.name.encode("ascii", "ignore")
mc = MemcachedClientHelper.direct_client(self.master, name)
for key in self.bucket_data[name]["inserted_keys"]:
mc.get(key)
def set_get_test(self, value_size, number_of_items):
fixed_value = MemcachedClientHelper.create_value("S", value_size)
specs = [("default", 0),
("set-get-bucket-replica-1", 1),
("set-get-bucket-replica-2", 2),
("set-get-bucket-replica-3", 3)]
serverInfo = self.master
rest = RestConnection(serverInfo)
bucket_ram = int(rest.get_nodes_self().memoryQuota / 4)
mcport = rest.get_nodes_self().memcached
for name, replica in specs:
rest.create_bucket(name, bucket_ram, "sasl", "password", replica, mcport)
bucket_data = {}
buckets = RestConnection(serverInfo).get_buckets()
for bucket in buckets:
bucket_data[bucket.name] = {}
ready = BucketOperationHelper.wait_for_memcached(serverInfo, bucket.name)
self.test.assertTrue(ready, "wait_for_memcached failed")
client = MemcachedClientHelper.direct_client(serverInfo, bucket.name)
inserted = []
rejected = []
while len(inserted) <= number_of_items and len(rejected) <= number_of_items:
try:
key = str(uuid.uuid4())
client.set(key, 0, 0, fixed_value)
inserted.append(key)
except mc_bin_client.MemcachedError:
pass
retry = 0
remaining_items = []
remaining_items.extend(inserted)
msg = "memcachedError : {0} - unable to get a pre-inserted key : {1}"
while retry < 10 and len(remaining_items) > 0:
verified_keys = []
for key in remaining_items:
try:
flag, keyx, value = client.get(key=key)
if not value == fixed_value:
self.test.fail("value mismatch for key {0}".format(key))
verified_keys.append(key)
except mc_bin_client.MemcachedError as error:
self.log.error(msg.format(error.status, key))
retry += 1
[remaining_items.remove(x) for x in verified_keys]
print_count = 0
for key in remaining_items:
if print_count > 100:
break
print_count += 1
self.log.error("unable to verify key : {0}".format(key))
if remaining_items:
self.test.fail("unable to verify {0} keys".format(len(remaining_items)))
def _verify_direct_client_stats(self, bucket, command, output):
mc_conn = MemcachedClientHelper.direct_client(self.master, bucket.name, self.timeout)
for line in output:
stats = line.split(":", 1)
self.log.info("CbStats###### for %s:::%s==%s" % (stats[0], mc_conn.stats(command)[stats[0].strip()], stats[1].strip()))
if stats[1].strip() == mc_conn.stats(command)[stats[0].strip()]:
continue
else:
raise Exception("Command does not throw out error message but cbstats does not match.")
def setUp(self):
super(EvictionDCP, self).setUp()
self.dcp_client = DcpClient(self.master.ip, int(11210))
self.dcp_client.sasl_auth_plain(self.master.rest_username, self.master.rest_password)
self.dcp_client.bucket_select('default')
self.dcp_client.open_producer(name='eviction', delete_times=True)
self.rest = RestConnection(self.servers[0])
self.client = MemcachedClientHelper.direct_client(self.master, 'default')
def collect_vbucket_stats(self,
buckets,
servers,
collect_vbucket=True,
collect_vbucket_seqno=True,
collect_vbucket_details=True,
perNode=True):
"""
Method to extract the vbuckets stats given by cbstats tool
Paramters:
buckets: bucket information
servers: server information
collect_vbucket: take vbucket type stats
collect_vbucket_seqno: take vbucket-seqno type stats
collect_vbucket_details: take vbucket-details type stats
perNode: if True collects data per node else takes a union across nodes
Returns:
The output can be in two formats
if we are doing per node data collection
Vbucket Information :: {bucket { node : [vbucket_seqno {key:value}
U vbucket_details {key:value} U vbucket {key:value}]}}
if we are not doing per node data collection
Vbucket Information :: {bucket : [vbucket_seqno {key:value}
U vbucket_details {key:value} U vbucket {key:value}]}
"""
bucketMap = {}
vbucket = []
vbucket_seqno = []
vbucket_details = []
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
map_data = {}
client = MemcachedClientHelper.direct_client(server, bucket)
if collect_vbucket:
vbucket = client.stats('vbucket')
self.createMapVbucket(vbucket, map_data)
if collect_vbucket_seqno:
vbucket_seqno = client.stats('vbucket-seqno')
self.createMapVbucket(vbucket_seqno, map_data)
if collect_vbucket_details:
vbucket_details = client.stats('vbucket-details')
self.createMapVbucket(vbucket_details, map_data)
if perNode:
dataMap[server.ip] = map_data
else:
dataMap.update(map_data)
bucketMap[bucket.name] = dataMap
return bucketMap
def _warmup(self):
warmed_up = False
for server in self.servers:
mc = MemcachedClientHelper.direct_client(server, self.bucket_name)
start = time.time()
if server == self.servers[0]:
wait_time = 300
else:
wait_time = 60
# Try to get the stats for 5 minutes, else hit out.
while time.time() - start < wait_time:
# Get the wamrup time for each server
try:
stats = mc.stats()
if stats is not None:
warmup_time = int(stats["ep_warmup_time"])
self.log.info("ep_warmup_time is %s " % warmup_time)
self.log.info(
"Collected the stats %s for server %s:%s" % (stats["ep_warmup_time"], server.ip,
server.port))
break
else:
self.log.info(" Did not get the stats from the server yet, trying again.....")
time.sleep(2)
except Exception as e:
self.log.error(
"Could not get warmup_time stats from server %s:%s, exception %s" % (server.ip,
server.port, e))
else:
self.fail(
"Fail! Unable to get the warmup-stats from server %s:%s after trying for %s seconds." % (
server.ip, server.port, wait_time))
# Waiting for warm-up
start = time.time()
warmed_up = False
while time.time() - start < self.timeout and not warmed_up:
if mc.stats()["ep_warmup_thread"] == "complete":
self.log.info("warmup completed, awesome!!! Warmed up. %s items " % (mc.stats()["curr_items_tot"]))
time.sleep(5)
if mc.stats()["curr_items_tot"] == self.pre_warmup_stats["%s:%s" % (server.ip, server.port)]["curr_items_tot"]:
self._stats_report(server, mc.stats(self.stat_str))
warmed_up = True
else:
continue
elif mc.stats()["ep_warmup_thread"] == "running":
self.log.info(
"still warming up .... curr_items_tot : %s" % (mc.stats()["curr_items_tot"]))
else:
self.fail("Value of ep warmup thread does not exist, exiting from this server")
time.sleep(5)
mc.close()
if warmed_up:
return True
else:
return False
def wait_for_mc_stats_all_nodes(master, bucket, stat_key, stat_value, timeout_in_seconds=120, verbose=True):
log.info("waiting for bucket {0} stat : {1} to match {2} on {3}".format(bucket, stat_key, \
stat_value, master.ip))
time_to_timeout = 0
previous_stat_value = -1
curr_stat_value = -1
verified = False
all_stats = {}
while not verified:
rest = RestConnection(master)
nodes = rest.node_statuses()
for node in nodes:
_server = {"ip": node.ip, "port": node.port, "username": master.rest_username,
"password": master.rest_password}
#failed over node is part of node_statuses but since its failed over memcached connections
#to this node will fail
node_self = RestConnection(_server).get_nodes_self()
if node_self.clusterMembership == 'active':
mc = MemcachedClientHelper.direct_client(_server, bucket)
n_stats = mc.stats("")
mc.close()
all_stats[node.id] = n_stats
actual_stat_value = -1
for k in all_stats:
if all_stats[k] and stat_key in all_stats[k]:
if actual_stat_value == -1:
print all_stats[k][stat_key]
actual_stat_value = int(all_stats[k][stat_key])
else:
actual_stat_value += int(all_stats[k][stat_key])
if actual_stat_value == stat_value:
log.info("{0} : {1}".format(stat_key, actual_stat_value))
verified = True
break
else:
if verbose:
log.info("{0} : {1}".format(stat_key, actual_stat_value))
curr_stat_value = actual_stat_value
# values are changing so clear any timeout
if curr_stat_value != previous_stat_value:
time_to_timeout = 0
else:
if time_to_timeout == 0:
time_to_timeout = time.time() + timeout_in_seconds
if time_to_timeout < time.time():
log.info("no change in {0} stat after {1} seconds (value = {2})".format(stat_key, timeout_in_seconds, curr_stat_value))
break
previous_stat_value = curr_stat_value
if not verbose:
time.sleep(0.1)
else:
time.sleep(2)
return verified
def test_MB_12751(self):
mc = MemcachedClientHelper.direct_client(self.master, "default")
mc.set("hello", 0, 0, "world")
mc.getl("hello", 15)
try:
ret = mc.replace("hello", 0, 0, "hello")
self.fail("The document should be locked")
except MemcachedError as e:
if e.status != memcacheConstants.ERR_EXISTS:
self.fail("Expected replace to return EEXISTS, returned: {0}".format(e.status))
def test_MB_12751(self):
mc = MemcachedClientHelper.direct_client(self.master, "default")
mc.set("hello", 0, 0, "world")
mc.getl("hello", 15)
try:
ret = mc.replace("hello", 0, 0, "hello")
self.fail("The document should be locked")
except MemcachedError, e:
if e.status != memcacheConstants.ERR_EXISTS:
self.fail("Expected replace to return EEXISTS, returned: {0}".format(e.status))
def _wait_warmup_completed(self, servers, bucket_name, wait_time=300):
warmed_up = False
log = logger.Logger.get_logger()
for server in servers:
mc = None
start = time.time()
# Try to get the stats for 5 minutes, else hit out.
while time.time() - start < wait_time:
# Get the wamrup time for each server
try:
mc = MemcachedClientHelper.direct_client(
server, bucket_name)
stats = mc.stats()
if stats is not None:
warmup_time = int(stats["ep_warmup_time"])
log.info("ep_warmup_time is %s " % warmup_time)
log.info(
"Collected the stats 'ep_warmup_time' %s for server %s:%s"
%
(stats["ep_warmup_time"], server.ip, server.port))
break
else:
log.info(
" Did not get the stats from the server yet, trying again....."
)
time.sleep(2)
except Exception as e:
log.error(
"Could not get ep_warmup_time stats from server %s:%s, exception %s"
% (server.ip, server.port, e))
else:
self.fail(
"Fail! Unable to get the warmup-stats from server %s:%s after trying for %s seconds."
% (server.ip, server.port, wait_time))
# Waiting for warm-up
start = time.time()
warmed_up = False
while time.time() - start < wait_time and not warmed_up:
if mc.stats()["ep_warmup_thread"] == "complete":
log.info(
"warmup completed, awesome!!! Warmed up. %s items " %
(mc.stats()["curr_items_tot"]))
warmed_up = True
continue
elif mc.stats()["ep_warmup_thread"] == "running":
log.info("still warming up .... curr_items_tot : %s" %
(mc.stats()["curr_items_tot"]))
else:
fail(
"Value of ep warmup thread does not exist, exiting from this server"
)
time.sleep(5)
mc.close()
return warmed_up
def test_warmup(self):
ep_threshold = self.input.param("ep_threshold", "ep_mem_low_wat")
active_resident_threshold = int(
self.input.param("active_resident_threshold", 110))
access_log_time = self.input.param("access_log_time", 2)
mc = MemcachedClientHelper.direct_client(self.servers[0],
self.bucket_name)
stats = mc.stats()
threshold = int(self.input.param('threshold', stats[ep_threshold]))
threshold_reached = False
self.num_items = self.input.param("items", 10000)
self._load_doc_data_all_buckets('create')
#load items till reached threshold or mem-ratio is less than resident ratio threshold
while not threshold_reached:
mem_used = int(mc.stats()["mem_used"])
if mem_used < threshold or int(mc.stats(
)["vb_active_perc_mem_resident"]) >= active_resident_threshold:
self.log.info(
"mem_used and vb_active_perc_mem_resident_ratio reached at %s/%s and %s "
% (mem_used, threshold,
mc.stats()["vb_active_perc_mem_resident"]))
items = self.num_items
self.num_items += self.input.param("items", 10000)
self._load_doc_data_all_buckets('create', items)
else:
threshold_reached = True
self.log.info("DGM state achieved!!!!")
#parallel load of data
items = self.num_items
self.num_items += 10000
tasks = self._async_load_doc_data_all_buckets('create', items)
#wait for draining of data before restart and warm up
rest = RestConnection(self.servers[0])
self.servers = rest.get_nodes()
self._wait_for_stats_all_buckets(self.servers)
self._stats_befor_warmup()
for task in tasks:
task.result()
#If warmup is done through access log then run access scanner
if self.access_log:
scanner_runs = int(mc.stats()["ep_num_access_scanner_runs"])
self.log.info("setting access scanner time %s minutes" %
access_log_time)
self.log.info("current access scanner run is %s" % scanner_runs)
ClusterOperationHelper.flushctl_set(self.servers[0],
"alog_sleep_time",
access_log_time,
self.bucket_name)
if not self._wait_for_access_run(access_log_time, scanner_runs,
mc):
self.fail("Not able to create access log within %s" %
access_log_time)
self._restart_memcache()
if self._warmup():
self._load_doc_data_all_buckets('update', self.num_items - items)
def _test_backup_and_restore_bucket_overwriting_body(self, overwrite_flag=True):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master, test_case=self)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node, self.remote_tmp_folder)
shell.disconnect()
for key in keys:
try:
client.replace(key, expiry, 0, "2")
except mc_bin_client.MemcachedError as error:
msg = "unable to replace key : {0} in bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("replaced {0} keys with expiry set to {1}".format(len(keys), expiry))
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder, overwrite_flag)
time.sleep(10)
self.log.info('verifying that all those keys...')
for key in keys:
if overwrite_flag:
self.assertEqual("2", client.get(key=key), key + " should has value = 2")
else:
self.assertNotEqual("2", client.get(key=key), key + " should not has value = 2")
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def _test_backup_and_restore_bucket_overwriting_body(self, overwrite_flag=True):
bucket = "default"
BucketOperationHelper.create_bucket(serverInfo=self.master, test_case=self)
BucketOperationHelper.wait_for_memcached(self.master, bucket)
self.add_nodes_and_rebalance()
client = MemcachedClientHelper.direct_client(self.master, "default")
expiry = 2400
test_uuid = uuid.uuid4()
keys = ["key_%s_%d" % (test_uuid, i) for i in range(500)]
self.log.info("pushing keys with expiry set to {0}".format(expiry))
for key in keys:
try:
client.set(key, expiry, 0, "1")
except mc_bin_client.MemcachedError as error:
msg = "unable to push key : {0} to bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("inserted {0} keys with expiry set to {1}".format(len(keys), expiry))
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_queue_size', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
ready = RebalanceHelper.wait_for_stats_on_all(self.master, bucket, 'ep_flusher_todo', 0)
self.assertTrue(ready, "wait_for ep_queue_size == 0 failed")
for server in self.servers:
shell = RemoteMachineShellConnection(server)
output, error = shell.execute_command(self.perm_command)
shell.log_command_output(output, error)
node = RestConnection(server).get_nodes_self()
BackupHelper(server, self).backup(bucket, node, self.remote_tmp_folder)
shell.disconnect()
for key in keys:
try:
client.replace(key, expiry, 0, "2")
except mc_bin_client.MemcachedError as error:
msg = "unable to replace key : {0} in bucket : {1} error : {2}"
self.log.error(msg.format(key, client.vbucketId, error.status))
self.fail(msg.format(key, client.vbucketId, error.status))
self.log.info("replaced {0} keys with expiry set to {1}".format(len(keys), expiry))
for server in self.servers:
BackupHelper(server, self).restore(self.remote_tmp_folder, overwrite_flag)
time.sleep(10)
self.log.info('verifying that all those keys...')
for key in keys:
if overwrite_flag:
self.assertEqual("2", client.get(key=key), key + " should has value = 2")
else:
self.assertNotEqual("2", client.get(key=key), key + " should not has value = 2")
self.log.info("verified that those keys inserted with expiry set to {0} have expired".format(expiry))
def init_clients(self):
"""Initialise clients for all servers there are vBuckets on
It returns a dict with 'ip:port' as key (this information is also
stored this way in every vBucket in the `master` property) and
the MemcachedClient as the value
"""
clients = {}
for vbucket in self.bucket.vbuckets:
if vbucket.master not in clients:
ip, port = vbucket.master.split(':')
sport = str((int(port[-2:])) / 2 + int(self.master.port))
if sport:
clients[vbucket.master] = \
MemcachedClientHelper.direct_client(self._get_server(sport),\
self.default_bucket_name)
else:
clients[vbucket.master] = MemcachedClientHelper.direct_client(self.master,\
self.default_bucket_name)
return clients
def test_partial_rollback_via_memcached_restart_and_persistance_stopped(self):
self.log.info("Load data in the default bucket")
self.perform_doc_ops_in_all_cb_buckets("create", 0, self.num_items, exp=0)
self.bucket_util.verify_stats_all_buckets(self.num_items)
self.log.info("Create connection")
self.cbas_util.createConn(self.cb_bucket_name)
self.log.info("Create additional CBAS bucket and connect after failover logs are generated")
secondary_cbas_bucket_name = self.cbas_bucket_name + "_secondary"
secondary_dataset = self.cbas_dataset_name + "_secondary"
self.log.info("Create dataset on the CBAS bucket")
self.cbas_util.create_dataset_on_bucket(cbas_bucket_name=self.cb_bucket_name,
cbas_dataset_name=self.cbas_dataset_name)
self.log.info("Create dataset on the CBAS secondary bucket")
self.cbas_util.create_dataset_on_bucket(cbas_bucket_name=self.cb_bucket_name,
cbas_dataset_name=secondary_dataset)
self.log.info("Connect to Bucket")
self.cbas_util.connect_to_bucket(cbas_bucket_name=self.cbas_bucket_name,
cb_bucket_password=self.cb_bucket_password)
self.log.info("Validate count on CBAS")
self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items)
self.log.info("Establish remote shell to master node")
shell = RemoteMachineShellConnection(self.cluster.master)
number_of_times_memcached_restart = self.input.param("number_of_times_memcached_restart", 16)
for i in range(number_of_times_memcached_restart):
self.log.info("Stop persistance on KV node")
mem_client = MemcachedClientHelper.direct_client(self.cluster.master,
self.cb_bucket_name)
mem_client.stop_persistence()
self.log.info("Add documents with persistance stopped")
self.perform_doc_ops_in_all_cb_buckets("create", self.num_items, self.num_items + (self.num_items / 2), exp=0)
self.log.info("Validate count on CBAS")
self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items+ (self.num_items / 2))
self.log.info("Kill memcached on KV node %s" %str(i))
shell.kill_memcached()
self.sleep(2, "Wait for for DCP rollback sent to CBAS and memcached restart")
self.log.info("Validate count on CBAS")
self.assertTrue(self.cbas_util.validate_cbas_dataset_items_count(self.cbas_dataset_name, self.num_items), msg="Count mismatch")
self.log.info("Verify connect to second CBAS Bucket succeeds post long failure logs")
self.assertTrue(self.cbas_util.connect_to_bucket(cbas_bucket_name=secondary_cbas_bucket_name,
cb_bucket_password=self.cb_bucket_password), msg="Failed to connect CBAS bucket after long failover logs")
def membase_stats(self,nodes, bucket, frequency, verbose=False):
mcs = []
for node in nodes:
try:
bucket = RestConnection(node).get_buckets()[0].name
mcs.append(MemcachedClientHelper.direct_client(node, bucket))
except:
pass
self._task["membasestats"] = []
self._task["timings"] = []
self._task["dispatcher"] = []
d = {}
# "pname":"x","pid":"y","snapshots":[{"time":time,"value":value}]
for mc in mcs:
d[mc.host] = {"snapshots": [], "timings":[], "dispatcher":[]}
while not self._aborted():
time_left = frequency
# at minimum we want to check for aborted every minute
while not self._aborted() and time_left > 0:
time.sleep(min(time_left, 60))
time_left -= 60
for mc in mcs:
stats = mc.stats()
stats["time"] = time.time()
stats["ip"] = mc.host
d[mc.host]["snapshots"].append(stats)
timings = mc.stats('timings')
d[mc.host]["timings"].append(timings)
dispatcher = mc.stats('dispatcher')
d[mc.host]["dispatcher"].append(dispatcher)
start_time = str(self._task["time"])
for mc in mcs:
ip = mc.host
unique_id = ip+'-'+start_time
current_time = time.time()
for snapshot in d[mc.host]["snapshots"]:
snapshot['unique_id'] = unique_id
snapshot['time'] = current_time
snapshot['ip'] = ip
self._task["membasestats"].append(snapshot)
for timing in d[mc.host]["timings"]:
timing['unique_id'] = unique_id
timing['time'] = current_time
timing['ip'] = ip
self._task["timings"].append(timing)
for dispatcher in d[mc.host]["dispatcher"]:
dispatcher['unique_id'] = unique_id
dispatcher['time'] = current_time
dispatcher['ip'] = ip
self._task["dispatcher"].append(dispatcher)
print " finished membase_stats"
def test_rollback_and_persistence_race_condition(self):
nodeA = self.servers[0]
vbucket_client = VBucketAwareMemcached(RestConnection(self.master), 'default')
gen_create = BlobGenerator('dcp', 'dcp-', 64, start=0, end=self.num_items)
self._load_all_buckets(nodeA, gen_create, "create", 0)
# stop persistence
for bucket in self.buckets:
for s in self.servers:
client = MemcachedClientHelper.direct_client(s, bucket)
client.stop_persistence()
vb_uuid, seqno, high_seqno = self.vb_info(self.servers[0], 5)
time.sleep(10)
# more (non-intersecting) load
gen_create = BlobGenerator('dcp-secondgroup', 'dcpsecondgroup-', 64, start=0, end=self.num_items)
self._load_all_buckets(nodeA, gen_create, "create", 0)
shell = RemoteMachineShellConnection(self.servers[0])
shell.kill_memcached()
time.sleep(10)
mc1 = MemcachedClientHelper.direct_client(self.servers[0], "default")
mc2 = MemcachedClientHelper.direct_client(self.servers[1], "default")
node1_items = mc1.stats()["curr_items_tot"]
node2_items = mc2.stats()["curr_items_tot"]
self.assertTrue( node1_items == node2_items,
'Node items not equal. Node 1:{0}, node 2:{1}'.format(node1_items, node2_items ))
def test_rollback_and_persistence_race_condition(self):
nodeA = self.servers[0]
vbucket_client = VBucketAwareMemcached(RestConnection(self.master), 'default')
gen_create = BlobGenerator('dcp', 'dcp-', 64, start=0, end=self.num_items)
self._load_all_buckets(nodeA, gen_create, "create", 0)
# stop persistence
for bucket in self.buckets:
for s in self.servers[:self.nodes_init]:
client = MemcachedClientHelper.direct_client(s, bucket)
try:
client.stop_persistence()
except MemcachedError as e:
if self.bucket_type == 'ephemeral':
self.assertTrue("Memcached error #4 'Invalid': Flusher not running. for vbucket :0 to mc " in e.message)
return
else:
raise
vb_uuid, seqno, high_seqno = self.vb_info(self.servers[0], 5)
time.sleep(10)
# more (non-intersecting) load
gen_create = BlobGenerator('dcp-secondgroup', 'dcpsecondgroup-', 64, start=0, end=self.num_items)
self._load_all_buckets(nodeA, gen_create, "create", 0)
shell = RemoteMachineShellConnection(self.servers[0])
shell.kill_memcached()
time.sleep(10)
mc1 = MemcachedClientHelper.direct_client(self.servers[0], "default")
mc2 = MemcachedClientHelper.direct_client(self.servers[1], "default")
node1_items = mc1.stats()["curr_items_tot"]
node2_items = mc2.stats()["curr_items_tot"]
self.assertTrue(node1_items == node2_items,
'Node items not equal. Node 1:{0}, node 2:{1}'.format(node1_items, node2_items))
def collect_failovers_stats(self, buckets, servers, perNode=True):
"""
Method to extract the failovers stats given by cbstats tool
Paramters:
buckets: bucket informaiton
servers: server information
perNode: if set collect per node information else all
Returns:
Failover stats as follows:
if not collecting per node :: {bucket : [{key:value}]}
if collecting per node :: {bucket : {node:[{key:value}]}}
"""
bucketMap = {}
for bucket in buckets:
bucketMap[bucket.name] = {}
for bucket in buckets:
dataMap = {}
for server in servers:
client = MemcachedClientHelper.direct_client(server, bucket)
stats = client.stats('failovers')
map_data = {}
num_map = {}
for o in stats.keys():
tokens = o.split(":")
vb = tokens[0]
key = tokens[1]
value = stats[o].split()
num = -1
if len(tokens) == 3:
vb = tokens[0]
num = int(tokens[1])
key = tokens[2]
if vb in map_data.keys() and (num == num_map[vb]
or num > num_map[vb]):
map_data[vb][key] = value[0]
num_map[vb] = num
elif vb in map_data.keys() and key == "num_entries":
map_data[vb][key] = value[0]
elif vb not in map_data.keys():
m = {}
m[key] = value[0]
map_data[vb] = m
num_map[vb] = num
if perNode:
dataMap[server.ip] = map_data
else:
dataMap.update(map_data)
bucketMap[bucket.name] = dataMap
return bucketMap
def get_checkpoints_from_cluster(self, master, bucket):
parser = CheckpointStatParser()
rest = RestConnection(master)
servers = rest.get_nodes()
merged = {}
for server in servers:
mc = MemcachedClientHelper.direct_client(server, bucket)
per_node_checkpoint = mc.stats("checkpoint")
parsed = parser.parse_output(per_node_checkpoint, server.id)
merged = parser.merge_results(merged, parsed)
mc.close()
return merged
