def test_crash_while_streaming(self):
bucket = self.bucket_util.buckets[0]
vbucket = randint(0, self.vbuckets)
nodeA = self.servers[0]
self.load_docs(bucket, vbucket, 0, self.num_items, "create")
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
_ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0)
self.load_docs(nodeA, vbucket, self.num_items)
self.assertTrue(self.stop_node(0), msg="Failed during stop_node")
self.sleep(2, "Sleep after stop_node")
self.assertTrue(self.start_node(0), msg="Failed during start_node")
rest = RestHelper(RestConnection(nodeA))
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
self.sleep(30, "Sleep to wait for ns_server to run")
vb_info = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_info[vbucket]["high_seqno"], 0)
stream.run()
self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"],
msg="Mismatch in high_seqno. {0} == {1}".format(
vb_info[vbucket]["high_seqno"],
stream.last_by_seqno))
# Disconnect shell Connection for the node
shell_conn.disconnect()
def test_mutations_during_rebalance(self):
# start rebalance
task = self.cluster.async_rebalance([self.master], self.servers[1:],
[])
# load some data
vbucket = 0
self.load_docs(self.master, vbucket, self.num_items)
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cb_stat_obj = Cbstats(self.log, shell_conn)
# Fetch vbucket seqno stats
vb_stat = cb_stat_obj.vbucket_seqno(self.bucket_util.buckets[0].name)
# stream
self.log.info("Streaming vb {0} to seqno {1}"
.format(vbucket, vb_stat[vbucket]["high_seqno"]))
self.assertEquals(vb_stat[vbucket]["high_seqno"], self.num_items)
dcp_client = self.dcp_client(self.master, PRODUCER, vbucket)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_stat[vbucket]["high_seqno"],
vb_stat[vbucket]["uuid"])
stream.run()
last_seqno = stream.last_by_seqno
assert last_seqno == vb_stat[vbucket]["high_seqno"], last_seqno
# verify rebalance
assert task.result()
def test_failover_swap_rebalance(self):
""" add and failover node then perform swap rebalance """
assert len(self.servers) > 2, "not enough servers"
nodeA = self.servers[0]
nodeB = self.servers[1]
nodeC = self.servers[2]
gen_create = doc_generator('dcp', 0, self.num_items, doc_size=64)
self._load_all_buckets(nodeA, gen_create, "create", 0)
vbucket = 0
# rebalance in nodeB
assert self.cluster.rebalance([nodeA], [nodeB], [])
# add nodeC
rest = RestConnection(nodeB)
rest.add_node(user=nodeC.rest_username,
password=nodeC.rest_password,
remoteIp=nodeC.ip,
port=nodeC.port)
# stop and failover nodeA
assert self.stop_node(0)
self.stopped_nodes.append(0)
self.master = nodeB
assert self.cluster.failover([nodeB], [nodeA])
try:
assert self.cluster.rebalance([nodeB], [], [])
except Exception:
pass
self.add_built_in_server_user()
# verify seqnos and stream mutations
rest = RestConnection(nodeB)
total_mutations = 0
# Create connection for CbStats
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cb_stat_obj = Cbstats(shell_conn)
vb_info = cb_stat_obj.vbucket_seqno(self.bucket_util.buckets[0].name)
for vb in range(0, self.vbuckets):
total_mutations += int(vb_info[vb]["high_seqno"])
# Disconnect the Cbstats shell_conn
shell_conn.disconnect()
# / 2 # divide by because the items are split between 2 servers
self.assertTrue(total_mutations == self.num_items,
msg="Number mismatch. {0} != {1}".format(
total_mutations, self.num_items))
task = self.cluster.async_rebalance([nodeB], [], [nodeC])
task.result()
def test_stream_all_buckets(self):
doc_gen = doc_generator(self.key, 0, self.num_items)
self._load_all_buckets(self.master, doc_gen, "create", 0)
user_name = self.input.param("user_name", None)
password = self.input.param("password", None)
nodeA = self.servers[0]
vbuckets = [vb for vb in range(self.vbuckets)]
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
for bucket in self.bucket_util.buckets:
if user_name is not None:
self.add_built_in_server_user([{
'id': user_name,
'name': user_name,
'password': password
}], [{
'id': user_name,
'name': user_name,
'roles': 'data_dcp_reader[default]'
}], self.master)
dcp_client = self.dcp_client(nodeA,
PRODUCER,
bucket_name=bucket.name,
auth_user=user_name,
auth_password=password)
else:
dcp_client = self.dcp_client(nodeA,
PRODUCER,
bucket_name=bucket)
vb_info = cb_stat_obj.vbucket_seqno(bucket.name)
for vb in vbuckets[0:16]:
vbucket = vb.id
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_info[vb]["high_seqno"],
vb_info[vb]["uuid"])
_ = stream.run()
self.assertTrue(
vb_info[vb]["high_seqno"] == stream.last_by_seqno,
msg="Mismatch in high_seqno. {0} == {1}".format(
vb_info[vb]["high_seqno"], stream.last_by_seqno))
# Disconnect the shell_conn
shell_conn.disconnect()
def test_stream_after_n_crashes(self):
crashes = self.input.param("crash_num", 5)
vbucket = randint(0, self.vbuckets)
bucket = self.bucket_util.buckets[0]
self.log.info("Chosen vbucket {0} for {1} crashes".format(
vbucket, crashes))
start = 0
end = self.num_items
nodeA = self.cluster.servers[0]
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
rest = RestHelper(RestConnection(nodeA))
for _ in xrange(crashes):
# Load data into the selected vbucket
self.load_docs(bucket, vbucket, start, end, "create")
self.assertTrue(self.stop_node(0), msg="Failed during stop_node")
self.sleep(5, "Sleep after stop_node")
self.assertTrue(self.start_node(0), msg="Failed during start_node")
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
self.sleep(5, "Waiting after ns_server started")
# Fetch vbucket seqno stats
vb_stat = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_stat[vbucket]["high_seqno"],
vb_stat[vbucket]["uuid"])
stream.run()
self.assertTrue(
stream.last_by_seqno == vb_stat[vbucket]["high_seqno"],
msg="Mismatch in high_seqno. {0} == {1}".format(
vb_stat[vbucket]["high_seqno"], stream.last_by_seqno))
# Update start/end values for next loop
start = end
end += self.num_items
# Disconnect shell Connection for the node
shell_conn.disconnect()
def test_crash_entire_cluster(self):
self.cluster.rebalance([self.master], self.servers[1:], [])
bucket = self.bucket_util.buckets[0]
vbucket = randint(0, self.vbuckets)
nodeA = self.servers[0]
self.load_docs(bucket, vbucket, 0, self.num_items, "create")
shell_conn = RemoteMachineShellConnection(nodeA)
cb_stat_obj = Cbstats(shell_conn)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
_ = dcp_client.stream_req(vbucket, 0, 0, 2 * self.num_items, 0)
self.load_docs(nodeA, vbucket, self.num_items)
# stop all nodes
node_range = range(len(self.servers))
for i in node_range:
self.assertTrue(self.stop_node(i), msg="Failed during stoip_node")
self.sleep(2, "Wait after stop_node")
# start all nodes in reverse order
node_range.reverse()
for i in node_range:
self.assertTrue(self.start_node(i), msg="Failed during start_node")
rest = RestHelper(RestConnection(nodeA))
self.assertTrue(rest.is_ns_server_running(),
msg="Failed while is_ns_server_running check")
vb_info = cb_stat_obj.vbucket_seqno(bucket.name)
dcp_client = self.dcp_client(nodeA, dcp.constants.PRODUCER)
stream = dcp_client.stream_req(vbucket, 0, 0,
vb_info[vbucket]["high_seqno"], 0)
stream.run()
self.assertTrue(stream.last_by_seqno == vb_info[vbucket]["high_seqno"],
msg="Seq-no mismatch. {0} != {1}".format(
stream.last_by_seqno,
vb_info[vbucket]["high_seqno"]))
# Disconnect shell Connection for the node
shell_conn.disconnect()
def collect_vbucket_stats(self,
buckets,
servers,
collect_vbucket=True,
collect_vbucket_seqno=True,
collect_vbucket_details=True,
perNode=True):
"""
Method to extract the vbuckets stats given by cbstats tool
Paramters:
buckets: bucket information
servers: server information
collect_vbucket: take vbucket type stats
collect_vbucket_seqno: take vbucket-seqno type stats
collect_vbucket_details: take vbucket-details type stats
perNode: if True collects data per node else takes a union
across nodes
Returns:
The output can be in two formats
if we are doing per node data collection
Vbucket Information :: {bucket { node : [vbucket_seqno {key:value} U vbucket_details {key:value} U vbucket {key:value}]}}
if we are not doing per node data collection
Vbucket Information :: {bucket : [vbucket_seqno {key:value} U vbucket_details {key:value} U vbucket {key:value}]}
"""
bucketMap = dict()
for bucket in buckets:
if bucket.bucketType == Bucket.Type.MEMCACHED:
continue
dataMap = dict()
for server in servers:
map_data = dict()
cbstat = Cbstats(server)
if collect_vbucket:
result = dict()
for vb_type in ["active", "replica"]:
vb_list = cbstat.vbucket_list(bucket.name, vb_type)
for vb_num in vb_list:
result['vb_%s' % vb_num] = dict()
result['vb_%s' % vb_num]["state"] = vb_type
map_data.update(result)
# vbucket = client.stats('vbucket')
# self.createMapVbucket(vbucket, map_data)
if collect_vbucket_seqno:
result = cbstat.vbucket_seqno(bucket.name)
for key in result.keys():
result['vb_' + key] = result.pop(key)
map_data.update(result)
# vbucket_seqno = client.stats('vbucket-seqno')
# self.createMapVbucket(vbucket_seqno, map_data)
if collect_vbucket_details:
result = cbstat.vbucket_details(bucket.name)
for key in result.keys():
result['vb_' + key] = result.pop(key)
map_data.update(result)
# vbucket_details = client.stats('vbucket-details')
# self.createMapVbucket(vbucket_details, map_data)
if perNode:
dataMap[server.ip] = map_data
else:
dataMap.update(map_data)
bucketMap[bucket.name] = dataMap
return bucketMap
def test_failover_log_table_updated(self):
"""
Verifies failover table entries are updated when vbucket
ownership changes
"""
# rebalance in nodeB
nodeA = self.servers[0]
nodeB = self.servers[1]
# load nodeA only
rest = RestConnection(nodeA)
vbuckets = rest.get_vbuckets()
for vb_info in vbuckets[0:4]:
vbucket = vb_info.id
self.load_docs(nodeA, vbucket, self.num_items)
# add nodeB
self.cluster.rebalance([nodeA], [nodeB], [])
# stop nodeA and failover
assert self.stop_node(0)
self.stopped_nodes.append(0)
self.master = nodeB
assert self.cluster.failover([nodeB], [nodeA])
assert self.cluster.rebalance([nodeB], [], [])
# load nodeB only
rest = RestConnection(nodeB)
vbuckets = rest.get_vbuckets()
for vb_info in vbuckets[0:4]:
vbucket = vb_info.id
self.load_docs(nodeB, vbucket, self.num_items)
# add nodeA back
assert self.start_node(0)
del self.stopped_nodes[0]
rest = RestHelper(RestConnection(nodeA))
assert rest.is_ns_server_running()
time.sleep(10)
self.cluster.rebalance([nodeB], [nodeA], [])
# stop nodeB and failover
assert self.stop_node(1)
self.master = nodeA
self.stopped_nodes.append(1)
assert self.cluster.failover([nodeA], [nodeB])
assert self.cluster.rebalance([nodeA], [], [])
# load nodeA only
rest = RestConnection(nodeA)
vbuckets = rest.get_vbuckets()
for vb_info in vbuckets[0:4]:
vbucket = vb_info.id
self.load_docs(nodeA, vbucket, self.num_items)
# Create connection for CbStats
shell_conn = RemoteMachineShellConnection(self.cluster.master)
cb_stat_obj = Cbstats(shell_conn)
# Fetch bucket's failover stats
bucket = self.bucket_util.buckets[0]
stats = cb_stat_obj.failover_stats(bucket.name)
# Disconnect the Cbstats shell_conn
shell_conn.disconnect()
# Fetch vbucket seqno stats
vb_stat = cb_stat_obj.vbucket_seqno(bucket.name)
# Check failover table entries
for vb_info in vbuckets[0:4]:
vb = vb_info.id
assert long(stats['vb_'+str(vb)+':num_entries']) == 2
dcp_client = self.dcp_client(nodeA, PRODUCER)
stream = dcp_client.stream_req(vb, 0, 0, self.num_items*3,
vb_stat[vb]["uuid"])
_ = stream.run()
assert stream.last_by_seqno == self.num_items*3, \
stream.last_by_seqno
