class XDCRCallable:
def __init__(self, nodes, num_clusters=2):
self.log = logger.Logger.get_logger()
self.cluster_list = []
self.__clusterop = Cluster()
self.setup_xdcr(nodes, num_clusters)
def setup_xdcr(self, nodes, num_clusters):
self._setup_xdcr_topology(nodes, num_clusters)
def __assign_nodes_to_clusters(self, nodes, num_nodes_per_cluster):
for _ in range(0, len(nodes), num_nodes_per_cluster):
yield nodes[_:_ + num_nodes_per_cluster]
def _setup_xdcr_topology(self, nodes, num_clusters):
num_nodes_per_cluster = len(nodes) // num_clusters
count = 1
for cluster_nodes in list(self.__assign_nodes_to_clusters(nodes, num_nodes_per_cluster)):
cluster = CouchbaseCluster(name="C" + str(count), nodes=cluster_nodes, log=self.log)
self.cleanup_cluster(cluster)
#cluster.cleanup_cluster(test_case="xdcr upgrade")
self.__init_cluster(cluster)
self.log.info("Cluster {0}:{1} created".format(cluster.get_name(), cluster.get_nodes()))
self.cluster_list.append(cluster)
count += 1
# TODO: implementing chain topology for now, need to extend to other xdcr topologies
# C1->C2, C2->C3..
for count, cluster in enumerate(self.cluster_list):
if count < len(self.cluster_list) - 1:
cluster.add_remote_cluster(self.cluster_list[count + 1],
'C' + str(count) + "-to-" + 'C' + str(count + 1))
def ___init_nodes(self, cluster, disabled_consistent_view=None):
"""Initialize all nodes.
"""
tasks = []
for node in cluster.get_nodes():
tasks.append(
self.__clusterop.async_init_node(
node))
for task in tasks:
task.result(60)
def __init_cluster(self, cluster):
"""Initialize cluster.
1. Initialize all nodes.
2. Add all nodes to the cluster.
"""
self.___init_nodes(cluster)
self.__clusterop.async_rebalance(
cluster.get_nodes(),
cluster.get_nodes()[1:],
[]).result()
def cleanup_cluster(self, cluster):
"""Cleanup cluster.
1. Remove all remote cluster references.
2. Remove all replications.
3. Remove all buckets.
"""
self.log.info("removing xdcr/nodes settings")
rest = RestConnection(cluster.get_master_node())
rest.remove_all_replications()
rest.remove_all_remote_clusters()
rest.remove_all_recoveries()
cluster.cleanup_cluster("upgradeXDCR")
def _create_replication(self):
for cluster in self.cluster_list:
self.log.info("Creating replication from {0}->{1}".format(cluster.get_name, cluster.get_remote_clusters()))
def _set_replication_properties(self, param_str):
pass
def _get_replication_properties(self, replid):
pass
def __del__(self):
for cluster in self.cluster_list:
self.cleanup_cluster(cluster)
class nwusage(XDCRNewBaseTest):
def setUp(self):
super(nwusage, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.cluster = Cluster()
self.num_src_nodes = len(self.src_cluster.get_nodes())
self.num_dest_nodes = len(self.dest_cluster.get_nodes())
def tearDown(self):
super(nwusage, self).tearDown()
def _get_nwusage_limit(self):
# Pick random nw_limit between 1-100 MB
return random.randint(1, 100)
def _set_nwusage_limit(self, cluster, nw_limit):
repl_id = cluster.get_remote_clusters()[0].get_replications(
)[0].get_repl_id()
shell = RemoteMachineShellConnection(cluster.get_master_node())
repl_id = str(repl_id).replace('/', '%2F')
self.log.info(
"Network bandwidth is throttled at {0} MB".format(nw_limit))
base_url = "http://" + cluster.get_master_node(
).ip + ":8091/settings/replications/" + repl_id
command = "curl -X POST -u Administrator:password " + base_url + " -d networkUsageLimit=" + str(
nw_limit)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
def _set_doc_size_num(self):
# Weighted randomization of doc sizes ranging from 500 KB to 20 MB
self.doc_sizes = [500] * 10 + [10 ** 3] * 30 + [10 ** 4] * 20 + [10 ** 5] * 15 + \
[10 ** 6] * 10 + [10 ** 7] * 10 + [2 * 10 ** 7] * 5
self._value_size = random.choice(self.doc_sizes)
self._num_items = 10**7
self._temp = self._value_size
# Decrease number of docs as size increases
while self._temp > 10:
self._temp //= 10
self._num_items //= 10
self._value_size *= self.num_src_nodes
self._num_items *= self.num_src_nodes
self.log.info("Doc size = {0} bytes, Number of docs = {1}".format(
self._value_size, self._num_items))
def _extract_timestamp(self, logmsg):
# matches timestamp format : 2018-10-11T00:02:35
timestamp_str = re.search(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}',
logmsg)
timestamp = datetime.datetime.strptime(timestamp_str.group(),
'%Y-%m-%dT%H:%M:%S')
return timestamp
'''Extract current, non zero bandwidth usage stats from logs'''
def _extract_bandwidth_usage(self, node, time_to_compare, nw_max, nw_usage,
end_time):
valid_count = 0
skip_count = 0
matches, count = NodeHelper.check_goxdcr_log(
node,
"\\\"bandwidth_usage\\\": " + nw_usage,
print_matches=True,
timeout=60)
for item in matches:
item_datetime = self._extract_timestamp(item)
# Ignore entries that happened before the replication was set up
if item_datetime < time_to_compare:
skip_count += 1
continue
if end_time:
end_datetime = self._extract_timestamp(end_time)
if item_datetime > end_datetime:
skip_count += 1
continue
bandwidth_usage = int(
float(((item.split('"bandwidth_usage": ')[1]).split(' ')[0]
).rstrip(',')))
if bandwidth_usage > nw_max:
self.fail(
"Bandwidth usage {0} is higher than Bandwidth limit {1} in {2}"
.format(bandwidth_usage, nw_max, item))
self.log.info("BANDWIDTH_USAGE ={0}".format(bandwidth_usage))
if nw_usage == "0" and bandwidth_usage != 0:
self.fail(
"Expecting bandwidth usage to be 0 but it is {0}".format(
bandwidth_usage))
valid_count += 1
self.log.info("Stale entries :{0}, Valid entries :{1}".format(
skip_count, valid_count))
return valid_count
def _extract_bandwith_quota(self, node):
matches, count = NodeHelper.check_goxdcr_log(node,
"bandwidth_usage_quota=" +
"[0-9][0-9]*",
print_matches=True,
timeout=60)
bandwidth_quota = int(
float(((
matches[-1].split('bandwidth_usage_quota=')[1]).rstrip(' '))))
return bandwidth_quota
def _verify_bandwidth_usage(self,
node,
nw_limit,
no_of_nodes,
event_time=None,
nw_usage="[0-9][0-9]*",
end_time=None):
#nw_max = (nw_limit * 1024 * 1024) // no_of_nodes
if event_time:
time_to_compare = self._extract_timestamp(event_time)
else:
matches, count = NodeHelper.check_goxdcr_log(
node,
"Success adding replication specification",
print_matches=True,
timeout=60)
# Time when replication was set up
if count > 0:
time_to_compare = self._extract_timestamp(matches[-1])
else:
self.fail("Replication not successful")
nw_max = self._extract_bandwith_quota(node)
self.sleep(60, 'Waiting for bandwidth usage logs..')
# Try 3 times to extract current bandwidth usage from logs
iter = 0
while iter < 3:
self.sleep(30, 'Waiting for bandwidth usage logs..')
valid_count = self._extract_bandwidth_usage(
node, time_to_compare, nw_max, nw_usage, end_time)
if valid_count == 0 and self._input.param(
"replication_type") == "capi" or nw_limit == 0:
self.log.info(
"Bandwidth Throttler not enabled on replication as expected"
)
break
if valid_count > 0:
break
iter += 1
else:
self.fail("Bandwidth Throttler not enabled!")
# Check if large docs are not getting stuck
matches, src_count = NodeHelper.check_goxdcr_log(
self.src_master,
"The connection is ruined",
print_matches=True,
timeout=10)
if src_count:
for item in matches:
item_datetime = self._extract_timestamp(item)
# Ignore errors that happened before the replication was set up
if item_datetime < time_to_compare:
continue
else:
self.fail("Possibly hit MB-31765")
def _get_current_time(self, server):
shell = RemoteMachineShellConnection(server)
command = "date +'%Y-%m-%dT%H:%M:%S'"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
curr_time = output[0].strip()
return curr_time
def _get_generator(self, prefix, docsize, numitems):
if self._use_java_sdk:
gen_create = SDKDataLoader(num_ops=numitems,
percent_create=100,
key_prefix=prefix,
doc_size=docsize,
timeout=1000)
else:
gen_create = BlobGenerator(prefix, prefix, docsize, end=numitems)
return gen_create
def test_nwusage_with_unidirection(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_src_nodes)
def test_nwusage_with_bidirection(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
src_nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
src_nw_limit * self.num_src_nodes)
self._set_doc_size_num()
dest_nw_limit = self._input.param("nw_limit",
self._get_nwusage_limit())
self._set_nwusage_limit(self.dest_cluster,
dest_nw_limit * self.num_dest_nodes)
gen_create1 = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = self._get_generator('nwTwo', self._value_size,
self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create2)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=src_nw_limit,
no_of_nodes=self.num_src_nodes)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=dest_nw_limit,
no_of_nodes=self.num_dest_nodes)
def test_nwusage_with_unidirection_pause_resume(self):
self.setup_xdcr()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
self.src_cluster.pause_all_replications()
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
self.src_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_src_nodes)
def test_nwusage_with_bidirection_pause_resume(self):
self.setup_xdcr()
gen_create1 = self._get_generator('nwOne', self._value_size,
self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create1)
gen_create2 = self._get_generator('nwTwo', self._value_size,
self._num_items)
tasks.extend(
self.dest_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create2))
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
self._set_nwusage_limit(self.dest_cluster,
nw_limit * self.num_dest_nodes)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_src_nodes)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_dest_nodes)
def test_nwusage_with_unidirection_in_parallel(self):
self.setup_xdcr()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_src_nodes)
def test_nwusage_with_bidirection_in_parallel(self):
self.setup_xdcr()
gen_create1 = self._get_generator('nwOne', self._value_size,
self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create1)
gen_create2 = self._get_generator('nwTwo', self._value_size,
self._num_items)
tasks.extend(
self.dest_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create2))
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
self._set_nwusage_limit(self.dest_cluster,
nw_limit * self.num_dest_nodes)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_src_nodes)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_dest_nodes)
def test_nwusage_with_rebalance_in(self):
self.setup_xdcr()
self.sleep(60)
no_of_nodes = self.num_src_nodes + 1
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster, nw_limit * no_of_nodes)
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_in()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=no_of_nodes)
def test_nwusage_with_rebalance_out(self):
self.setup_xdcr()
self.sleep(60)
no_of_nodes = self.num_src_nodes - 1
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster, nw_limit * no_of_nodes)
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_out()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=no_of_nodes)
def test_nwusage_reset_to_zero(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
self.sleep(30)
self._set_nwusage_limit(self.src_cluster, 0)
event_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Network limit reset to 0 at {0}".format(event_time))
for task in tasks:
task.result()
self.verify_results()
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=self.num_dest_nodes,
end_time=event_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=0,
no_of_nodes=self.num_src_nodes,
event_time=event_time,
nw_usage="0")
def test_nwusage_with_hard_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
self.src_cluster.pause_all_replications()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
failover_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node failed over at {0}".format(failover_time))
self.sleep(15)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
end_time=failover_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=failover_time,
end_time=node_back_time,
no_of_nodes=self.num_src_nodes - 1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time,
no_of_nodes=self.num_src_nodes)
def test_nwusage_with_hard_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
self.src_cluster.pause_all_replications()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
self.sleep(15)
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
bw_enable_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info(
"Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=bw_enable_time,
end_time=node_back_time,
no_of_nodes=self.num_src_nodes - 1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
no_of_nodes=self.num_src_nodes,
event_time=node_back_time)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
self.src_cluster.rebalance_in()
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.update_autofailover_settings(enabled=True, timeout=30)
self.src_cluster.pause_all_replications()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(30)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [],
[])
task.result()
failover_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node auto failed over at {0}".format(failover_time))
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(45)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
end_time=failover_time,
no_of_nodes=3)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=failover_time,
end_time=node_back_time,
no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time,
no_of_nodes=3)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.sleep(60)
self._set_doc_size_num()
self.src_cluster.rebalance_in()
self.src_cluster.pause_all_replications()
gen_create = self._get_generator('nwOne', self._value_size,
self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(45)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [],
[])
task.result()
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
nw_limit = self._input.param("nw_limit", self._get_nwusage_limit())
self._set_nwusage_limit(self.src_cluster,
nw_limit * self.num_src_nodes)
bw_enable_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info(
"Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(30)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(
self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=bw_enable_time,
end_time=node_back_time,
no_of_nodes=self.num_src_nodes)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time,
no_of_nodes=self.num_src_nodes + 1)
class Capi(XDCRNewBaseTest, NewUpgradeBaseTest):
def setUp(self):
super(Capi, self).setUp()
self.cluster = Cluster()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.use_hostnames = self._input.param("use_hostnames", False)
self.src_init = self._input.param('src_init', 2)
self.dest_init = self._input.param('dest_init', 1)
self.product = self._input.param('product', 'couchbase-server')
self.initial_version = self._input.param('initial_version', '2.5.1-1083')
self.initial_vbuckets = self._input.param('initial_vbuckets', 1024)
self.init_nodes = self._input.param('init_nodes', True)
self.initial_build_type = self._input.param('initial_build_type', None)
self.upgrade_build_type = self._input.param('upgrade_build_type', self.initial_build_type)
self.master = self.src_master
self.rest = RestConnection(self.src_master)
def tearDown(self):
super(Capi, self).tearDown()
def _start_es_replication(self, bucket='default', xdcr_params={}):
rest_conn = RestConnection(self.src_cluster.get_master_node())
if bucket == 'default':
self.log.info("Creating default bucket")
rest_conn.create_bucket(bucket='default', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='default', authType='none',
saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket == 'sasl':
self.log.info("Creating sasl bucket")
rest_conn.create_bucket(bucket='sasl', ramQuotaMB=100, authType='sasl', saslPassword='******', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='sasl', authType='sasl',
saslPassword='******', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket == 'standard':
self.log.info("Creating standard bucket")
rest_conn.create_bucket(bucket='standard', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=STANDARD_BUCKET_PORT, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=False)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='standard', authType='none',
saslPassword='', replicaNumber=1, proxyPort=STANDARD_BUCKET_PORT, bucketType='membase',
evictionPolicy='valueOnly')
elif bucket== 'lww':
self.log.info("Creating lww bucket")
rest_conn.create_bucket(bucket='lww', ramQuotaMB=100, authType='none', saslPassword='', replicaNumber=1,
proxyPort=11211, bucketType='membase', replica_index=1, threadsNumber=3,
flushEnabled=1, lww=True)
self.src_cluster.add_bucket(ramQuotaMB=100, bucket='lww', authType='none',
saslPassword='', replicaNumber=1, proxyPort=11211, bucketType='membase',
evictionPolicy='valueOnly')
esrest_conn = EsRestConnection(self.dest_cluster.get_master_node())
esrest_conn.create_index(bucket)
rest_conn.add_remote_cluster(remoteIp=self.dest_master.ip, remotePort=9091, username='******',
password='******', name='es')
self.src_cluster.get_remote_clusters().append(XDCRRemoteClusterRef(self.src_cluster, self.dest_cluster,
Utility.get_rc_name(self.src_cluster.get_name(),
self.dest_cluster.get_name())))
repl_id = rest_conn.start_replication(replicationType='continuous', fromBucket=bucket, toCluster='es',
rep_type='capi', toBucket=bucket, xdcr_params=xdcr_params)
return repl_id
def _verify_es_results(self, bucket='default'):
esrest_conn = EsRestConnection(self.dest_master)
es_docs = esrest_conn.all_docs()
self.log.info("Retrieved ES Docs")
rest_conn = RestConnection(self.src_master)
memcached_conn = VBucketAwareMemcached(rest_conn, bucket)
self.log.info("Comparing CB and ES data")
for doc in es_docs:
es_data = doc['doc']
mc_active = memcached_conn.memcached(str(es_data['_id']))
cb_flags, cb_cas, cb_data = mc_active.get(str(es_data['_id']))
self.assertDictEqual(es_data, json.loads(cb_data), "Data mismatch found - es data: {0} cb data: {1}".
format(str(es_data), str(cb_data)))
self.log.info("Data verified")
def test_crud_ops_from_cb_to_es(self):
bucket = self._input.param("bucket", 'default')
repl_id = self._start_es_replication(bucket=bucket)
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results(bucket=bucket)
def test_incr_crud_ops_from_cb_to_es(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.async_perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_pause_resume(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.async_load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
self.sleep(30)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_checkpointing(self):
repl_id = self._start_es_replication(xdcr_params={"checkpointInterval":"60"})
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.sleep(120)
vb0_node = None
nodes = self.src_cluster.get_nodes()
ip = VBucketAwareMemcached(rest_conn,'default').vBucketMap[0].split(':')[0]
for node in nodes:
if ip == node.ip:
vb0_node = node
if not vb0_node:
raise XDCRCheckpointException("Error determining the node containing active vb0")
vb0_conn = RestConnection(vb0_node)
try:
checkpoint_record = vb0_conn.get_recent_xdcr_vb_ckpt(repl_id)
self.log.info("Checkpoint record : {0}".format(checkpoint_record))
except Exception as e:
raise XDCRCheckpointException("Error retrieving last checkpoint document - {0}".format(e))
self._verify_es_results()
def test_capi_with_optimistic_replication(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.set_xdcr_param('default', 'default', 'optimisticReplicationThreshold', self._optimistic_threshold)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_filter(self):
repl_id = self._start_es_replication(xdcr_params={'filterExpression':'es-5*'})
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_advanced_settings(self):
batch_count = self._input.param("batch_count", 10)
batch_size = self._input.param("batch_size", 2048)
source_nozzle = self._input.param("source_nozzle", 2)
target_nozzle = self._input.param("target_nozzle", 2)
enable_firewall = self._input.param("enable_firewall", False)
capi_data_chan_size_multi = self._input.param("capi_data_chan_size_multi", None)
if capi_data_chan_size_multi:
shell = RemoteMachineShellConnection(self.src_master)
command = "curl -X POST -u Administrator:password http://127.0.0.1:9998/xdcr/internalSettings " + \
"-d CapiDataChanSizeMultiplier=" + str(capi_data_chan_size_multi)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.set_xdcr_param('default', 'default', 'workerBatchSize', batch_count)
rest_conn.set_xdcr_param('default', 'default', 'docBatchSizeKb', batch_size)
rest_conn.set_xdcr_param('default', 'default', 'sourceNozzlePerNode', source_nozzle)
rest_conn.set_xdcr_param('default', 'default', 'targetNozzlePerNode', target_nozzle)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
if enable_firewall:
NodeHelper.enable_firewall(self.dest_cluster.get_master_node())
self.sleep(120)
NodeHelper.disable_firewall(self.dest_cluster.get_master_node())
self._verify_es_results()
def test_capi_with_rebalance_in(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.src_cluster.rebalance_in()
self._wait_for_es_replication_to_catchup(timeout=900)
self._verify_es_results()
def test_capi_with_rebalance_out(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.src_cluster.rebalance_out()
self._wait_for_es_replication_to_catchup(timeout=900)
self._verify_es_results()
def test_capi_with_swap_rebalance(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.src_cluster.swap_rebalance()
self._wait_for_es_replication_to_catchup(timeout=600)
self._verify_es_results()
def test_capi_with_failover(self):
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
graceful = self._input.param("graceful", False)
self.recoveryType = self._input.param("recoveryType", None)
self.src_cluster.failover(graceful=graceful)
self.sleep(30)
if self.recoveryType:
server_nodes = rest_conn.node_statuses()
for node in server_nodes:
if node.ip == self._input.servers[1].ip:
rest_conn.set_recovery_type(otpNode=node.id, recoveryType=self.recoveryType)
self.sleep(30)
rest_conn.add_back_node(otpNode=node.id)
rebalance = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
rebalance.result()
self._verify_es_results()
def test_capi_with_malformed_http_resp(self):
repl_id = self._start_es_replication(xdcr_params={'workerBatchSize':'2000',
'docBatchSizeKb':'8096',
'targetNozzlePerNode':'64'})
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
goxdcr_log = NodeHelper.get_goxdcr_log_dir(self.src_master)\
+ '/goxdcr.log*'
for node in self.src_cluster.get_nodes():
count = NodeHelper.check_goxdcr_log(
node,
"malformed HTTP response",
goxdcr_log)
self.assertEqual(count, 0, "malformed HTTP response error message found in " + str(node.ip))
self.log.info("malformed HTTP response error message not found in " + str(node.ip))
self._verify_es_results()
def test_capi_with_offline_upgrade(self):
self._install(self._input.servers[:self.src_init + self.dest_init])
upgrade_nodes = self.src_cluster.get_nodes()
upgrade_version = self._input.param("upgrade_version", "5.0.0-1797")
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
self._install(servers=upgrade_nodes, version=upgrade_version)
self.log.info("######### Upgrade of CB cluster completed ##########")
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value"}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_online_upgrade(self):
self._install(self._input.servers[:self.src_init + self.dest_init])
upgrade_version = self._input.param("upgrade_version", "5.0.0-1797")
upgrade_nodes = self.src_cluster.get_nodes()
extra_nodes = self._input.servers[self.src_init + self.dest_init:]
repl_id = self._start_es_replication()
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
RestConnection(upgrade_nodes[0]).get_nodes_versions()
added_versions = RestConnection(extra_nodes[0]).get_nodes_versions()
self.cluster.rebalance(upgrade_nodes + extra_nodes, extra_nodes, [])
self.log.info("Rebalance in all {0} nodes completed".format(added_versions[0]))
RestConnection(upgrade_nodes[0]).get_nodes_versions()
self.sleep(15)
status, content = ClusterOperationHelper.find_orchestrator(upgrade_nodes[0])
self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\
format(status, content))
self.log.info("after rebalance in the master is {0}".format(content))
find_master = False
for new_server in extra_nodes:
if content.find(new_server.ip) >= 0:
find_master = True
self.log.info("{0} Node {1} becomes the master".format(added_versions[0], new_server.ip))
break
if not find_master:
raise Exception("After rebalance in {0} Nodes, one of them doesn't become the master".
format(added_versions[0]))
self.log.info("Rebalancing out all old version nodes")
self.cluster.rebalance(upgrade_nodes + extra_nodes, [], upgrade_nodes)
self.src_master = self._input.servers[self.src_init + self.dest_init]
self._install(self.src_cluster.get_nodes(), version=upgrade_version)
upgrade_nodes = self._input.servers[self.src_init + self.dest_init:]
extra_nodes = self.src_cluster.get_nodes()
RestConnection(upgrade_nodes[0]).get_nodes_versions()
added_versions = RestConnection(extra_nodes[0]).get_nodes_versions()
self.cluster.rebalance(upgrade_nodes + extra_nodes, extra_nodes, [])
self.log.info("Rebalance in all {0} nodes completed".format(added_versions[0]))
RestConnection(upgrade_nodes[0]).get_nodes_versions()
self.sleep(15)
status, content = ClusterOperationHelper.find_orchestrator(upgrade_nodes[0])
self.assertTrue(status, msg="Unable to find orchestrator: {0}:{1}".\
format(status, content))
self.log.info("after rebalance in the master is {0}".format(content))
self.log.info("Rebalancing out all old version nodes")
self.cluster.rebalance(upgrade_nodes + extra_nodes, [], upgrade_nodes)
self.src_master = self._input.servers[0]
self.log.info("######### Upgrade of CB cluster completed ##########")
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value"}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
self.perform_update_delete()
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self._wait_for_es_replication_to_catchup()
self._verify_es_results()
def test_capi_with_cb_stop_and_start(self):
bucket = self._input.param("bucket", 'default')
repl_id = self._start_es_replication(bucket=bucket)
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.async_perform_update_delete()
conn = RemoteMachineShellConnection(self.src_master)
conn.stop_couchbase()
conn.start_couchbase()
self.sleep(30)
self._wait_for_es_replication_to_catchup()
self._verify_es_results(bucket=bucket)
def test_capi_with_erlang_crash(self):
bucket = self._input.param("bucket", 'default')
repl_id = self._start_es_replication(bucket=bucket)
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.async_perform_update_delete()
conn = RemoteMachineShellConnection(self.src_master)
conn.kill_erlang()
conn.start_couchbase()
self.sleep(30)
self._wait_for_es_replication_to_catchup()
self._verify_es_results(bucket=bucket)
def test_capi_with_memcached_crash(self):
bucket = self._input.param("bucket", 'default')
repl_id = self._start_es_replication(bucket=bucket)
rest_conn = RestConnection(self.src_master)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'true')
gen = DocumentGenerator('es', '{{"key":"value","mutated":0}}', xrange(100), start=0, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(gen)
rest_conn.pause_resume_repl_by_id(repl_id, REPL_PARAM.PAUSE_REQUESTED, 'false')
self.async_perform_update_delete()
conn = RemoteMachineShellConnection(self.src_master)
conn.pause_memcached()
conn.unpause_memcached()
self.sleep(30)
self._wait_for_es_replication_to_catchup()
self._verify_es_results(bucket=bucket)
class compression(XDCRNewBaseTest):
def setUp(self):
super(compression, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.chain_length = self._input.param("chain_length", 2)
self.topology = self._input.param("ctopology", "chain")
if self.chain_length > 2:
self.c3_cluster = self.get_cb_cluster_by_name('C3')
self.c3_master = self.c3_cluster.get_master_node()
self.cluster = Cluster()
def tearDown(self):
super(compression, self).tearDown()
def _set_compression_type(self,
cluster,
bucket_name,
compression_type="None"):
repls = cluster.get_remote_clusters()[0].get_replications()
for repl in repls:
if bucket_name in str(repl):
repl_id = repl.get_repl_id()
shell = RemoteMachineShellConnection(cluster.get_master_node())
repl_id = str(repl_id).replace('/', '%2F')
base_url = "http://" + cluster.get_master_node(
).ip + ":8091/settings/replications/" + repl_id
command = "curl -X POST -u Administrator:password " + base_url + " -d compressionType=" + str(
compression_type)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
return output, error
def _verify_compression(self,
cluster,
compr_bucket_name="",
uncompr_bucket_name="",
compression_type="None",
repl_time=0):
repls = cluster.get_remote_clusters()[0].get_replications()
for repl in repls:
if compr_bucket_name in str(repl):
compr_repl_id = repl.get_repl_id()
elif uncompr_bucket_name in str(repl):
uncompr_repl_id = repl.get_repl_id()
compr_repl_id = str(compr_repl_id).replace('/', '%2F')
uncompr_repl_id = str(uncompr_repl_id).replace('/', '%2F')
base_url = "http://" + cluster.get_master_node(
).ip + ":8091/settings/replications/" + compr_repl_id
shell = RemoteMachineShellConnection(cluster.get_master_node())
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
self.assertTrue(
'"compressionType":"Snappy"' in output[0],
"Compression Type for replication " + compr_repl_id +
" is not Snappy")
self.log.info("Compression Type for replication " + compr_repl_id +
" is Snappy")
base_url = "http://" + cluster.get_master_node().ip + ":8091/pools/default/buckets/" + compr_bucket_name + \
"/stats/replications%2F" + compr_repl_id + "%2Fdata_replicated?haveTStamp=" + str(repl_time)
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
output = json.loads(output[0])
compressed_data_replicated = 0
for node in cluster.get_nodes():
items = output["nodeStats"]["{0}:8091".format(node.ip)]
for item in items:
compressed_data_replicated += item
self.log.info("Compressed data for replication {0} is {1}".format(
compr_repl_id, compressed_data_replicated))
base_url = "http://" + cluster.get_master_node().ip + ":8091/pools/default/buckets/" + uncompr_bucket_name + \
"/stats/replications%2F" + uncompr_repl_id + "%2Fdata_replicated?haveTStamp=" + str(repl_time)
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
output = json.loads(output[0])
uncompressed_data_replicated = 0
for node in cluster.get_nodes():
items = output["nodeStats"]["{0}:8091".format(node.ip)]
for item in items:
uncompressed_data_replicated += item
self.log.info("Uncompressed data for replication {0} is {1}".format(
uncompr_repl_id, uncompressed_data_replicated))
self.assertTrue(
uncompressed_data_replicated > compressed_data_replicated,
"Compression did not work as expected")
self.log.info("Compression worked as expected")
shell.disconnect()
def test_compression_with_unixdcr_incr_load(self):
bucket_prefix = self._input.param("bucket_prefix", "standard_bucket_")
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, bucket_prefix + "1",
compression_type)
self._set_compression_type(self.src_cluster, bucket_prefix + "2")
if self.chain_length > 2 and self.topology == TOPOLOGY.CHAIN:
self._set_compression_type(self.dest_cluster, bucket_prefix + "1",
compression_type)
self._set_compression_type(self.dest_cluster, bucket_prefix + "2")
if self.chain_length > 2 and self.topology == TOPOLOGY.RING:
self._set_compression_type(self.dest_cluster, bucket_prefix + "1",
compression_type)
self._set_compression_type(self.dest_cluster, bucket_prefix + "2")
self._set_compression_type(self.c3_cluster, bucket_prefix + "1",
compression_type)
self._set_compression_type(self.c3_cluster, bucket_prefix + "2")
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
if self.chain_length > 2 and self.topology == TOPOLOGY.CHAIN:
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
if self.chain_length > 2 and self.topology == TOPOLOGY.RING:
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.c3_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_unixdcr_backfill_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_bixdcr_incr_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._set_compression_type(self.dest_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.dest_cluster, "standard_bucket_2")
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-',
'comprTwo-',
self._value_size,
end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_bixdcr_backfill_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._set_compression_type(self.dest_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.dest_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-',
'comprTwo-',
self._value_size,
end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_pause_resume(self):
repeat = self._input.param("repeat", 5)
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
for i in range(0, repeat):
self.src_cluster.pause_all_replications()
self.sleep(30)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_optimistic_threshold_change(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1',
'optimisticReplicationThreshold',
self._optimistic_threshold)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2',
'optimisticReplicationThreshold',
self._optimistic_threshold)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_advanced_settings(self):
batch_count = self._input.param("batch_count", 10)
batch_size = self._input.param("batch_size", 2048)
source_nozzle = self._input.param("source_nozzle", 2)
target_nozzle = self._input.param("target_nozzle", 2)
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1',
'workerBatchSize', batch_count)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1',
'docBatchSizeKb', batch_size)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1',
'sourceNozzlePerNode', source_nozzle)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1',
'targetNozzlePerNode', target_nozzle)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2',
'workerBatchSize', batch_count)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2',
'docBatchSizeKb', batch_size)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2',
'sourceNozzlePerNode', source_nozzle)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2',
'targetNozzlePerNode', target_nozzle)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_capi(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
output, error = self._set_compression_type(self.src_cluster, "default",
compression_type)
self.assertTrue(
"The value can not be specified for CAPI replication" in output[0],
"Compression enabled for CAPI")
self.log.info("Compression not enabled for CAPI as expected")
def test_compression_with_rebalance_in(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_in()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_out()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_swap_rebalance(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.swap_rebalance()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_failover(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
src_conn = RestConnection(self.src_cluster.get_master_node())
graceful = self._input.param("graceful", False)
self.recoveryType = self._input.param("recoveryType", None)
self.src_cluster.failover(graceful=graceful)
self.sleep(30)
if self.recoveryType:
server_nodes = src_conn.node_statuses()
for node in server_nodes:
if node.ip == self._input.servers[1].ip:
src_conn.set_recovery_type(otpNode=node.id,
recoveryType=self.recoveryType)
self.sleep(30)
src_conn.add_back_node(otpNode=node.id)
rebalance = self.cluster.async_rebalance(
self.src_cluster.get_nodes(), [], [])
rebalance.result()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_replication_delete_and_create(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
rest_conn = RestConnection(self.src_master)
rest_conn.remove_all_replications()
rest_conn.remove_all_remote_clusters()
self.src_cluster.get_remote_clusters()[0].clear_all_replications()
self.src_cluster.clear_all_remote_clusters()
self.setup_xdcr()
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1",
compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_bixdcr_and_compression_one_way(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default",
compression_type)
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-',
'comprTwo-',
self._value_size,
end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_enabling_later(self):
self.setup_xdcr()
self.sleep(60)
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
self.sleep(10)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default",
compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_disabling_later(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default",
compression_type)
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
self.sleep(10)
self._set_compression_type(self.src_cluster, "default", "None")
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out_target_and_disabling(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default",
compression_type)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.dest_cluster.rebalance_out()
self._set_compression_type(self.src_cluster, "default", "None")
self.sleep(5)
self._set_compression_type(self.src_cluster, "default",
compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out_src_and_disabling(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default",
compression_type)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-',
'comprOne-',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_out()
self._set_compression_type(self.src_cluster, "default", "None")
self.sleep(5)
self._set_compression_type(self.src_cluster, "default",
compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
class compression(XDCRNewBaseTest):
def setUp(self):
super(compression, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.chain_length = self._input.param("chain_length", 2)
self.topology = self._input.param("ctopology", "chain")
if self.chain_length > 2:
self.c3_cluster = self.get_cb_cluster_by_name('C3')
self.c3_master = self.c3_cluster.get_master_node()
self.cluster = Cluster()
def tearDown(self):
super(compression, self).tearDown()
def _set_compression_type(self, cluster, bucket_name, compression_type="None"):
repls = cluster.get_remote_clusters()[0].get_replications()
for repl in repls:
if bucket_name in str(repl):
repl_id = repl.get_repl_id()
shell = RemoteMachineShellConnection(cluster.get_master_node())
repl_id = str(repl_id).replace('/','%2F')
base_url = "http://" + cluster.get_master_node().ip + ":8091/settings/replications/" + repl_id
command = "curl -X POST -u Administrator:password " + base_url + " -d compressionType=" + str(compression_type)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
shell.disconnect()
return output, error
def _verify_compression(self, cluster, compr_bucket_name="", uncompr_bucket_name="",
compression_type="None", repl_time=0):
repls = cluster.get_remote_clusters()[0].get_replications()
for repl in repls:
if compr_bucket_name in str(repl):
compr_repl_id = repl.get_repl_id()
elif uncompr_bucket_name in str(repl):
uncompr_repl_id = repl.get_repl_id()
compr_repl_id = str(compr_repl_id).replace('/', '%2F')
uncompr_repl_id = str(uncompr_repl_id).replace('/', '%2F')
base_url = "http://" + cluster.get_master_node().ip + ":8091/settings/replications/" + compr_repl_id
shell = RemoteMachineShellConnection(cluster.get_master_node())
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
self.assertTrue('"compressionType":"Snappy"' in output[0],
"Compression Type for replication " + compr_repl_id + " is not Snappy")
self.log.info("Compression Type for replication " + compr_repl_id + " is Snappy")
base_url = "http://" + cluster.get_master_node().ip + ":8091/pools/default/buckets/" + compr_bucket_name + \
"/stats/replications%2F" + compr_repl_id + "%2Fdata_replicated?haveTStamp=" + str(repl_time)
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
output = json.loads(output[0])
compressed_data_replicated = 0
for node in cluster.get_nodes():
items = output["nodeStats"]["{0}:8091".format(node.ip)]
for item in items:
compressed_data_replicated += item
self.log.info("Compressed data for replication {0} is {1}".format(compr_repl_id, compressed_data_replicated))
base_url = "http://" + cluster.get_master_node().ip + ":8091/pools/default/buckets/" + uncompr_bucket_name + \
"/stats/replications%2F" + uncompr_repl_id + "%2Fdata_replicated?haveTStamp=" + str(repl_time)
command = "curl -u Administrator:password " + base_url
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
output = json.loads(output[0])
uncompressed_data_replicated = 0
for node in cluster.get_nodes():
items = output["nodeStats"]["{0}:8091".format(node.ip)]
for item in items:
uncompressed_data_replicated += item
self.log.info("Uncompressed data for replication {0} is {1}".format(uncompr_repl_id, uncompressed_data_replicated))
self.assertTrue(uncompressed_data_replicated > compressed_data_replicated,
"Compression did not work as expected")
self.log.info("Compression worked as expected")
shell.disconnect()
def test_compression_with_unixdcr_incr_load(self):
bucket_prefix = self._input.param("bucket_prefix", "standard_bucket_")
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, bucket_prefix + "1", compression_type)
self._set_compression_type(self.src_cluster, bucket_prefix + "2")
if self.chain_length > 2 and self.topology == TOPOLOGY.CHAIN:
self._set_compression_type(self.dest_cluster, bucket_prefix + "1", compression_type)
self._set_compression_type(self.dest_cluster, bucket_prefix + "2")
if self.chain_length > 2 and self.topology == TOPOLOGY.RING:
self._set_compression_type(self.dest_cluster, bucket_prefix + "1", compression_type)
self._set_compression_type(self.dest_cluster, bucket_prefix + "2")
self._set_compression_type(self.c3_cluster, bucket_prefix + "1", compression_type)
self._set_compression_type(self.c3_cluster, bucket_prefix + "2")
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
if self.chain_length > 2 and self.topology == TOPOLOGY.CHAIN:
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
if self.chain_length > 2 and self.topology == TOPOLOGY.RING:
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.c3_cluster,
compr_bucket_name=bucket_prefix + "1",
uncompr_bucket_name=bucket_prefix + "2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_unixdcr_backfill_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_bixdcr_incr_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._set_compression_type(self.dest_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.dest_cluster, "standard_bucket_2")
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-', 'comprTwo-', self._value_size, end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_bixdcr_backfill_load(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._set_compression_type(self.dest_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.dest_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-', 'comprTwo-', self._value_size, end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self._verify_compression(cluster=self.dest_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_pause_resume(self):
repeat = self._input.param("repeat", 5)
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
for i in range(0, repeat):
self.src_cluster.pause_all_replications()
self.sleep(30)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_optimistic_threshold_change(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1', 'optimisticReplicationThreshold',
self._optimistic_threshold)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2', 'optimisticReplicationThreshold',
self._optimistic_threshold)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_advanced_settings(self):
batch_count = self._input.param("batch_count", 10)
batch_size = self._input.param("batch_size", 2048)
source_nozzle = self._input.param("source_nozzle", 2)
target_nozzle = self._input.param("target_nozzle", 2)
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1', 'workerBatchSize', batch_count)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1', 'docBatchSizeKb', batch_size)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1', 'sourceNozzlePerNode', source_nozzle)
src_conn.set_xdcr_param('standard_bucket_1', 'standard_bucket_1', 'targetNozzlePerNode', target_nozzle)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2', 'workerBatchSize', batch_count)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2', 'docBatchSizeKb', batch_size)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2', 'sourceNozzlePerNode', source_nozzle)
src_conn.set_xdcr_param('standard_bucket_2', 'standard_bucket_2', 'targetNozzlePerNode', target_nozzle)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_capi(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
output, error = self._set_compression_type(self.src_cluster, "default", compression_type)
self.assertTrue("The value can not be specified for CAPI replication" in output[0], "Compression enabled for CAPI")
self.log.info("Compression not enabled for CAPI as expected")
def test_compression_with_rebalance_in(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_in()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_out()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_swap_rebalance(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.swap_rebalance()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_failover(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
src_conn = RestConnection(self.src_cluster.get_master_node())
graceful = self._input.param("graceful", False)
self.recoveryType = self._input.param("recoveryType", None)
self.src_cluster.failover(graceful=graceful)
self.sleep(30)
if self.recoveryType:
server_nodes = src_conn.node_statuses()
for node in server_nodes:
if node.ip == self._input.servers[1].ip:
src_conn.set_recovery_type(otpNode=node.id, recoveryType=self.recoveryType)
self.sleep(30)
src_conn.add_back_node(otpNode=node.id)
rebalance = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
rebalance.result()
self._wait_for_replication_to_catchup()
self._verify_compression(cluster=self.src_cluster,
compr_bucket_name="standard_bucket_1",
uncompr_bucket_name="standard_bucket_2",
compression_type=compression_type,
repl_time=repl_time)
self.verify_results()
def test_compression_with_replication_delete_and_create(self):
self.setup_xdcr()
repl_time = int(time.time())
self.sleep(60)
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
rest_conn = RestConnection(self.src_master)
rest_conn.remove_all_replications()
rest_conn.remove_all_remote_clusters()
self.src_cluster.get_remote_clusters()[0].clear_all_replications()
self.src_cluster.clear_all_remote_clusters()
self.setup_xdcr()
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "standard_bucket_1", compression_type)
self._set_compression_type(self.src_cluster, "standard_bucket_2")
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_bixdcr_and_compression_one_way(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default", compression_type)
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
gen_create = BlobGenerator('comprTwo-', 'comprTwo-', self._value_size, end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_enabling_later(self):
self.setup_xdcr()
self.sleep(60)
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
self.sleep(10)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default", compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_disabling_later(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default", compression_type)
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.async_perform_update_delete()
self.sleep(10)
self._set_compression_type(self.src_cluster, "default", "None")
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out_target_and_disabling(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default", compression_type)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.dest_cluster.rebalance_out()
self._set_compression_type(self.src_cluster, "default", "None")
self.sleep(5)
self._set_compression_type(self.src_cluster, "default", compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
def test_compression_with_rebalance_out_src_and_disabling(self):
self.setup_xdcr()
self.sleep(60)
compression_type = self._input.param("compression_type", "Snappy")
self._set_compression_type(self.src_cluster, "default", compression_type)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('comprOne-', 'comprOne-', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.async_perform_update_delete()
self.src_cluster.rebalance_out()
self._set_compression_type(self.src_cluster, "default", "None")
self.sleep(5)
self._set_compression_type(self.src_cluster, "default", compression_type)
self._wait_for_replication_to_catchup()
self.verify_results()
class SpatialQueryTests(unittest.TestCase):
def setUp(self):
self.log = logger.Logger.get_logger()
self.helper = SpatialHelper(self, "default")
self.helper.setup_cluster()
self.cluster = Cluster()
self.servers = self.helper.servers
def tearDown(self):
self.helper.cleanup_cluster()
def test_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init(data_set)
def test_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init(data_set)
def test_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init(data_set)
def test_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init(data_set)
def test_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init(data_set)
def test_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init(data_set)
def test_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init(data_set)
def test_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init(data_set)
## Rebalance In
def test_rebalance_in_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._rebalance_cluster(data_set)
#Rebalance Out
def test_rebalance_out_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._rebalance_cluster(data_set)
# Warmup Tests
def test_warmup_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with skip and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init_integration(data_set)
# Reboot Tests
def test_reboot_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init_integration(data_set)
# Failover Tests
def test_failover_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Failover and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._failover_cluster(data_set)
###
# load the data defined for this dataset.
# create views and query the data as it loads.
# verification is optional, and best practice is to
# set to False if you plan on running _query_all_views()
# later in the test case
###
def _query_test_init(self, data_set, verify_results = True):
views = data_set.views
# start loading data
t = Thread(target=data_set.load,
name="load_data_set",
args=())
t.start()
# run queries while loading data
while(t.is_alive()):
self._query_all_views(views, False)
time.sleep(5)
t.join()
# results will be verified if verify_results set
if verify_results:
self._query_all_views(views, verify_results)
else:
self._check_view_intergrity(views)
def _query_test_init_integration(self, data_set, verify_results = True):
views = data_set.views
inserted_keys = data_set.load()
target_fn = ()
if self.helper.num_nodes_reboot >= 1:
target_fn = self._reboot_cluster(data_set)
elif self.helper.num_nodes_warmup >= 1:
target_fn = self._warmup_cluster(data_set)
elif self.helper.num_nodes_to_add >= 1 or self.helper.num_nodes_to_remove >= 1:
target_fn = self._rebalance_cluster(data_set)
t = Thread(target=self._query_all_views(views, False))
t.start()
# run queries while loading data
while t.is_alive():
self._rebalance_cluster(data_set)
time.sleep(5)
t.join()
# results will be verified if verify_results set
if verify_results:
self._query_all_views(views, verify_results)
else:
self._check_view_intergrity(views)
##
# run all queries for all views in parallel
##
def _query_all_views(self, views, verify_results = True):
query_threads = []
for view in views:
t = RunQueriesThread(view, verify_results)
query_threads.append(t)
t.start()
[t.join() for t in query_threads]
self._check_view_intergrity(query_threads)
##
# If an error occured loading or querying data for a view
# it is queued and checked here. Fail on the first one that
# occurs.
##
def _check_view_intergrity(self, thread_results):
for result in thread_results:
if result.test_results.errors:
self.fail(result.test_results.errors[0][1])
if result.test_results.failures:
self.fail(result.test_results.failures[0][1])
###
# Rebalance
###
def _rebalance_cluster(self, data_set):
if self.helper.num_nodes_to_add >= 1:
rebalance = self.cluster.async_rebalance(self.servers[:1],
self.servers[1:self.helper.num_nodes_to_add + 1],
[])
self._query_test_init(data_set)
rebalance.result()
elif self.helper.num_nodes_to_remove >= 1:
rebalance = self.cluster.async_rebalance(self.servers[:1],[],
self.servers[1:self.helper.num_nodes_to_add + 1])
self._query_test_init(data_set)
rebalance.result()
def _failover_cluster(self, data_set):
failover_nodes = self.servers[1 : self.helper.failover_factor + 1]
try:
# failover and verify loaded data
#self.cluster.failover(self.servers, failover_nodes)
self.cluster.failover(self.servers, self.servers[1:2])
self.log.info("120 seconds sleep after failover before invoking rebalance...")
time.sleep(120)
rebalance = self.cluster.async_rebalance(self.servers,
[], self.servers[1:2])
self._query_test_init(data_set)
msg = "rebalance failed while removing failover nodes {0}".format(failover_nodes)
self.assertTrue(rebalance.result(), msg=msg)
#verify queries after failover
self._query_test_init(data_set)
finally:
self.log.info("Completed the failover testing for spatial querying")
###
# Warmup
###
def _warmup_cluster(self, data_set):
for server in self.servers[0:self.helper.num_nodes_warmup]:
remote = RemoteMachineShellConnection(server)
remote.stop_server()
remote.start_server()
remote.disconnect()
self.log.info("Node {0} should be warming up ".format(server.ip))
time.sleep(120)
self._query_test_init(data_set)
# REBOOT
def _reboot_cluster(self, data_set):
try:
for server in self.servers[0:self.helper.num_nodes_reboot]:
shell = RemoteMachineShellConnection(server)
if shell.extract_remote_info().type.lower() == 'windows':
o, r = shell.execute_command("shutdown -r -f -t 0")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} is being stopped".format(server.ip))
elif shell.extract_remote_info().type.lower() == 'linux':
o, r = shell.execute_command("reboot")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} is being stopped".format(server.ip))
time.sleep(120)
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
o, r = shell.execute_command(command)
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} backup".format(server.ip))
finally:
self.log.info("Warming-up server ..".format(server.ip))
time.sleep(100)
class SpatialQueryTests(unittest.TestCase):
def setUp(self):
self.log = logger.Logger.get_logger()
self.helper = SpatialHelper(self, "default")
self.helper.setup_cluster()
self.cluster = Cluster()
self.servers = self.helper.servers
def tearDown(self):
self.helper.cleanup_cluster()
def test_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init(data_set)
def test_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init(data_set)
def test_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init(data_set)
def test_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init(data_set)
def test_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init(data_set)
def test_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Make skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init(data_set)
def test_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init(data_set)
def test_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Make range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init(data_set)
## Rebalance In
def test_rebalance_in_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance In and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_in_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance In and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._rebalance_cluster(data_set)
#Rebalance Out
def test_rebalance_out_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._rebalance_cluster(data_set)
def test_rebalance_out_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._rebalance_cluster(data_set)
# Warmup Tests
def test_warmup_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with skip and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init_integration(data_set)
def test_warmup_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Warmup with range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_warmup_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Warmup with range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init_integration(data_set)
# Reboot Tests
def test_reboot_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Reboot with skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Reboot with bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._query_test_init_integration(data_set)
def test_reboot_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Reboot with limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Reboot with skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Reboot with range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._query_test_init_integration(data_set)
def test_reboot_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Reboot with range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._query_test_init_integration(data_set)
# Failover Tests
def test_failover_simple_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Failover and limit queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and skip (and limit) queries on a "
"simple dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_bbox_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and bounding box queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_bbox_queries()
self._failover_cluster(data_set)
def test_failover_simple_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and range queries on a simple "
"dataset with {0} docs".format(num_docs))
data_set = SimpleDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and limit queries on a multidimensional "
"dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_limit_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_skip_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and skip (and limit) queries on a "
"multidimensional dataset with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_skip_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_range_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info("description : Rebalance Out and range queries on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_queries()
self._failover_cluster(data_set)
def test_failover_multidim_dataset_range_and_limit_queries(self):
num_docs = self.helper.input.param("num-docs")
self.log.info(
"description : Rebalance Out and range queries with limits on a "
"multidimensional with {0} docs".format(num_docs))
data_set = MultidimDataSet(self.helper, num_docs)
data_set.add_range_and_limit_queries()
self._failover_cluster(data_set)
###
# load the data defined for this dataset.
# create views and query the data as it loads.
# verification is optional, and best practice is to
# set to False if you plan on running _query_all_views()
# later in the test case
###
def _query_test_init(self, data_set, verify_results=True):
views = data_set.views
# start loading data
t = Thread(target=data_set.load, name="load_data_set", args=())
t.start()
# run queries while loading data
while (t.is_alive()):
self._query_all_views(views, False)
time.sleep(5)
t.join()
# results will be verified if verify_results set
if verify_results:
self._query_all_views(views, verify_results)
else:
self._check_view_intergrity(views)
def _query_test_init_integration(self, data_set, verify_results=True):
views = data_set.views
inserted_keys = data_set.load()
target_fn = ()
if self.helper.num_nodes_reboot >= 1:
target_fn = self._reboot_cluster(data_set)
elif self.helper.num_nodes_warmup >= 1:
target_fn = self._warmup_cluster(data_set)
elif self.helper.num_nodes_to_add >= 1 or self.helper.num_nodes_to_remove >= 1:
target_fn = self._rebalance_cluster(data_set)
t = Thread(target=self._query_all_views(views, False))
t.start()
# run queries while loading data
while t.is_alive():
self._rebalance_cluster(data_set)
time.sleep(5)
t.join()
# results will be verified if verify_results set
if verify_results:
self._query_all_views(views, verify_results)
else:
self._check_view_intergrity(views)
##
# run all queries for all views in parallel
##
def _query_all_views(self, views, verify_results=True):
query_threads = []
for view in views:
t = RunQueriesThread(view, verify_results)
query_threads.append(t)
t.start()
[t.join() for t in query_threads]
self._check_view_intergrity(query_threads)
##
# If an error occured loading or querying data for a view
# it is queued and checked here. Fail on the first one that
# occurs.
##
def _check_view_intergrity(self, thread_results):
for result in thread_results:
if result.test_results.errors:
self.fail(result.test_results.errors[0][1])
if result.test_results.failures:
self.fail(result.test_results.failures[0][1])
###
# Rebalance
###
def _rebalance_cluster(self, data_set):
if self.helper.num_nodes_to_add >= 1:
rebalance = self.cluster.async_rebalance(
self.servers[:1],
self.servers[1:self.helper.num_nodes_to_add + 1], [])
self._query_test_init(data_set)
rebalance.result()
elif self.helper.num_nodes_to_remove >= 1:
rebalance = self.cluster.async_rebalance(
self.servers[:1], [],
self.servers[1:self.helper.num_nodes_to_add + 1])
self._query_test_init(data_set)
rebalance.result()
def _failover_cluster(self, data_set):
failover_nodes = self.servers[1:self.helper.failover_factor + 1]
try:
# failover and verify loaded data
#self.cluster.failover(self.servers, failover_nodes)
self.cluster.failover(self.servers, self.servers[1:2])
self.log.info(
"120 seconds sleep after failover before invoking rebalance..."
)
time.sleep(120)
rebalance = self.cluster.async_rebalance(self.servers, [],
self.servers[1:2])
self._query_test_init(data_set)
msg = "rebalance failed while removing failover nodes {0}".format(
failover_nodes)
self.assertTrue(rebalance.result(), msg=msg)
#verify queries after failover
self._query_test_init(data_set)
finally:
self.log.info(
"Completed the failover testing for spatial querying")
###
# Warmup
###
def _warmup_cluster(self, data_set):
for server in self.servers[0:self.helper.num_nodes_warmup]:
remote = RemoteMachineShellConnection(server)
remote.stop_server()
remote.start_server()
remote.disconnect()
self.log.info("Node {0} should be warming up ".format(server.ip))
time.sleep(120)
self._query_test_init(data_set)
# REBOOT
def _reboot_cluster(self, data_set):
try:
for server in self.servers[0:self.helper.num_nodes_reboot]:
shell = RemoteMachineShellConnection(server)
if shell.extract_remote_info().type.lower() == 'windows':
o, r = shell.execute_command("shutdown -r -f -t 0")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} is being stopped".format(
server.ip))
elif shell.extract_remote_info().type.lower() == 'linux':
o, r = shell.execute_command("reboot")
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} is being stopped".format(
server.ip))
time.sleep(120)
shell = RemoteMachineShellConnection(server)
command = "/sbin/iptables -F"
o, r = shell.execute_command(command)
shell.log_command_output(o, r)
shell.disconnect()
self.log.info("Node {0} backup".format(server.ip))
finally:
self.log.info("Warming-up server ..".format(server.ip))
time.sleep(100)
class nwusage(XDCRNewBaseTest):
def setUp(self):
super(nwusage, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.cluster = Cluster()
def tearDown(self):
super(nwusage, self).tearDown()
def _set_nwusage_limit(self, cluster, nw_limit=0):
repl_id = cluster.get_remote_clusters()[0].get_replications()[0].get_repl_id()
shell = RemoteMachineShellConnection(cluster.get_master_node())
repl_id = str(repl_id).replace('/','%2F')
base_url = "http://" + cluster.get_master_node().ip + ":8091/settings/replications/" + repl_id
command = "curl -X POST -u Administrator:password " + base_url + " -d networkUsageLimit=" + str(nw_limit)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
def _extract_timestamp(self, logmsg):
#matches timestamp format : 2018-10-11T00:02:35
timestamp_str = re.search(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}', logmsg)
timestamp = datetime.datetime.strptime(timestamp_str.group(), '%Y-%m-%dT%H:%M:%S')
return timestamp
def _verify_bandwidth_usage(self, node, nw_limit=1, no_of_nodes=2, event_time=None,
nw_usage="[1-9][0-9]*", end_time=None):
goxdcr_log = NodeHelper.get_goxdcr_log_dir(node) + '/goxdcr.log'
nw_max = (nw_limit * 1024 * 1024)/no_of_nodes
if event_time:
time_to_compare = datetime.datetime.strptime(event_time.group(), '%Y-%m-%dT%H:%M:%S')
else:
matches, _ = NodeHelper.check_goxdcr_log(node, "Success adding replication specification",
goxdcr_log, print_matches=True)
#Time when replication was set up
time_to_compare = self._extract_timestamp(matches[-1])
matches, count = NodeHelper.check_goxdcr_log(node, "\\\"bandwidth_usage\\\": " + nw_usage, goxdcr_log, print_matches=True)
if count < 1:
self.fail("Bandwidth usage information not found in logs")
match_count = 0
skip_count = 0
for item in matches:
item_datetime = self._extract_timestamp(item)
#Ignore entries that happened before the replication was set up
if item_datetime < time_to_compare:
skip_count += 1
continue
if end_time:
end_datetime = datetime.datetime.strptime(end_time.group(), '%Y-%m-%dT%H:%M:%S')
if item_datetime > end_datetime:
skip_count += 1
continue
bandwidth_usage = ((item.split('{"bandwidth_usage": ')[1]).split(' ')[0]).rstrip(',')
if int(float(bandwidth_usage)) < nw_max:
match_count += 1
continue
else:
self.fail("Bandwidth usage {0} is higher than Bandwidth limit {1} in {2}".format(bandwidth_usage,nw_max,item))
if match_count + skip_count == count:
self.log.info("{0} stale entries skipped".format(skip_count))
if match_count > 0:
self.log.info("{0} entries checked - Bandwidth usage always lower than Bandwidth limit as expected".
format(match_count))
else:
if self._input.param("replication_type") == "capi":
self.log.info("Bandwidth Throttler not enabled on replication as expected")
else:
self.fail("Bandwidth Throttler not enabled on replication")
def _get_current_time(self, server):
shell = RemoteMachineShellConnection(server)
command = "date +'%Y-%m-%dT%H:%M:%S'"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
curr_time = output[0].strip()
return curr_time
def test_nwusage_with_unidirection(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create2)
self.perform_update_delete()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_unidirection_pause_resume(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self.src_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection_pause_resume(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
tasks.extend(self.dest_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create2))
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_unidirection_in_parallel(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection_in_parallel(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
tasks.extend(self.dest_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create2))
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_rebalance_in(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_in()
self.perform_update_delete()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, no_of_nodes=3)
def test_nwusage_with_rebalance_out(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_out()
self.perform_update_delete()
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, no_of_nodes=1)
def test_nwusage_reset_to_zero(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
self._set_nwusage_limit(self.src_cluster, 0)
event_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Network limit reset to 0 at {0}".format(event_time))
for task in tasks:
task.result()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, end_time=event_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=0, no_of_nodes=2, event_time=event_time, nw_usage="0")
def test_nwusage_with_hard_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup(timeout=60)
self.src_cluster.failover_and_rebalance_nodes()
failover_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node failed over at {0}".format(failover_time))
self._wait_for_replication_to_catchup(timeout=60)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=60)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), end_time=failover_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=failover_time, end_time=node_back_time, no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time)
def test_nwusage_with_hard_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup(timeout=60)
self.src_cluster.failover_and_rebalance_nodes()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
bw_enable_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=bw_enable_time, end_time=node_back_time, no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.update_autofailover_settings(enabled=True, timeout=30)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup(timeout=60)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
task.result()
failover_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node auto failed over at {0}".format(failover_time))
FloatingServers._serverlist.append(self._input.servers[1])
shell.start_couchbase()
shell.disable_firewall()
self.wait_service_started(self._input.servers[1])
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), end_time=failover_time, no_of_nodes=3)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=failover_time, end_time=node_back_time, no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time, no_of_nodes=3)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self._wait_for_replication_to_catchup(timeout=60)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
task.result()
FloatingServers._serverlist.append(self._input.servers[1])
self._wait_for_replication_to_catchup(timeout=60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
bw_enable_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Bandwidth throttler enabled at {0}".format(bw_enable_time))
shell.start_couchbase()
shell.disable_firewall()
self.wait_service_started(self._input.servers[1])
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=bw_enable_time, end_time=node_back_time, no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time, no_of_nodes=3)
class nwusage(XDCRNewBaseTest):
def setUp(self):
super(nwusage, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.cluster = Cluster()
def tearDown(self):
super(nwusage, self).tearDown()
def _verify_bandwidth_usage(self,
node,
nw_limit=1,
no_of_nodes=2,
event_time=None,
nw_usage="[1-9][0-9]*",
end_time=None):
goxdcr_log = NodeHelper.get_goxdcr_log_dir(node) + '/goxdcr.log'
nw_max = (nw_limit * 1024 * 1024) / no_of_nodes
if event_time:
time_to_compare = time.strptime(event_time, '%Y-%m-%dT%H:%M:%S')
else:
matches, _ = NodeHelper.check_goxdcr_log(
node,
"Success adding replication specification",
goxdcr_log,
print_matches=True)
time_to_compare_str = matches[-1].split(' ')[0].split('.')[0]
time_to_compare = time.strptime(time_to_compare_str,
'%Y-%m-%dT%H:%M:%S')
matches, count = NodeHelper.check_goxdcr_log(
node,
"bandwidth_limit=" + str(nw_max) + ", bandwidth_usage=" + nw_usage,
goxdcr_log,
print_matches=True)
match_count = 0
skip_count = 0
for item in matches:
items = item.split(' ')
item_time = items[0].split('.')[0]
item_datetime = time.strptime(item_time, '%Y-%m-%dT%H:%M:%S')
if item_datetime < time_to_compare:
skip_count += 1
continue
if end_time:
end_datetime = time.strptime(end_time, '%Y-%m-%dT%H:%M:%S')
if item_datetime > end_datetime:
skip_count += 1
continue
bandwidth_usage = items[-1].split('=')[-1]
if int(bandwidth_usage) <= nw_max:
match_count += 1
continue
else:
self.fail("Bandwidth usage higher than Bandwidth limit in {0}".
format(item))
if match_count + skip_count == count:
self.log.info("{0} stale entries skipped".format(skip_count))
if match_count > 0:
self.log.info(
"{0} entries checked - Bandwidth usage always lower than Bandwidth limit as expected"
.format(match_count))
else:
if self._input.param("replication_type") == "capi":
self.log.info(
"Bandwidth Throttler not enabled on replication as expected"
)
else:
self.fail("Bandwidth Throttler not enabled on replication")
def test_nwusage_with_unidirection(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_bidirection(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.dest_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
gen_create1 = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo',
'nwTwo',
self._value_size,
end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create2)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_unidirection_pause_resume(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
self.src_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.src_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_bidirection_pause_resume(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo',
'nwTwo',
self._value_size,
end=self._num_items)
tasks.extend(
self.dest_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create2))
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.dest_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_unidirection_in_parallel(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_bidirection_in_parallel(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo',
'nwTwo',
self._value_size,
end=self._num_items)
tasks.extend(
self.dest_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create2))
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.dest_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(),
nw_limit=nw_limit)
def test_nwusage_with_rebalance_in(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_in()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=3)
def test_nwusage_with_rebalance_out(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_out()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
no_of_nodes=1)
def test_nwusage_reset_to_zero(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(
kv_gen=gen_create)
self.sleep(30)
self.src_cluster.set_xdcr_param("networkUsageLimit", 0)
event_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Network limit reset to 0 at {0}".format(event_time))
for task in tasks:
task.result()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=nw_limit,
end_time=event_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
nw_limit=0,
no_of_nodes=2,
event_time=event_time,
nw_usage="0")
def test_nwusage_with_hard_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
failover_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node failed over at {0}".format(failover_time))
self.sleep(15)
self.src_cluster.rebalance_in()
node_back_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
end_time=failover_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=failover_time,
end_time=node_back_time,
no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time)
def test_nwusage_with_hard_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
self.sleep(15)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
bw_enable_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info(
"Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
self.src_cluster.rebalance_in()
node_back_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=bw_enable_time,
end_time=node_back_time,
no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.update_autofailover_settings(enabled=True, timeout=30)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(30)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [],
[])
task.result()
failover_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node auto failed over at {0}".format(failover_time))
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(45)
self.src_cluster.rebalance_in()
node_back_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
end_time=failover_time,
no_of_nodes=3)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=failover_time,
end_time=node_back_time,
no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time,
no_of_nodes=3)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne',
'nwOne',
self._value_size,
end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(45)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [],
[])
task.result()
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
nw_limit = self._input.param("nw_limit", 1)
self.src_cluster.set_xdcr_param("networkUsageLimit", nw_limit)
bw_enable_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info(
"Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(30)
self.src_cluster.rebalance_in()
node_back_time = time.strftime('%Y-%m-%dT%H:%M:%S')
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=bw_enable_time,
end_time=node_back_time,
no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(),
event_time=node_back_time,
no_of_nodes=3)
class nwusage(XDCRNewBaseTest):
def setUp(self):
super(nwusage, self).setUp()
self.src_cluster = self.get_cb_cluster_by_name('C1')
self.src_master = self.src_cluster.get_master_node()
self.dest_cluster = self.get_cb_cluster_by_name('C2')
self.dest_master = self.dest_cluster.get_master_node()
self.cluster = Cluster()
def tearDown(self):
super(nwusage, self).tearDown()
def _set_nwusage_limit(self, cluster, nw_limit=0):
repl_id = cluster.get_remote_clusters()[0].get_replications()[0].get_repl_id()
shell = RemoteMachineShellConnection(cluster.get_master_node())
repl_id = str(repl_id).replace('/','%2F')
base_url = "http://" + cluster.get_master_node().ip + ":8091/settings/replications/" + repl_id
command = "curl -X POST -u Administrator:password " + base_url + " -d networkUsageLimit=" + str(nw_limit)
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
def _verify_bandwidth_usage(self, node, nw_limit=1, no_of_nodes=2, event_time=None,
nw_usage="[1-9][0-9]*", end_time=None):
goxdcr_log = NodeHelper.get_goxdcr_log_dir(node) + '/goxdcr.log'
nw_max = (nw_limit * 1024 * 1024)/no_of_nodes
if event_time:
time_to_compare = time.strptime(event_time, '%Y-%m-%dT%H:%M:%S')
else:
matches, _ = NodeHelper.check_goxdcr_log(node, "Success adding replication specification",
goxdcr_log, print_matches=True)
time_to_compare_str = matches[-1].split(' ')[0].split('.')[0]
time_to_compare = time.strptime(time_to_compare_str, '%Y-%m-%dT%H:%M:%S')
matches, count = NodeHelper.check_goxdcr_log(node, "bandwidth_limit=" + str(nw_max) +
", bandwidth_usage=" + nw_usage, goxdcr_log, print_matches=True)
match_count = 0
skip_count = 0
for item in matches:
items = item.split(' ')
item_time = items[0].split('.')[0]
item_datetime = time.strptime(item_time, '%Y-%m-%dT%H:%M:%S')
if item_datetime < time_to_compare:
skip_count += 1
continue
if end_time:
end_datetime = time.strptime(end_time, '%Y-%m-%dT%H:%M:%S')
if item_datetime > end_datetime:
skip_count += 1
continue
bandwidth_usage = items[-1].split('=')[-1]
if int(bandwidth_usage) <= nw_max:
match_count += 1
continue
else:
self.fail("Bandwidth usage higher than Bandwidth limit in {0}".format(item))
if match_count + skip_count == count:
self.log.info("{0} stale entries skipped".format(skip_count))
if match_count > 0:
self.log.info("{0} entries checked - Bandwidth usage always lower than Bandwidth limit as expected".
format(match_count))
else:
if self._input.param("replication_type") == "capi":
self.log.info("Bandwidth Throttler not enabled on replication as expected")
else:
self.fail("Bandwidth Throttler not enabled on replication")
def _get_current_time(self, server):
shell = RemoteMachineShellConnection(server)
command = "date +'%Y-%m-%dT%H:%M:%S'"
output, error = shell.execute_command(command)
shell.log_command_output(output, error)
curr_time = output[0].strip()
return curr_time
def test_nwusage_with_unidirection(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
self.dest_cluster.load_all_buckets_from_generator(kv_gen=gen_create2)
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_unidirection_pause_resume(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self.src_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection_pause_resume(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
tasks.extend(self.dest_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create2))
self.src_cluster.pause_all_replications()
self.dest_cluster.pause_all_replications()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
self.src_cluster.resume_all_replications()
self.dest_cluster.resume_all_replications()
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_unidirection_in_parallel(self):
self.setup_xdcr()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_bidirection_in_parallel(self):
self.setup_xdcr()
gen_create1 = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create1)
gen_create2 = BlobGenerator('nwTwo', 'nwTwo', self._value_size, end=self._num_items)
tasks.extend(self.dest_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create2))
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self._set_nwusage_limit(self.dest_cluster, nw_limit)
for task in tasks:
task.result()
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit)
self._verify_bandwidth_usage(node=self.dest_cluster.get_master_node(), nw_limit=nw_limit)
def test_nwusage_with_rebalance_in(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_in()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, no_of_nodes=3)
def test_nwusage_with_rebalance_out(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.rebalance_out()
self.perform_update_delete()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, no_of_nodes=1)
def test_nwusage_reset_to_zero(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
tasks = self.src_cluster.async_load_all_buckets_from_generator(kv_gen=gen_create)
self.sleep(30)
self._set_nwusage_limit(self.src_cluster, 0)
event_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Network limit reset to 0 at {0}".format(event_time))
for task in tasks:
task.result()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=nw_limit, end_time=event_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), nw_limit=0, no_of_nodes=2, event_time=event_time, nw_usage="0")
def test_nwusage_with_hard_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.sleep(60)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
failover_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node failed over at {0}".format(failover_time))
self.sleep(15)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup()
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), end_time=failover_time)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=failover_time, end_time=node_back_time, no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time)
def test_nwusage_with_hard_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
self.src_cluster.failover_and_rebalance_nodes()
self.sleep(15)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
bw_enable_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=bw_enable_time, end_time=node_back_time, no_of_nodes=1)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
src_conn = RestConnection(self.src_cluster.get_master_node())
src_conn.update_autofailover_settings(enabled=True, timeout=30)
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(30)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
task.result()
failover_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node auto failed over at {0}".format(failover_time))
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(45)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), end_time=failover_time, no_of_nodes=3)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=failover_time, end_time=node_back_time, no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time, no_of_nodes=3)
def test_nwusage_with_auto_failover_and_bwthrottle_enabled_later(self):
self.setup_xdcr()
self.src_cluster.rebalance_in()
self.src_cluster.pause_all_replications()
gen_create = BlobGenerator('nwOne', 'nwOne', self._value_size, end=self._num_items)
self.src_cluster.load_all_buckets_from_generator(kv_gen=gen_create)
self.src_cluster.resume_all_replications()
self.sleep(15)
shell = RemoteMachineShellConnection(self._input.servers[1])
shell.stop_couchbase()
self.sleep(45)
task = self.cluster.async_rebalance(self.src_cluster.get_nodes(), [], [])
task.result()
FloatingServers._serverlist.append(self._input.servers[1])
self.sleep(15)
nw_limit = self._input.param("nw_limit", 1)
self._set_nwusage_limit(self.src_cluster, nw_limit)
bw_enable_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Bandwidth throttler enabled at {0}".format(bw_enable_time))
self.sleep(60)
shell.start_couchbase()
shell.disable_firewall()
self.sleep(30)
self.src_cluster.rebalance_in()
node_back_time = self._get_current_time(self.src_cluster.get_master_node())
self.log.info("Node added back at {0}".format(node_back_time))
self._wait_for_replication_to_catchup(timeout=600)
self.verify_results()
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=bw_enable_time, end_time=node_back_time, no_of_nodes=2)
self._verify_bandwidth_usage(node=self.src_cluster.get_master_node(), event_time=node_back_time, no_of_nodes=3)
