def test_missing_num_shards(params_from_base_test_setup, sg_conf):
"""
1. Launch sg_accels missing the following property in the config.
"num_shards":16
2. Verify there are 16 shards
3. Verify they are distributed evenly across the nodes
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_missing_num_shards'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# CBGT REST Admin API endpoint
admin_api = Admin(cluster.sg_accels[1])
cbgt_cfg = admin_api.get_cbgt_config()
# Verify that default number of pindex shards is 16.
# This may change in the future in which case this test will need to be updated.
assert cbgt_cfg.num_shards == 16
# Verify sharding is correct
assert cluster.validate_cbgt_pindex_distribution_retry(
num_running_sg_accels=3)
def save_cbgt_diagnostics(self):
# CBGT REST Admin API endpoint
for sync_gateway_writer in self.sg_accels:
adminApi = Admin(sync_gateway_writer)
cbgt_diagnostics = adminApi.get_cbgt_diagnostics()
adminApi.get_cbgt_config()
# dump raw diagnostics
pretty_print_json = json.dumps(cbgt_diagnostics,
sort_keys=True,
indent=4,
separators=(',', ': '))
log_info("SG {} CBGT diagnostic output: {}".format(
sync_gateway_writer, pretty_print_json))
def validate_cbgt_pindex_distribution(self, num_running_sg_accels):
if num_running_sg_accels < 1:
raise keywords.exceptions.ClusterError(
"Need at least one sg_accel running to verify pindexes")
# build a map of node -> num_pindexes
node_defs_pindex_counts = {}
# CBGT REST Admin API endpoint
adminApi = Admin(self.sg_accels[0])
cbgt_cfg = adminApi.get_cbgt_config()
# loop over the planpindexes and update the count for the node where it lives
# this will end up with a dictionary like:
# {'74c818f04b99b169': 32, '11886131c807a30e': 32} (each node uuid has 32 pindexes)
plan_pindexes = cbgt_cfg.p_indexes
for data_bucket_key, data_bucket_val in plan_pindexes.iteritems():
# get the nodes where this pindex lives
nodes = data_bucket_val["nodes"]
# it should only live on one node. if not, abort.
if len(nodes) > 1:
raise Exception(
"Unexpected: a CBGT Pindex was assigned to more than one node"
)
# loop over the nodes where this pindex lives and increment the count
for node in nodes:
# add a key for this node if we don't already have one
if node not in node_defs_pindex_counts:
node_defs_pindex_counts[node] = 0
current_pindex_count = node_defs_pindex_counts[node]
current_pindex_count += 1
node_defs_pindex_counts[node] = current_pindex_count
log_info(
"CBGT node to pindex counts: {}".format(node_defs_pindex_counts))
# make sure number of unique node uuids is equal to the number of sync gateway writers
if len(node_defs_pindex_counts) != num_running_sg_accels:
log_info(
"CBGT len(unique_node_uuids) != len(self.sync_gateway_writers) ({} != {})"
.format(len(node_defs_pindex_counts), num_running_sg_accels))
return False
# make sure that all of the nodes have approx the same number of pindexes assigneed to them
i = 0
num_pindex_first_node = 0
for node_def_uuid, num_pindexes in node_defs_pindex_counts.iteritems():
if i == 0:
# it's the first node we've looked at, just record number of pindexes and continue
num_pindex_first_node = num_pindexes
i += 1
continue
# ok, it's the 2nd+ node, make sure the delta with the first node is less than or equal to 1
# (the reason we can't compare for equality is that sometimes the pindexes can't be
# divided evenly across the cluster)
delta = abs(num_pindex_first_node - num_pindexes)
if delta > 1:
log_info(
"CBGT Sync Gateway node {} has {} pindexes, but other node has {} pindexes."
.format(node_def_uuid, num_pindexes,
num_pindex_first_node))
return False
return True
