def save_cbgt_diagnostics(self):
# CBGT REST Admin API endpoint
for sync_gateway_writer in self.sg_accels:
adminApi = Admin(sync_gateway_writer)
cbgt_diagnostics = adminApi.get_cbgt_diagnostics()
adminApi.get_cbgt_cfg()
# dump raw diagnostics
pretty_print_json = json.dumps(cbgt_diagnostics, sort_keys=True, indent=4, separators=(',', ': '))
log_info("SG {} CBGT diagnostic output: {}".format(sync_gateway_writer, pretty_print_json))
def validate_cbgt_pindex_distribution(self, num_running_sg_accels):
if num_running_sg_accels < 1:
raise keywords.exceptions.ClusterError("Need at least one sg_accel running to verify pindexes")
# build a map of node -> num_pindexes
node_defs_pindex_counts = {}
# CBGT REST Admin API endpoint
adminApi = Admin(self.sg_accels[0])
cbgt_cfg = adminApi.get_cbgt_cfg()
# loop over the planpindexes and update the count for the node where it lives
# this will end up with a dictionary like:
# {'74c818f04b99b169': 32, '11886131c807a30e': 32} (each node uuid has 32 pindexes)
plan_pindexes = cbgt_cfg["planPIndexes"]["planPIndexes"]
for data_bucket_key, data_bucket_val in plan_pindexes.iteritems():
# get the nodes where this pindex lives
nodes = data_bucket_val["nodes"]
# it should only live on one node. if not, abort.
if len(nodes) > 1:
raise Exception("Unexpected: a CBGT Pindex was assigned to more than one node")
# loop over the nodes where this pindex lives and increment the count
for node in nodes:
# add a key for this node if we don't already have one
if node not in node_defs_pindex_counts:
node_defs_pindex_counts[node] = 0
current_pindex_count = node_defs_pindex_counts[node]
current_pindex_count += 1
node_defs_pindex_counts[node] = current_pindex_count
log_info("CBGT node to pindex counts: {}".format(node_defs_pindex_counts))
# make sure number of unique node uuids is equal to the number of sync gateway writers
if len(node_defs_pindex_counts) != num_running_sg_accels:
log_info("CBGT len(unique_node_uuids) != len(self.sync_gateway_writers) ({} != {})".format(
len(node_defs_pindex_counts),
num_running_sg_accels
))
return False
# make sure that all of the nodes have approx the same number of pindexes assigneed to them
i = 0
num_pindex_first_node = 0
for node_def_uuid, num_pindexes in node_defs_pindex_counts.iteritems():
if i == 0:
# it's the first node we've looked at, just record number of pindexes and continue
num_pindex_first_node = num_pindexes
i += 1
continue
# ok, it's the 2nd+ node, make sure the delta with the first node is less than or equal to 1
# (the reason we can't compare for equality is that sometimes the pindexes can't be
# divided evenly across the cluster)
delta = abs(num_pindex_first_node - num_pindexes)
if delta > 1:
log_info("CBGT Sync Gateway node {} has {} pindexes, but other node has {} pindexes.".format(
node_def_uuid,
num_pindexes,
num_pindex_first_node
))
return False
return True
