def _check_key_for_gt(self, d={}, keys=(), v=0, is_and=False, type_check=int):
if not keys:
return True
if not d:
return False
if not isinstance(keys, tuple):
keys = (keys,)
if is_and:
if all(util.get_value_from_dict(d, k, v, type_check) > v for k in keys):
return True
else:
if any(util.get_value_from_dict(d, k, v, type_check) > v for k in keys):
return True
return False
def get_pmap(self, nodes='all'):
getter = GetStatisticsController(self.cluster)
node_ids = util.Future(self.cluster.info, 'node', nodes=nodes).start()
pmap_info = util.Future(self.cluster.info,
'partition-info',
nodes=nodes).start()
service_stats = getter.get_service(nodes=nodes)
namespace_stats = getter.get_namespace(nodes=nodes)
node_ids = node_ids.result()
pmap_info = pmap_info.result()
cluster_keys = {}
for node in service_stats.keys():
if not service_stats[node] or isinstance(service_stats[node],
Exception):
cluster_keys[node] = "N/E"
else:
cluster_keys[node] = util.get_value_from_dict(
service_stats[node], ('cluster_key'), default_value="N/E")
ns_info = self._get_namespace_data(namespace_stats, cluster_keys)
pmap_data = self._get_pmap_data(pmap_info, ns_info, cluster_keys,
node_ids)
return pmap_data
def _get_namespace_data(self, namespace_stats, cluster_keys):
ns_info = {}
for ns, nodes in namespace_stats.items():
repl_factor = {}
for node, params in nodes.items():
if isinstance(params, Exception):
continue
if cluster_keys[node] not in repl_factor:
repl_factor[cluster_keys[node]] = 0
repl_factor[cluster_keys[node]] = max(
repl_factor[cluster_keys[node]],
util.get_value_from_dict(
params,
('repl-factor', 'replication-factor',
'effective_replication_factor'
), # introduced post 3.15.0.1
default_value=0,
return_type=int))
for ck in repl_factor:
if ck not in ns_info:
ns_info[ck] = {}
if ns not in ns_info[ck]:
ns_info[ck][ns] = {}
ns_info[ck][ns]['repl_factor'] = repl_factor[ck]
return ns_info
def _get_namespace_data(self, namespace_stats, cluster_keys):
ns_info = {}
# stats to fetch
stats = ["dead_partitions", "unavailable_partitions"]
for ns, nodes in namespace_stats.items():
for node, params in nodes.items():
if isinstance(params, Exception):
continue
if cluster_keys[node] not in ns_info:
ns_info[cluster_keys[node]] = {}
d = ns_info[cluster_keys[node]]
if ns not in d:
d[ns] = {}
d = d[ns]
if node not in d:
d[node] = {}
for s in stats:
util.set_value_in_dict(
d[node], s, util.get_value_from_dict(params, (s, )))
return ns_info
def _get_namespace_data(self, namespace_stats, cluster_keys):
ns_info = {}
# stats to fetch
stats = ["dead_partitions", "unavailable_partitions"]
for ns, nodes in namespace_stats.items():
for node, params in nodes.items():
if isinstance(params, Exception):
continue
if cluster_keys[node] not in ns_info:
ns_info[cluster_keys[node]] = {}
d = ns_info[cluster_keys[node]]
if ns not in d:
d[ns] = {}
d = d[ns]
if node not in d:
d[node] = {}
for s in stats:
util.set_value_in_dict(d[node], s,
util.get_value_from_dict(params, (s,))
)
return ns_info
def _check_key_for_gt(self,
d={},
keys=(),
v=0,
is_and=False,
type_check=int):
if not keys:
return True
if not d:
return False
if not isinstance(keys, tuple):
keys = (keys, )
if is_and:
if all(
util.get_value_from_dict(d, k, v, type_check) > v
for k in keys):
return True
else:
if any(
util.get_value_from_dict(d, k, v, type_check) > v
for k in keys):
return True
return False
def get_pmap(self, nodes='all'):
getter = GetStatisticsController(self.cluster)
node_ids = util.Future(self.cluster.info, 'node', nodes=nodes).start()
pmap_info = util.Future(self.cluster.info, 'partition-info', nodes=nodes).start()
service_stats = getter.get_service(nodes=nodes)
namespace_stats = getter.get_namespace(nodes=nodes)
node_ids = node_ids.result()
pmap_info = pmap_info.result()
cluster_keys = {}
for node in service_stats.keys():
if not service_stats[node] or isinstance(service_stats[node], Exception):
cluster_keys[node] = "N/E"
else:
cluster_keys[node] = util.get_value_from_dict(service_stats[node], ('cluster_key'), default_value="N/E")
ns_info = self._get_namespace_data(namespace_stats, cluster_keys)
pmap_data = self._get_pmap_data(pmap_info, ns_info, cluster_keys, node_ids)
return pmap_data
def merge_dicts_with_new_tuple_keys(dict_from,
main_dict,
new_tuple_keys,
forced_all_new_keys=True):
if not dict_from and dict_from != 0:
return
if main_dict is None:
main_dict = {}
if not isinstance(main_dict, dict):
return
if not isinstance(dict_from, dict):
if isinstance(main_dict, dict) and not main_dict:
main_dict = copy.deepcopy(dict_from)
return
poped_nks, key_level_separator_found = pop_tuple_keys_for_next_level(
new_tuple_keys)
for _key in dict_from.keys():
temp_dict = main_dict
last_level = False
if isinstance(dict_from[_key], dict):
_k = _key
else:
# last _key:value, need to create tuple (_key, "KEY")
_k = (_key, "KEY")
last_level = True
if poped_nks and (not last_level or forced_all_new_keys):
for i, k in enumerate(poped_nks):
type = k[0]
name = k[1]
if type:
# This is valid tuple key requirement, as type is
# available.
if (not name or isinstance(name, list)
or isinstance(name, dict)):
# _key is the name for type
_k = (_key, type)
elif isinstance(name, tuple):
# name of key to fetch is present
_k = (get_value_from_dict(dict_from[_key], name,
_key), type)
else:
# static name provided
_k = (name, type)
if _k not in temp_dict:
temp_dict[_k] = {}
if i < len(poped_nks) - 1:
# added all keys till this path
temp_dict = temp_dict[_k]
else:
if _k not in temp_dict:
temp_dict[_k] = {}
if last_level:
temp_dict[_k] = copy.deepcopy(dict_from[_key])
else:
merge_dicts_with_new_tuple_keys(
dict_from[_key],
temp_dict[_k],
new_tuple_keys,
forced_all_new_keys=forced_all_new_keys)
# Need to push back all poped tuple keys, as same should go to other
# siblings
if key_level_separator_found:
new_tuple_keys.insert(0, (None, None))
while poped_nks:
new_tuple_keys.insert(0, poped_nks.pop())
return
def merge_dicts_with_new_tuple_keys(dict_from, main_dict, new_tuple_keys,
forced_all_new_keys=True):
if not dict_from and dict_from != 0:
return
if main_dict is None:
main_dict = {}
if not isinstance(main_dict, dict):
return
if not isinstance(dict_from, dict):
if isinstance(main_dict, dict) and not main_dict:
main_dict = copy.deepcopy(dict_from)
return
poped_nks, key_level_separator_found = pop_tuple_keys_for_next_level(
new_tuple_keys)
for _key in dict_from.keys():
temp_dict = main_dict
last_level = False
if isinstance(dict_from[_key], dict):
_k = _key
else:
# last _key:value, need to create tuple (_key, "KEY")
_k = (_key, "KEY")
last_level = True
if poped_nks and (not last_level or forced_all_new_keys):
for i, k in enumerate(poped_nks):
type = k[0]
name = k[1]
if type:
# This is valid tuple key requirement, as type is
# available.
if (not name or isinstance(name, list)
or isinstance(name, dict)):
# _key is the name for type
_k = (_key, type)
elif isinstance(name, tuple):
# name of key to fetch is present
_k = (get_value_from_dict(dict_from[_key], name, _key),
type)
else:
# static name provided
_k = (name, type)
if _k not in temp_dict:
temp_dict[_k] = {}
if i < len(poped_nks) - 1:
# added all keys till this path
temp_dict = temp_dict[_k]
else:
if _k not in temp_dict:
temp_dict[_k] = {}
if last_level:
temp_dict[_k] = copy.deepcopy(dict_from[_key])
else:
merge_dicts_with_new_tuple_keys(dict_from[_key], temp_dict[_k],
new_tuple_keys, forced_all_new_keys=forced_all_new_keys)
# Need to push back all poped tuple keys, as same should go to other
# siblings
if key_level_separator_found:
new_tuple_keys.insert(0, (None, None))
while poped_nks:
new_tuple_keys.insert(0, poped_nks.pop())
return