def operate(self,
arg1,
arg2=None,
group_by=None,
result_comp_op=None,
result_comp_val=None,
on_common_only=False,
save_param=None):
if not arg1:
# if empty opearand
raise HealthException("Wrong operand for Complex operation.")
if group_by:
arg1 = do_multiple_group_by(arg1, group_by)
if not arg1:
# if not valid group_by ids, we will get empty arg1
raise HealthException(
"Invalid group ids %s for Complex operation." %
(str(group_by)))
try:
return apply_operator(
arg1,
NOKEY,
lambda kv, sp: self.op_fn(kv, operators[result_comp_op],
result_comp_val, sp),
group_by[-1] if group_by else "CLUSTER",
save_param=save_param,
update_saved_list=True)
except Exception as e:
raise HealthException(str(e) + " for Complex Operation")
def operate(self,
arg1,
arg2=None,
group_by=None,
result_comp_op=None,
result_comp_val=None,
on_common_only=False,
save_param=None):
if not arg1:
raise HealthException("Wrong operand for Aggregation operation.")
if group_by:
arg1 = do_multiple_group_by(arg1, group_by)
if not arg1:
# if not valid group_by ids, we will get empty arg1
raise HealthException(
"Invalid group ids %s for Aggregation operation." %
(str(group_by)))
try:
return apply_operator(
arg1,
NOKEY,
self.op_fn,
group_by[-1] if group_by else "CLUSTER",
on_all_keys=False if self.op == "COUNT" else True,
save_param=save_param,
update_saved_list=True)
except Exception as e:
raise HealthException(str(e) + " for Aggregation Operation")
def select_keys(data={}, select_keys=[], select_from_keys=[], ignore_keys=[], save_param=None):
if not data or not isinstance(data, dict):
raise HealthException("Wrong Input Data for select operation.")
if not select_keys:
raise HealthException("No key provided for select operation.")
if not select_from_keys:
select_from_keys = []
if (not select_from_keys or (select_from_keys[0] != "ALL"
and not select_from_keys[0].startswith(SNAPSHOT_KEY_PREFIX))):
select_from_keys.insert(0, create_snapshot_key(len(data.keys()) - 1))
elif select_from_keys[0].startswith(SNAPSHOT_KEY_PREFIX):
select_from_keys[0] = create_snapshot_key(int(re.search(SNAPSHOT_KEY_PATTERN, select_from_keys[0]).group(1)))
config_param = False
if "CONFIG" in select_from_keys:
config_param = True
result = select_keys_from_dict(data=data, keys=select_keys, from_keys=select_from_keys, ignore_keys=ignore_keys,
save_param=save_param, config_param=config_param)
if not result:
raise HealthException(
"Wrong input for select operation, Nothing matches with input keys.")
return result
def apply_operator(data,
key,
op_fn,
group_by=None,
arg2=None,
recurse=False,
on_all_keys=True,
save_param=None,
update_saved_list=False):
res_dict = {}
if not data or not isinstance(data, dict):
raise HealthException("Wrong Input Data ")
if not group_by:
raise HealthException("No Group Id ")
for _key in data.keys():
k = merge_key(key, _key, recurse)
if _key[1] == group_by:
# User merged key for aggregation result
if on_all_keys:
# Apply operation on all leaf values
res_dict[k] = op_fn(
find_kv_vector(NOKEY,
data[_key],
recurse=True,
update_saved_list=update_saved_list),
save_param)
else:
# Apply operation on next level only, no further
if isinstance(data[_key], dict):
# Next level is dict, so apply operation on keys
res_dict[k] = op_fn(data[_key].keys(), save_param)
else:
# Next level is not dict, so apply operation on value
res_dict[k] = op_fn([data[_key]], save_param)
else:
res_dict[_key] = apply_operator(
data[_key],
k,
op_fn,
group_by,
arg2,
recurse,
on_all_keys=on_all_keys,
save_param=save_param,
update_saved_list=update_saved_list)
return res_dict
def do_multiple_group_by(d, group_by_list):
if not group_by_list:
raise HealthException("No Group ID for group by operation")
if not d or not isinstance(d, dict):
raise HealthException("Wrong Input Data for group by operation.")
res = {}
group_by = group_by_list[0]
res = do_group_by(d, group_by)
if res and len(group_by_list) > 1:
for _k in res:
res[_k] = do_multiple_group_by(res[_k], group_by_list[1:])
return res
def vector_to_scalar_value_uniform_operation(op, v):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return boolean scalar result True if all values are uniformly distributed
"""
if not v or not isinstance(v, list):
raise HealthException(
"Insufficient input for Value Uniform operation ")
d = {}
for i in v:
k2, v2 = get_kv(i)
v2 = get_value_from_health_internal_tuple(v2)
if v2 and isinstance(v2, list):
v2 = sorted(v2)
if v2 not in d:
d[v2] = 1
else:
d[v2] = d[v2] + 1
minv = min(d.values())
maxv = max(d.values())
if (maxv - minv) > 1:
return False
return True
def vector_to_scalar_equal_operation(op, v):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return boolean scalar result True if all values are equal
"""
if not op or not v or not isinstance(v, list):
raise HealthException("Insufficient input for Equal operation ")
i0 = v[0]
k1, v1 = get_kv(i0)
v1 = get_value_from_health_internal_tuple(v1)
if v1 and isinstance(v1, list):
v1 = sorted(v1)
for i in v[1:]:
k2, v2 = get_kv(i)
v2 = get_value_from_health_internal_tuple(v2)
if v2 and isinstance(v2, list):
v2 = sorted(v2)
if not op(v1, v2):
return False
return True
def operate(self,
arg1,
arg2,
group_by=None,
result_comp_op=None,
result_comp_val=None,
on_common_only=False,
save_param=None):
if arg1 is None or arg2 is None:
raise HealthException("Wrong operands for Apply operation.")
if result_comp_op is None:
raise HealthException("Wrong operator for Apply operation.")
# No Group By So No Key Merging
return self._operate_dicts(arg1,
arg2,
comp_op=operators[result_comp_op],
save_param=save_param)
def do_group_by(data, group_by, keys=[]):
if not group_by:
raise HealthException("No Group ID for group by operation")
if not data:
raise HealthException("Wrong Input Data for group by operation.")
if not isinstance(data, dict):
raise HealthException("Wrong group id %s for group by operation." %
(str(group_by)))
res = {}
for k, t in data.keys():
temp_d = res
if t == group_by:
if (k, t) not in temp_d:
temp_d[(k, t)] = {}
if keys:
temp_d = temp_d[(k, t)]
for _k in keys[:-1]:
if _k not in temp_d:
temp_d[_k] = {}
temp_d = temp_d[_k]
temp_d[keys[-1]] = data[(k, t)]
else:
temp_d[(k, t)] = data[(k, t)]
else:
keys.append((k, t))
try:
res = deep_merge_dicts(
res, do_group_by(data[(k, t)], group_by, keys))
except Exception as e:
raise e
finally:
keys.remove((k, t))
return res
def operate(self,
data={},
check_val=create_health_internal_tuple(True, []),
error=None,
category=None,
level=None,
description=None,
success_msg=None):
if not data:
raise HealthException("Wrong Input Data for ASSERT operation.")
res = {}
res[AssertResultKey.FAIL_MSG] = str(error)
res[AssertResultKey.DESCRIPTION] = description
res[AssertResultKey.SUCCESS_MSG] = success_msg
res[AssertResultKey.KEYS] = []
try:
res[AssertResultKey.CATEGORY] = category.upper().split(".")
except Exception:
res[AssertResultKey.CATEGORY] = None
res[AssertResultKey.LEVEL] = level
if not isinstance(data, dict):
if not self.op(get_value_from_health_internal_tuple(data),
get_value_from_health_internal_tuple(check_val)):
return (ParserResultType.ASSERT, res)
return None
kv = find_kv_vector(NOKEY, data, recurse=True, update_saved_list=False)
if not kv:
return (ParserResultType.ASSERT, res)
fail = False
for i in kv:
k, v = get_kv(i)
kv_tuple = (k, None)
value_to_check = get_value_from_health_internal_tuple(v)
if v[1]:
kv_tuple = (k, v[1])
if not self.op(value_to_check,
get_value_from_health_internal_tuple(check_val)):
res[AssertResultKey.SUCCESS] = False
fail = True
res[AssertResultKey.KEYS].append(kv_tuple)
if not fail:
res[AssertResultKey.SUCCESS] = True
return (ParserResultType.ASSERT, res)
def _operate_dicts(self, arg1, arg2, save_param=None):
if isinstance(arg2, dict):
raise HealthException(
"Wrong parameter type (dictionary) for Simple operation.")
if isinstance(arg1, dict):
res_dict = {}
for _k in arg1.keys():
res_dict[_k] = self._operate_dicts(arg1[_k],
arg2,
save_param=save_param)
return res_dict
else:
return self._operate_each_key(arg1, arg2, save_param=save_param)
def vector_to_scalar_first_operation(op, v):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Returns first value from values
"""
if not v or not isinstance(v, list):
raise HealthException("Insufficient input for Random operation ")
try:
return get_value_from_health_internal_tuple(get_kv(v[0])[1])
except Exception:
return None
def vector_to_vector_no_match_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return health internal tuple
(True/False , [(key, value, formatting), (key, value, formatting), ...])
"""
res = {}
operand = get_value_from_health_internal_tuple(a)
if not kv:
raise HealthException("Insufficient input for NO_MATCH operation ")
try:
values = [
get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv
]
match_operand = _find_match_operand_value(operand, values)
result = False
val_to_save = []
for x in kv:
k, v = get_kv(x)
_val = get_value_from_health_internal_tuple(v)
if not op(_val, match_operand):
result |= True
val_to_save += create_value_list_to_save(save_param=None,
value=result,
op1=v)
if operand and operand == MAJORITY:
key = "Majority Value"
else:
key = "Expected Value"
val_to_save += create_value_list_to_save(save_param=save_param,
key=key,
value=match_operand)
res = create_health_internal_tuple(result, val_to_save)
except Exception:
res = create_health_internal_tuple(False, None)
return res
def basic_vector_to_scalar_operation(op, kv, typecast=int, initial_value=None):
"""
Passed Vector values and type of value
[ {(name, tag) : value}, {(name, tag) : value} ...
Return aggregated result along with count of element processed
"""
if not op or not kv or not isinstance(kv, list):
raise HealthException("Insufficient input for vector operation ")
if initial_value:
found_first = True
res = initial_value
else:
found_first = False
res = None
cnt = 0
for i in kv:
k1, v1 = get_kv(i)
v1 = get_value_from_health_internal_tuple(v1)
try:
if not found_first:
res = typecast(v1)
found_first = True
cnt = cnt + 1
continue
except Exception:
continue
try:
res = op(res, v1)
cnt = cnt + 1
except Exception:
continue
return res, cnt
def _operate_dicts(self,
arg1,
arg2,
on_common_only=False,
save_param=None):
if isinstance(arg1, dict) and isinstance(arg2, dict):
k1_set = set(arg1.keys())
k2_set = set(arg2.keys())
if ((len(k1_set - k2_set) > 0 or len(k2_set - k1_set) > 0)
and not on_common_only):
raise HealthException(
"Wrong operands with non-matching keys for Binary operation."
)
res_dict = {}
for _k in k1_set.intersection(k2_set):
res_dict[_k] = self._operate_dicts(
arg1[_k],
arg2[_k],
on_common_only=on_common_only,
save_param=save_param)
return res_dict
else:
return self._operate_each_key(arg1, arg2, save_param=save_param)
def select_keys_from_dict(data={},
keys=[],
from_keys=[],
ignore_keys=[],
save_param=None,
config_param=False):
"""
Function takes dictionary, list of keys to fetch, list of from_keys to filter scope
Returns dictionary of selected keys and values
"""
if not data or not isinstance(data, dict):
raise HealthException("Wrong Input Data for select operation.")
result_dict = {}
if not keys:
raise HealthException("No key provided for select operation.")
for _key in data:
if from_keys:
f_key = from_keys[0]
if isinstance(_key, tuple):
# from_keys work with static component keys only, if we get
# tuple keys means we have done with checking of all component
# keys and not found any from key match so no need to check
# further in this direction
break
if (f_key == "ALL") or (_key == f_key):
# from_key is ALL or matching with _key
child_res = select_keys_from_dict(
data[_key],
keys=keys,
from_keys=from_keys[1:] if len(from_keys) > 1 else [],
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
else:
# no key match, need to check further
child_res = select_keys_from_dict(data[_key],
keys=keys,
from_keys=from_keys,
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
if child_res:
if f_key == "ALL":
# It assumes ALL is only for top snapshot level
result_dict[(_key, "SNAPSHOT")] = copy.deepcopy(child_res)
else:
result_dict = deep_merge_dicts(result_dict,
copy.deepcopy(child_res))
else:
# if (False, "*", None) in keys and isinstance(_key, tuple):
# result_dict[_key] = copy.deepcopy(data[_key])
if isinstance(_key, tuple) and _key[1] == "KEY":
for check_substring, s_key, new_name in keys:
if ((s_key == "*"
and not _is_key_in_ignore_keys(_key[0], ignore_keys))
or (check_substring and re.search(s_key, _key[0]))
or (not check_substring and _key[0] == s_key)):
val_to_save = create_value_list_to_save(
save_param=save_param,
key=_key[0],
value=data[_key],
formatting=not config_param)
if new_name:
result_dict[(
new_name,
"KEY")] = create_health_internal_tuple(
data[_key], val_to_save)
else:
result_dict[_key] = create_health_internal_tuple(
data[_key], val_to_save)
break
elif data[_key] and isinstance(data[_key], dict):
child_res = select_keys_from_dict(data[_key],
keys=keys,
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
if child_res:
if isinstance(_key, tuple):
result_dict[_key] = copy.deepcopy(child_res)
else:
result_dict = deep_merge_dicts(
result_dict, copy.deepcopy(child_res))
return result_dict
def vector_to_vector_diff_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return boolean dictionary result
{ (name, tag) : True/False , (name, tag) : True/False, ... }
"""
res = {}
temp_res = {}
if not kv or not a:
raise HealthException("Insufficient input for Diff operation ")
exception_found = False
try:
for x, y in itertools.combinations(kv, 2):
k1, v1 = get_kv(x)
k2, v2 = get_kv(y)
_v1 = get_value_from_health_internal_tuple(v1)
_v2 = get_value_from_health_internal_tuple(v2)
if op(abs(_v1 - _v2), a):
try:
temp_res[make_key(k1)] |= True
except Exception:
temp_res[make_key(k1)] = True
try:
temp_res[make_key(k2)] |= True
except Exception:
temp_res[make_key(k2)] = True
else:
try:
temp_res[make_key(k1)] |= False
except Exception:
temp_res[make_key(k1)] = False
try:
temp_res[make_key(k2)] |= False
except Exception:
temp_res[make_key(k2)] = False
for i in kv:
k, v = get_kv(i)
val_to_save = create_value_list_to_save(
save_param, value=temp_res[make_key(k)], op1=v)
res[make_key(k)] = create_health_internal_tuple(
temp_res[make_key(k)], val_to_save)
except Exception:
exception_found = True
if exception_found:
for x in kv:
k, v = get_kv(x)
res[make_key(k)] = create_health_internal_tuple(None, None)
return res
def vector_to_vector_sd_anomaly_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return health internal tuple
(True/False , [(key, value, formatting), (key, value, formatting), ...])
"""
res = {}
sd_multiplier = get_value_from_health_internal_tuple(a)
if not kv or not sd_multiplier:
raise HealthException("Insufficient input for SD_ANOMALY operation ")
try:
n = len(kv)
if n < 3:
no_anomaly = True
range_start = 0
range_end = 0
else:
values = [
get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv
]
no_anomaly = False
try:
# We should consider int and floats only
s = sum(values)
except Exception:
no_anomaly = True
if not no_anomaly:
mean = float(s) / float(n)
variance = 0
for v in values:
variance += pow((v - mean), 2)
variance = float(variance) / float(n)
sd = sqrt(variance)
range_start = mean - (sd_multiplier * sd)
range_end = mean + (sd_multiplier * sd)
result = False
val_to_save = []
for x in kv:
k, v = get_kv(x)
_val = get_value_from_health_internal_tuple(v)
if not no_anomaly and (float(_val) < float(range_start)
or float(_val) > float(range_end)):
result |= True
val_to_save += create_value_list_to_save(save_param=None,
value=result,
op1=v)
val_to_save += create_value_list_to_save(save_param=save_param,
value=result)
res = create_health_internal_tuple(result, val_to_save)
except Exception:
res = create_health_internal_tuple(False, None)
return res
