def _operate_each_key(self, arg1, arg2, save_param=None):
if isinstance(arg1, dict) and isinstance(arg2, dict):
return None
if not isinstance(arg1, dict) and not isinstance(arg2, dict):
try:
raw_arg1 = get_value_from_health_internal_tuple(arg1)
raw_arg2 = get_value_from_health_internal_tuple(arg2)
if self.op == operator.div and raw_arg2 == 0:
val_to_save = create_value_list_to_save(save_param,
value=0,
op1=arg1,
op2=arg2)
return (0, val_to_save)
# if any of the arg is type float or operation is division
# cast all argument to float
if self.op == operator.div or isinstance(
raw_arg1, float) or isinstance(raw_arg2, float):
raw_arg1 = float(raw_arg1)
raw_arg2 = float(raw_arg2)
result = self.op(raw_arg1, raw_arg2)
val_to_save = create_value_list_to_save(save_param,
value=result,
op1=arg1,
op2=arg2)
return create_health_internal_tuple(result, val_to_save)
except Exception:
return create_health_internal_tuple(None, [])
dict_first = True
if isinstance(arg1, dict):
d = arg1
v = arg2
elif isinstance(arg2, dict):
d = arg2
v = arg1
dict_first = False
res_dict = {}
for _k in d:
if dict_first:
res_dict[_k] = self._operate_each_key(d[_k],
v,
save_param=save_param)
else:
res_dict[_k] = self._operate_each_key(v,
d[_k],
save_param=save_param)
return res_dict
def _operate_each_key(self, arg1, arg2, save_param=None):
if isinstance(arg1, dict):
return None
try:
raw_arg1 = get_value_from_health_internal_tuple(arg1)
raw_arg2 = get_value_from_health_internal_tuple(arg2)
result = self.op(raw_arg1, raw_arg2)
val_to_save = create_value_list_to_save(save_param,
value=result,
op1=arg1,
op2=arg2)
return create_health_internal_tuple(result, val_to_save)
except Exception:
return create_health_internal_tuple(None, [])
def do_assert(op=None, data={}, check_val=create_health_internal_tuple(True,[]), error=None, category=None,
level=None, description=None, success_msg=None):
if op in assert_op_list:
return assert_op_list[op](data, check_val, error, category, level,
description, success_msg)
return None
def op_fn_distributor(self, v, save_param):
result = AggOperation.operator_and_function[self.op](v)
val_to_save = create_value_list_to_save(save_param,
value=result,
op1=v)
return create_health_internal_tuple(result, val_to_save)
def vector_to_vector_no_match_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return health internal tuple
(True/False , [(key, value, formatting), (key, value, formatting), ...])
"""
res = {}
operand = get_value_from_health_internal_tuple(a)
if not kv:
raise HealthException("Insufficient input for NO_MATCH operation ")
try:
values = [
get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv
]
match_operand = _find_match_operand_value(operand, values)
result = False
val_to_save = []
for x in kv:
k, v = get_kv(x)
_val = get_value_from_health_internal_tuple(v)
if not op(_val, match_operand):
result |= True
val_to_save += create_value_list_to_save(save_param=None,
value=result,
op1=v)
if operand and operand == MAJORITY:
key = "Majority Value"
else:
key = "Expected Value"
val_to_save += create_value_list_to_save(save_param=save_param,
key=key,
value=match_operand)
res = create_health_internal_tuple(result, val_to_save)
except Exception:
res = create_health_internal_tuple(False, None)
return res
def p_operand(self, p):
"""
operand : VAR
| constant
"""
if is_health_parser_variable(p[1]):
p[0] = p[1][2]
else:
p[0] = create_health_internal_tuple(p[1], [])
def p_opt_simple_operation_param(self, p):
"""
opt_simple_operation_param : COMMA constant
|
"""
if len(p) == 1:
p[0] = None
else:
p[0] = create_health_internal_tuple(p[2], [])
def p_operand(self, p):
"""
operand : VAR
| constant
"""
if is_health_parser_variable(p[1]):
p[0] = p[1][2]
else:
p[0] = create_health_internal_tuple(p[1], [])
def p_opt_simple_operation_param(self, p):
"""
opt_simple_operation_param : COMMA constant
|
"""
if len(p) == 1:
p[0] = None
else:
p[0] = create_health_internal_tuple(p[2], [])
def add_prefix_to_saved_keys(prefix, data):
if not prefix or not data or not data[1]:
return data
new_saved_value_list = []
for i in data[1]:
_k = prefix
if i[0] and len(i[0].strip()) > 0:
_k += "/%s" % (i[0])
new_saved_value_list.append((_k, i[1], i[2]))
return create_health_internal_tuple(data[0], new_saved_value_list)
def operate(self,
data={},
check_val=create_health_internal_tuple(True, []),
error=None,
category=None,
level=None,
description=None,
success_msg=None):
if not data:
raise HealthException("Wrong Input Data for ASSERT operation.")
res = {}
res[AssertResultKey.FAIL_MSG] = str(error)
res[AssertResultKey.DESCRIPTION] = description
res[AssertResultKey.SUCCESS_MSG] = success_msg
res[AssertResultKey.KEYS] = []
try:
res[AssertResultKey.CATEGORY] = category.upper().split(".")
except Exception:
res[AssertResultKey.CATEGORY] = None
res[AssertResultKey.LEVEL] = level
if not isinstance(data, dict):
if not self.op(get_value_from_health_internal_tuple(data),
get_value_from_health_internal_tuple(check_val)):
return (ParserResultType.ASSERT, res)
return None
kv = find_kv_vector(NOKEY, data, recurse=True, update_saved_list=False)
if not kv:
return (ParserResultType.ASSERT, res)
fail = False
for i in kv:
k, v = get_kv(i)
kv_tuple = (k, None)
value_to_check = get_value_from_health_internal_tuple(v)
if v[1]:
kv_tuple = (k, v[1])
if not self.op(value_to_check,
get_value_from_health_internal_tuple(check_val)):
res[AssertResultKey.SUCCESS] = False
fail = True
res[AssertResultKey.KEYS].append(kv_tuple)
if not fail:
res[AssertResultKey.SUCCESS] = True
return (ParserResultType.ASSERT, res)
def p_complex_comparison_operand(self, p):
"""
complex_comparison_operand : COMPLEX_PARAM
| operand
"""
if is_health_parser_variable(p[1]):
p[0] = p[1][2]
elif not isinstance(p[1], tuple):
p[0] = create_health_internal_tuple(p[1], [])
else:
p[0] = p[1]
def p_complex_comparison_operand(self, p):
"""
complex_comparison_operand : COMPLEX_PARAM
| operand
"""
if is_health_parser_variable(p[1]):
p[0] = p[1][2]
elif not isinstance(p[1], tuple):
p[0] = create_health_internal_tuple(p[1], [])
else:
p[0] = p[1]
def _operate_dicts(self,
arg1,
arg2,
comp_op=None,
check_common=True,
save_param=None):
if isinstance(arg1, dict):
if not isinstance(arg2, dict):
check_common = False
if check_common:
k1_set = set(arg1.keys())
k2_set = set(arg2.keys())
if not list(k1_set.intersection(k2_set)):
check_common = False
result_dict = {}
for _k in arg1:
if check_common:
if _k in arg2:
result_dict[_k] = self._operate_dicts(
arg1[_k],
arg2[_k],
comp_op=comp_op,
check_common=check_common,
save_param=save_param)
else:
result_dict[_k] = self._operate_dicts(
arg1[_k],
arg2,
comp_op=comp_op,
check_common=check_common,
save_param=save_param)
return result_dict
else:
result = self._operate_each_key(arg1, arg2, comp_op=comp_op)
val_to_save = create_value_list_to_save(save_param,
value=result,
op1=arg1)
return create_health_internal_tuple(result, val_to_save)
def p_assert_statement(self, p):
"""
assert_statement : ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg COMMA assert_if_condition RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL RPAREN
"""
if len(p) < 14:
p[0] = do_assert(
op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11])
elif len(p) < 16:
p[0] = do_assert(
op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13])
elif len(p) < 18:
p[0] = do_assert(
op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15])
else:
skip_assert, assert_filter_arg = p[17]
if skip_assert:
p[0] = None
else:
if assert_filter_arg is not None:
data = do_operation(op="==", arg1=p[3], arg2=p[5])
try:
# If key filtration throws exception (due to non-matching), it just passes that and executes main assert
new_data = do_operation(op="||", arg1=data, arg2=assert_filter_arg, on_common_only=True)
if new_data:
data = new_data
except Exception:
pass
p[0] = do_assert(
op=p[1], data=data, check_val=create_health_internal_tuple(True,[]), error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15])
else:
p[0] = do_assert(
op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15])
def p_assert_comparison_arg(self, p):
"""
assert_comparison_arg : constant
"""
p[0] = create_health_internal_tuple(p[1], [])
def vector_to_vector_sd_anomaly_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return health internal tuple
(True/False , [(key, value, formatting), (key, value, formatting), ...])
"""
res = {}
sd_multiplier = get_value_from_health_internal_tuple(a)
if not kv or not sd_multiplier:
raise HealthException("Insufficient input for SD_ANOMALY operation ")
try:
n = len(kv)
if n < 3:
no_anomaly = True
range_start = 0
range_end = 0
else:
values = [
get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv
]
no_anomaly = False
try:
# We should consider int and floats only
s = sum(values)
except Exception:
no_anomaly = True
if not no_anomaly:
mean = float(s) / float(n)
variance = 0
for v in values:
variance += pow((v - mean), 2)
variance = float(variance) / float(n)
sd = sqrt(variance)
range_start = mean - (sd_multiplier * sd)
range_end = mean + (sd_multiplier * sd)
result = False
val_to_save = []
for x in kv:
k, v = get_kv(x)
_val = get_value_from_health_internal_tuple(v)
if not no_anomaly and (float(_val) < float(range_start)
or float(_val) > float(range_end)):
result |= True
val_to_save += create_value_list_to_save(save_param=None,
value=result,
op1=v)
val_to_save += create_value_list_to_save(save_param=save_param,
value=result)
res = create_health_internal_tuple(result, val_to_save)
except Exception:
res = create_health_internal_tuple(False, None)
return res
def vector_to_vector_diff_operation(kv, op, a, save_param):
"""
Passed Vector values
[ {(name, tag) : value}, {(name, tag) : value} ...
Return boolean dictionary result
{ (name, tag) : True/False , (name, tag) : True/False, ... }
"""
res = {}
temp_res = {}
if not kv or not a:
raise HealthException("Insufficient input for Diff operation ")
exception_found = False
try:
for x, y in itertools.combinations(kv, 2):
k1, v1 = get_kv(x)
k2, v2 = get_kv(y)
_v1 = get_value_from_health_internal_tuple(v1)
_v2 = get_value_from_health_internal_tuple(v2)
if op(abs(_v1 - _v2), a):
try:
temp_res[make_key(k1)] |= True
except Exception:
temp_res[make_key(k1)] = True
try:
temp_res[make_key(k2)] |= True
except Exception:
temp_res[make_key(k2)] = True
else:
try:
temp_res[make_key(k1)] |= False
except Exception:
temp_res[make_key(k1)] = False
try:
temp_res[make_key(k2)] |= False
except Exception:
temp_res[make_key(k2)] = False
for i in kv:
k, v = get_kv(i)
val_to_save = create_value_list_to_save(
save_param, value=temp_res[make_key(k)], op1=v)
res[make_key(k)] = create_health_internal_tuple(
temp_res[make_key(k)], val_to_save)
except Exception:
exception_found = True
if exception_found:
for x in kv:
k, v = get_kv(x)
res[make_key(k)] = create_health_internal_tuple(None, None)
return res
def p_assert_statement(self, p):
"""
assert_statement : ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg COMMA assert_if_condition RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string RPAREN
| ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL RPAREN
"""
if len(p) < 14:
p[0] = do_assert(op=p[1],
data=p[3],
check_val=p[5],
error=p[7],
category=p[9],
level=p[11])
elif len(p) < 16:
p[0] = do_assert(op=p[1],
data=p[3],
check_val=p[5],
error=p[7],
category=p[9],
level=p[11],
description=p[13])
elif len(p) < 18:
p[0] = do_assert(op=p[1],
data=p[3],
check_val=p[5],
error=p[7],
category=p[9],
level=p[11],
description=p[13],
success_msg=p[15])
else:
skip_assert, assert_filter_arg = p[17]
if skip_assert:
p[0] = None
else:
if assert_filter_arg is not None:
data = do_operation(op="==", arg1=p[3], arg2=p[5])
try:
# If key filtration throws exception (due to non-matching), it just passes that and executes main assert
new_data = do_operation(op="||",
arg1=data,
arg2=assert_filter_arg,
on_common_only=True)
if new_data:
data = new_data
except Exception:
pass
p[0] = do_assert(op=p[1],
data=data,
check_val=create_health_internal_tuple(
True, []),
error=p[7],
category=p[9],
level=p[11],
description=p[13],
success_msg=p[15])
else:
p[0] = do_assert(op=p[1],
data=p[3],
check_val=p[5],
error=p[7],
category=p[9],
level=p[11],
description=p[13],
success_msg=p[15])
def p_assert_comparison_arg(self, p):
"""
assert_comparison_arg : constant
"""
p[0] = create_health_internal_tuple(p[1], [])
def select_keys_from_dict(data={},
keys=[],
from_keys=[],
ignore_keys=[],
save_param=None,
config_param=False):
"""
Function takes dictionary, list of keys to fetch, list of from_keys to filter scope
Returns dictionary of selected keys and values
"""
if not data or not isinstance(data, dict):
raise HealthException("Wrong Input Data for select operation.")
result_dict = {}
if not keys:
raise HealthException("No key provided for select operation.")
for _key in data:
if from_keys:
f_key = from_keys[0]
if isinstance(_key, tuple):
# from_keys work with static component keys only, if we get
# tuple keys means we have done with checking of all component
# keys and not found any from key match so no need to check
# further in this direction
break
if (f_key == "ALL") or (_key == f_key):
# from_key is ALL or matching with _key
child_res = select_keys_from_dict(
data[_key],
keys=keys,
from_keys=from_keys[1:] if len(from_keys) > 1 else [],
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
else:
# no key match, need to check further
child_res = select_keys_from_dict(data[_key],
keys=keys,
from_keys=from_keys,
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
if child_res:
if f_key == "ALL":
# It assumes ALL is only for top snapshot level
result_dict[(_key, "SNAPSHOT")] = copy.deepcopy(child_res)
else:
result_dict = deep_merge_dicts(result_dict,
copy.deepcopy(child_res))
else:
# if (False, "*", None) in keys and isinstance(_key, tuple):
# result_dict[_key] = copy.deepcopy(data[_key])
if isinstance(_key, tuple) and _key[1] == "KEY":
for check_substring, s_key, new_name in keys:
if ((s_key == "*"
and not _is_key_in_ignore_keys(_key[0], ignore_keys))
or (check_substring and re.search(s_key, _key[0]))
or (not check_substring and _key[0] == s_key)):
val_to_save = create_value_list_to_save(
save_param=save_param,
key=_key[0],
value=data[_key],
formatting=not config_param)
if new_name:
result_dict[(
new_name,
"KEY")] = create_health_internal_tuple(
data[_key], val_to_save)
else:
result_dict[_key] = create_health_internal_tuple(
data[_key], val_to_save)
break
elif data[_key] and isinstance(data[_key], dict):
child_res = select_keys_from_dict(data[_key],
keys=keys,
ignore_keys=ignore_keys,
save_param=save_param,
config_param=config_param)
if child_res:
if isinstance(_key, tuple):
result_dict[_key] = copy.deepcopy(child_res)
else:
result_dict = deep_merge_dicts(
result_dict, copy.deepcopy(child_res))
return result_dict
def test_create_health_internal_tuple(self):
self.assertEqual(
util.create_health_internal_tuple(1, [('conf2', 100, True),
('conf1', 6.0, True)]),
(1, [('conf2', 100, True), ('conf1', 6.0, True)]),
"create_health_internal_tuple did not return the expected result")
