def _operate_each_key(self, arg1, arg2, save_param=None): if isinstance(arg1, dict) and isinstance(arg2, dict): return None if not isinstance(arg1, dict) and not isinstance(arg2, dict): try: raw_arg1 = get_value_from_health_internal_tuple(arg1) raw_arg2 = get_value_from_health_internal_tuple(arg2) if self.op == operator.div and raw_arg2 == 0: val_to_save = create_value_list_to_save(save_param, value=0, op1=arg1, op2=arg2) return (0, val_to_save) # if any of the arg is type float or operation is division # cast all argument to float if self.op == operator.div or isinstance( raw_arg1, float) or isinstance(raw_arg2, float): raw_arg1 = float(raw_arg1) raw_arg2 = float(raw_arg2) result = self.op(raw_arg1, raw_arg2) val_to_save = create_value_list_to_save(save_param, value=result, op1=arg1, op2=arg2) return create_health_internal_tuple(result, val_to_save) except Exception: return create_health_internal_tuple(None, []) dict_first = True if isinstance(arg1, dict): d = arg1 v = arg2 elif isinstance(arg2, dict): d = arg2 v = arg1 dict_first = False res_dict = {} for _k in d: if dict_first: res_dict[_k] = self._operate_each_key(d[_k], v, save_param=save_param) else: res_dict[_k] = self._operate_each_key(v, d[_k], save_param=save_param) return res_dict
def _operate_each_key(self, arg1, arg2, save_param=None): if isinstance(arg1, dict): return None try: raw_arg1 = get_value_from_health_internal_tuple(arg1) raw_arg2 = get_value_from_health_internal_tuple(arg2) result = self.op(raw_arg1, raw_arg2) val_to_save = create_value_list_to_save(save_param, value=result, op1=arg1, op2=arg2) return create_health_internal_tuple(result, val_to_save) except Exception: return create_health_internal_tuple(None, [])
def do_assert(op=None, data={}, check_val=create_health_internal_tuple(True,[]), error=None, category=None, level=None, description=None, success_msg=None): if op in assert_op_list: return assert_op_list[op](data, check_val, error, category, level, description, success_msg) return None
def op_fn_distributor(self, v, save_param): result = AggOperation.operator_and_function[self.op](v) val_to_save = create_value_list_to_save(save_param, value=result, op1=v) return create_health_internal_tuple(result, val_to_save)
def vector_to_vector_no_match_operation(kv, op, a, save_param): """ Passed Vector values [ {(name, tag) : value}, {(name, tag) : value} ... Return health internal tuple (True/False , [(key, value, formatting), (key, value, formatting), ...]) """ res = {} operand = get_value_from_health_internal_tuple(a) if not kv: raise HealthException("Insufficient input for NO_MATCH operation ") try: values = [ get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv ] match_operand = _find_match_operand_value(operand, values) result = False val_to_save = [] for x in kv: k, v = get_kv(x) _val = get_value_from_health_internal_tuple(v) if not op(_val, match_operand): result |= True val_to_save += create_value_list_to_save(save_param=None, value=result, op1=v) if operand and operand == MAJORITY: key = "Majority Value" else: key = "Expected Value" val_to_save += create_value_list_to_save(save_param=save_param, key=key, value=match_operand) res = create_health_internal_tuple(result, val_to_save) except Exception: res = create_health_internal_tuple(False, None) return res
def p_operand(self, p): """ operand : VAR | constant """ if is_health_parser_variable(p[1]): p[0] = p[1][2] else: p[0] = create_health_internal_tuple(p[1], [])
def p_opt_simple_operation_param(self, p): """ opt_simple_operation_param : COMMA constant | """ if len(p) == 1: p[0] = None else: p[0] = create_health_internal_tuple(p[2], [])
def add_prefix_to_saved_keys(prefix, data): if not prefix or not data or not data[1]: return data new_saved_value_list = [] for i in data[1]: _k = prefix if i[0] and len(i[0].strip()) > 0: _k += "/%s" % (i[0]) new_saved_value_list.append((_k, i[1], i[2])) return create_health_internal_tuple(data[0], new_saved_value_list)
def operate(self, data={}, check_val=create_health_internal_tuple(True, []), error=None, category=None, level=None, description=None, success_msg=None): if not data: raise HealthException("Wrong Input Data for ASSERT operation.") res = {} res[AssertResultKey.FAIL_MSG] = str(error) res[AssertResultKey.DESCRIPTION] = description res[AssertResultKey.SUCCESS_MSG] = success_msg res[AssertResultKey.KEYS] = [] try: res[AssertResultKey.CATEGORY] = category.upper().split(".") except Exception: res[AssertResultKey.CATEGORY] = None res[AssertResultKey.LEVEL] = level if not isinstance(data, dict): if not self.op(get_value_from_health_internal_tuple(data), get_value_from_health_internal_tuple(check_val)): return (ParserResultType.ASSERT, res) return None kv = find_kv_vector(NOKEY, data, recurse=True, update_saved_list=False) if not kv: return (ParserResultType.ASSERT, res) fail = False for i in kv: k, v = get_kv(i) kv_tuple = (k, None) value_to_check = get_value_from_health_internal_tuple(v) if v[1]: kv_tuple = (k, v[1]) if not self.op(value_to_check, get_value_from_health_internal_tuple(check_val)): res[AssertResultKey.SUCCESS] = False fail = True res[AssertResultKey.KEYS].append(kv_tuple) if not fail: res[AssertResultKey.SUCCESS] = True return (ParserResultType.ASSERT, res)
def p_complex_comparison_operand(self, p): """ complex_comparison_operand : COMPLEX_PARAM | operand """ if is_health_parser_variable(p[1]): p[0] = p[1][2] elif not isinstance(p[1], tuple): p[0] = create_health_internal_tuple(p[1], []) else: p[0] = p[1]
def _operate_dicts(self, arg1, arg2, comp_op=None, check_common=True, save_param=None): if isinstance(arg1, dict): if not isinstance(arg2, dict): check_common = False if check_common: k1_set = set(arg1.keys()) k2_set = set(arg2.keys()) if not list(k1_set.intersection(k2_set)): check_common = False result_dict = {} for _k in arg1: if check_common: if _k in arg2: result_dict[_k] = self._operate_dicts( arg1[_k], arg2[_k], comp_op=comp_op, check_common=check_common, save_param=save_param) else: result_dict[_k] = self._operate_dicts( arg1[_k], arg2, comp_op=comp_op, check_common=check_common, save_param=save_param) return result_dict else: result = self._operate_each_key(arg1, arg2, comp_op=comp_op) val_to_save = create_value_list_to_save(save_param, value=result, op1=arg1) return create_health_internal_tuple(result, val_to_save)
def p_assert_statement(self, p): """ assert_statement : ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg COMMA assert_if_condition RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL RPAREN """ if len(p) < 14: p[0] = do_assert( op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11]) elif len(p) < 16: p[0] = do_assert( op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13]) elif len(p) < 18: p[0] = do_assert( op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15]) else: skip_assert, assert_filter_arg = p[17] if skip_assert: p[0] = None else: if assert_filter_arg is not None: data = do_operation(op="==", arg1=p[3], arg2=p[5]) try: # If key filtration throws exception (due to non-matching), it just passes that and executes main assert new_data = do_operation(op="||", arg1=data, arg2=assert_filter_arg, on_common_only=True) if new_data: data = new_data except Exception: pass p[0] = do_assert( op=p[1], data=data, check_val=create_health_internal_tuple(True,[]), error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15]) else: p[0] = do_assert( op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15])
def p_assert_comparison_arg(self, p): """ assert_comparison_arg : constant """ p[0] = create_health_internal_tuple(p[1], [])
def vector_to_vector_sd_anomaly_operation(kv, op, a, save_param): """ Passed Vector values [ {(name, tag) : value}, {(name, tag) : value} ... Return health internal tuple (True/False , [(key, value, formatting), (key, value, formatting), ...]) """ res = {} sd_multiplier = get_value_from_health_internal_tuple(a) if not kv or not sd_multiplier: raise HealthException("Insufficient input for SD_ANOMALY operation ") try: n = len(kv) if n < 3: no_anomaly = True range_start = 0 range_end = 0 else: values = [ get_value_from_health_internal_tuple(get_kv(m)[1]) for m in kv ] no_anomaly = False try: # We should consider int and floats only s = sum(values) except Exception: no_anomaly = True if not no_anomaly: mean = float(s) / float(n) variance = 0 for v in values: variance += pow((v - mean), 2) variance = float(variance) / float(n) sd = sqrt(variance) range_start = mean - (sd_multiplier * sd) range_end = mean + (sd_multiplier * sd) result = False val_to_save = [] for x in kv: k, v = get_kv(x) _val = get_value_from_health_internal_tuple(v) if not no_anomaly and (float(_val) < float(range_start) or float(_val) > float(range_end)): result |= True val_to_save += create_value_list_to_save(save_param=None, value=result, op1=v) val_to_save += create_value_list_to_save(save_param=save_param, value=result) res = create_health_internal_tuple(result, val_to_save) except Exception: res = create_health_internal_tuple(False, None) return res
def vector_to_vector_diff_operation(kv, op, a, save_param): """ Passed Vector values [ {(name, tag) : value}, {(name, tag) : value} ... Return boolean dictionary result { (name, tag) : True/False , (name, tag) : True/False, ... } """ res = {} temp_res = {} if not kv or not a: raise HealthException("Insufficient input for Diff operation ") exception_found = False try: for x, y in itertools.combinations(kv, 2): k1, v1 = get_kv(x) k2, v2 = get_kv(y) _v1 = get_value_from_health_internal_tuple(v1) _v2 = get_value_from_health_internal_tuple(v2) if op(abs(_v1 - _v2), a): try: temp_res[make_key(k1)] |= True except Exception: temp_res[make_key(k1)] = True try: temp_res[make_key(k2)] |= True except Exception: temp_res[make_key(k2)] = True else: try: temp_res[make_key(k1)] |= False except Exception: temp_res[make_key(k1)] = False try: temp_res[make_key(k2)] |= False except Exception: temp_res[make_key(k2)] = False for i in kv: k, v = get_kv(i) val_to_save = create_value_list_to_save( save_param, value=temp_res[make_key(k)], op1=v) res[make_key(k)] = create_health_internal_tuple( temp_res[make_key(k)], val_to_save) except Exception: exception_found = True if exception_found: for x in kv: k, v = get_kv(x) res[make_key(k)] = create_health_internal_tuple(None, None) return res
def p_assert_statement(self, p): """ assert_statement : ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg COMMA assert_if_condition RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string COMMA assert_success_msg RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL COMMA assert_desc_string RPAREN | ASSERT_OP LPAREN assert_arg COMMA assert_comparison_arg COMMA error_string COMMA assert_category COMMA ASSERT_LEVEL RPAREN """ if len(p) < 14: p[0] = do_assert(op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11]) elif len(p) < 16: p[0] = do_assert(op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13]) elif len(p) < 18: p[0] = do_assert(op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15]) else: skip_assert, assert_filter_arg = p[17] if skip_assert: p[0] = None else: if assert_filter_arg is not None: data = do_operation(op="==", arg1=p[3], arg2=p[5]) try: # If key filtration throws exception (due to non-matching), it just passes that and executes main assert new_data = do_operation(op="||", arg1=data, arg2=assert_filter_arg, on_common_only=True) if new_data: data = new_data except Exception: pass p[0] = do_assert(op=p[1], data=data, check_val=create_health_internal_tuple( True, []), error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15]) else: p[0] = do_assert(op=p[1], data=p[3], check_val=p[5], error=p[7], category=p[9], level=p[11], description=p[13], success_msg=p[15])
def select_keys_from_dict(data={}, keys=[], from_keys=[], ignore_keys=[], save_param=None, config_param=False): """ Function takes dictionary, list of keys to fetch, list of from_keys to filter scope Returns dictionary of selected keys and values """ if not data or not isinstance(data, dict): raise HealthException("Wrong Input Data for select operation.") result_dict = {} if not keys: raise HealthException("No key provided for select operation.") for _key in data: if from_keys: f_key = from_keys[0] if isinstance(_key, tuple): # from_keys work with static component keys only, if we get # tuple keys means we have done with checking of all component # keys and not found any from key match so no need to check # further in this direction break if (f_key == "ALL") or (_key == f_key): # from_key is ALL or matching with _key child_res = select_keys_from_dict( data[_key], keys=keys, from_keys=from_keys[1:] if len(from_keys) > 1 else [], ignore_keys=ignore_keys, save_param=save_param, config_param=config_param) else: # no key match, need to check further child_res = select_keys_from_dict(data[_key], keys=keys, from_keys=from_keys, ignore_keys=ignore_keys, save_param=save_param, config_param=config_param) if child_res: if f_key == "ALL": # It assumes ALL is only for top snapshot level result_dict[(_key, "SNAPSHOT")] = copy.deepcopy(child_res) else: result_dict = deep_merge_dicts(result_dict, copy.deepcopy(child_res)) else: # if (False, "*", None) in keys and isinstance(_key, tuple): # result_dict[_key] = copy.deepcopy(data[_key]) if isinstance(_key, tuple) and _key[1] == "KEY": for check_substring, s_key, new_name in keys: if ((s_key == "*" and not _is_key_in_ignore_keys(_key[0], ignore_keys)) or (check_substring and re.search(s_key, _key[0])) or (not check_substring and _key[0] == s_key)): val_to_save = create_value_list_to_save( save_param=save_param, key=_key[0], value=data[_key], formatting=not config_param) if new_name: result_dict[( new_name, "KEY")] = create_health_internal_tuple( data[_key], val_to_save) else: result_dict[_key] = create_health_internal_tuple( data[_key], val_to_save) break elif data[_key] and isinstance(data[_key], dict): child_res = select_keys_from_dict(data[_key], keys=keys, ignore_keys=ignore_keys, save_param=save_param, config_param=config_param) if child_res: if isinstance(_key, tuple): result_dict[_key] = copy.deepcopy(child_res) else: result_dict = deep_merge_dicts( result_dict, copy.deepcopy(child_res)) return result_dict
def test_create_health_internal_tuple(self): self.assertEqual( util.create_health_internal_tuple(1, [('conf2', 100, True), ('conf1', 6.0, True)]), (1, [('conf2', 100, True), ('conf1', 6.0, True)]), "create_health_internal_tuple did not return the expected result")