def test_to_obj():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject, 'load_definition',
return_value=maps.NamedDict()):
alert_json = {"alert_id": "Test_id",
"node_id": "Test_id",
"time_stamp": "Test_time",
"resource": "Test_resource",
"current_value": "Test_value",
"tags": {},
"alert_type": "Test_type",
"severity": "INFO",
"classification": "Test_clas",
"significance": "Test_sig",
"ackedby": "Test_ackedby",
"acked": "Test_acked",
"ack_comment": [],
"acked_at": "Test_acked_at",
"pid": 0,
"source": "Test_source",
"delivered": "Test_delivered"
}
alert_ex = AlertUtils()
alert_ex.to_obj(alert_json)
def test_to_obj():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject,
'load_definition',
return_value=maps.NamedDict()):
alert_json = {
"alert_id": "Test_id",
"node_id": "Test_id",
"time_stamp": "Test_time",
"resource": "Test_resource",
"current_value": "Test_value",
"tags": {},
"alert_type": "Test_type",
"severity": "INFO",
"classification": "Test_clas",
"significance": "Test_sig",
"ackedby": "Test_ackedby",
"acked": "Test_acked",
"ack_comment": [],
"acked_at": "Test_acked_at",
"pid": 0,
"source": "Test_source",
"delivered": "Test_delivered"
}
alert_ex = AlertUtils()
alert_ex.to_obj(alert_json)
def test_equals():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject, 'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert2 = Alert("Test 2")
alert_ex = AlertUtils()
alert_ex.equals(alert1, alert2)
def test_equals():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject,
'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert2 = Alert("Test 2")
alert_ex = AlertUtils()
alert_ex.equals(alert1, alert2)
def test_update():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject, 'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert1.severity = "INFO"
alert2 = Alert("Test 2")
alert2.severity = "WARNING"
alert_ex = AlertUtils()
alert_ex.update(alert1, alert2)
alert1.severity = "CRITICAL"
# Makes first if statement false
alert_ex.update(alert1, alert2)
def test_update():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject,
'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert1.severity = "INFO"
alert2 = Alert("Test 2")
alert2.severity = "WARNING"
alert_ex = AlertUtils()
alert_ex.update(alert1, alert2)
alert1.severity = "CRITICAL"
# Makes first if statement false
alert_ex.update(alert1, alert2)
def test_is_same():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject,
'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert2 = Alert("Test 2")
alert1.resource = "Test Resource1"
alert2.resource = "Test Resource2"
alert_ex = AlertUtils()
# Covers first false statement where resources are different
assert alert_ex.is_same(alert1, alert2) is False
alert2.resource = "Test Resource1"
alert1.tags = {"No plug": "Hi"}
alert1.classification = ["node"]
alert1.node_id = "node1"
alert2.classification = ["node"]
alert2.node_id = "node2"
# Makes second if false, not plugin in alert1
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi"}
alert2.tags = {'plugin': "Ho"}
# Covers if plugin is in alert1 but not alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Ho"}
# Covers if both have the plugin_instance but their data is different
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Hi", 'integration_id': 2}
alert1.tags = {'plugin_instance': "Hi", 'integration_id': 1}
# Covers if they both have the same plugin_instance
# but different int ids
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Hi"}
# Alert2 now doesn't have an int id
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi"}
alert2.tags = {'plugin_instance': "Hi", 'integration_id': 2}
# Now alert2 has an int id but alert1 does not
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi", 'integration_id': 2}
alert1.classification = ["node"]
alert1.node_id = "node1"
alert2.classification = ["node"]
alert2.node_id = "node2"
# Node in both classifications but different ids
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification = ["no_longer_the_same"]
# alert1 clas has node, but alert2 does not
assert alert_ex.is_same(alert1, alert2) is False
alert1.classification = ["no_longer_the_same"]
alert2.classification = ["fail"]
# Alert1 clas does not have node, and neither does Alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification = ["node"]
# Alert1 clas does not have node, but alert2 does
assert alert_ex.is_same(alert1, alert2) is False
alert1.classification = ["node"]
alert1.node_id = "node"
alert2.classification = ["node"]
alert2.node_id = "node"
alert1.alert_type = "Type1"
alert2.alert_type = "Type2"
# Alert types are not equal
assert alert_ex.is_same(alert1, alert2) is False
alert2.alert_type = "Type1"
alert2.classification = ["node", "extra"]
# The sets are not equal due to adding an extra element to alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification.remove("extra")
assert alert_ex.is_same(alert1, alert2) is True
def update_alert(message):
try:
lock = None
existing_alert = False
new_alert = json.loads(message.payload["message"])
new_alert['alert_id'] = message.message_id
new_alert_obj = AlertUtils().to_obj(new_alert)
if new_alert_obj.alert_type not in constants.SUPPORTED_ALERT_TYPES:
logger.log(
"error", NS.publisher_id,
{"message": "Invalid alert type in alert %s" % new_alert})
raise InvalidAlertType
if new_alert_obj.severity not in constants.SUPPORTED_ALERT_SEVERITY:
logger.log(
"error", NS.publisher_id,
{"message": "Invalid alert severity in alert %s" % new_alert})
raise InvalidAlertSeverity
alert_notify = message.payload.get('alert_notify', False)
if not alert_notify:
if (new_alert_obj.resource
in NS.tendrl.objects.ClusterAlert()._defs['relationship'][
new_alert_obj.alert_type.lower()]):
new_alert_obj.classification.append(constants.CLUSTER_ALERT)
if (new_alert_obj.resource
in NS.tendrl.objects.NodeAlert()._defs['relationship'][
new_alert_obj.alert_type.lower()]):
new_alert_obj.classification.append(constants.NODE_ALERT)
alerts = utils.get_alerts(new_alert_obj)
for curr_alert in alerts:
curr_alert.tags = json.loads(curr_alert.tags)
if AlertUtils().is_same(new_alert_obj, curr_alert):
new_alert_obj = AlertUtils().update(
new_alert_obj, curr_alert)
if not AlertUtils().equals(new_alert_obj, curr_alert):
# Lock only if new alert matches with existing alert
lock = Lock(
NS._int.wclient,
'alerting/alerts/%s' % new_alert_obj.alert_id)
lock.acquire(blocking=True, lock_ttl=60)
if lock.is_acquired:
# renew a lock
lock.acquire(lock_ttl=60)
existing_alert = True
utils.update_alert_count(new_alert_obj, existing_alert)
if message.payload["alert_condition_unset"]:
keep_alive = int(
NS.config.data["alert_retention_time"])
utils.classify_alert(new_alert_obj, keep_alive)
new_alert_obj.save(ttl=keep_alive)
else:
# Remove the clearing alert with same if exist
utils.remove_alert(new_alert_obj)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
return
else:
# Handle case where alert severity changes without
# coming to normal. In this case the previous alert
# should be overriden with new one
utils.remove_alert(new_alert_obj)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
return
# else add this new alert to etcd
if message.payload["alert_condition_state"] == \
constants.ALERT_SEVERITY["warning"]:
utils.update_alert_count(new_alert_obj, existing_alert)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
else:
logger.log(
"error", NS.publisher_id, {
"message":
"New alert can't be a clearing alert %s" % (new_alert)
})
else:
# SDS native events
utils.save_notification_only_alert(new_alert_obj)
except (AttributeError, TypeError, ValueError, KeyError, InvalidAlertType,
InvalidAlertSeverity, EtcdKeyNotFound, EtcdException) as ex:
logger.log(
"error", NS.publisher_id,
{"message": "Error %s in updating alert %s" % (ex, new_alert)})
finally:
if isinstance(lock, Lock) and lock.is_acquired:
lock.release()
def update_alert(message):
try:
lock = None
new_alert = json.loads(message.payload["message"])
new_alert['alert_id'] = message.message_id
new_alert_obj = AlertUtils().to_obj(new_alert)
if new_alert_obj.alert_type not in constants.SUPPORTED_ALERT_TYPES:
logger.log(
"error", NS.publisher_id,
{"message": "Invalid alert type in alert %s" % new_alert})
raise InvalidAlertType
if new_alert_obj.severity not in constants.SUPPORTED_ALERT_SEVERITY:
logger.log(
"error", NS.publisher_id,
{"message": "Invalid alert severity in alert %s" % new_alert})
raise InvalidAlertSeverity
alert_notify = message.payload.get('alert_notify', False)
if not alert_notify:
if (new_alert_obj.resource
in NS.tendrl.objects.ClusterAlert()._defs['relationship'][
new_alert_obj.alert_type.lower()]):
new_alert_obj.classification.append(constants.CLUSTER_ALERT)
if (new_alert_obj.resource
in NS.tendrl.objects.NodeAlert()._defs['relationship'][
new_alert_obj.alert_type.lower()]):
new_alert_obj.classification.append(constants.NODE_ALERT)
alerts = utils.get_alerts(new_alert_obj)
for curr_alert in alerts:
curr_alert.tags = json.loads(curr_alert.tags)
if AlertUtils().is_same(new_alert_obj, curr_alert):
if new_alert_obj.severity == \
constants.ALERT_SEVERITY["info"]:
if "clear_alert" in new_alert_obj.tags.keys():
if new_alert_obj.tags['clear_alert'] != \
curr_alert.severity:
# only warning clearing alert can clear
# the warning alert and critical clearing alert
# can clear the critical alert,
# Because critical/warning alert panels in
# grafana are indipendent from one another,
# So after critical alert raised if warning
# clearing came then tendrl can show only
# clearing alert, So this logic will help
# to prevent from the above case.
return
new_alert_obj = AlertUtils().update(
new_alert_obj, curr_alert)
if not AlertUtils().equals(new_alert_obj, curr_alert):
# Lock only if new alert matches with existing alert
lock = Lock(
NS._int.wclient,
'alerting/alerts/%s' % new_alert_obj.alert_id)
lock.acquire(blocking=True, lock_ttl=60)
if lock.is_acquired:
# renew a lock
lock.acquire(lock_ttl=60)
utils.update_alert_count(new_alert_obj, curr_alert)
if message.payload["alert_condition_unset"]:
keep_alive = int(
NS.config.data["alert_retention_time"])
utils.classify_alert(new_alert_obj, keep_alive)
new_alert_obj.save(ttl=keep_alive)
else:
# Remove the clearing alert with same if exist
utils.remove_alert(new_alert_obj)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
return
else:
# If alert raised again with same severity,
# then update the alert
utils.remove_alert(new_alert_obj)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
# message_id and alert_id is same
# When same alert raised multiple times
# then assign old alert_id to new message
# to avoid duplicates events
# here new alert object already have old_message_id
message.message_id = new_alert_obj.alert_id
return
# else add this new alert to etcd
severity = message.payload["alert_condition_state"]
if(severity == constants.ALERT_SEVERITY["warning"]) or \
(severity == constants.ALERT_SEVERITY["critical"]):
utils.update_alert_count(new_alert_obj)
utils.classify_alert(new_alert_obj)
new_alert_obj.save()
else:
logger.log(
"debug", NS.publisher_id, {
"message":
"New alert can't be a clearing alert %s" % (new_alert)
})
else:
# SDS native events
utils.save_notification_only_alert(new_alert_obj)
except (AttributeError, TypeError, ValueError, KeyError, InvalidAlertType,
InvalidAlertSeverity, EtcdKeyNotFound, EtcdException) as ex:
logger.log(
"error", NS.publisher_id,
{"message": "Error %s in updating alert %s" % (ex, new_alert)})
finally:
if isinstance(lock, Lock) and lock.is_acquired:
lock.release()
def test_is_same():
setattr(__builtin__, "NS", maps.NamedDict())
setattr(NS, "_int", maps.NamedDict())
NS.publisher_id = "Test_id"
with patch.object(objects.BaseObject, 'load_definition',
return_value=maps.NamedDict()):
alert1 = Alert()
alert2 = Alert("Test 2")
alert1.resource = "Test Resource1"
alert2.resource = "Test Resource2"
alert_ex = AlertUtils()
# Covers first false statement where resources are different
assert alert_ex.is_same(alert1, alert2) is False
alert2.resource = "Test Resource1"
alert1.tags = {"No plug": "Hi"}
alert1.classification = ["node"]
alert1.node_id = "node1"
alert2.classification = ["node"]
alert2.node_id = "node2"
# Makes second if false, not plugin in alert1
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi"}
alert2.tags = {'plugin': "Ho"}
# Covers if plugin is in alert1 but not alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Ho"}
# Covers if both have the plugin_instance but their data is different
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Hi",
'integration_id': 2}
alert1.tags = {'plugin_instance': "Hi",
'integration_id': 1}
# Covers if they both have the same plugin_instance
# but different int ids
assert alert_ex.is_same(alert1, alert2) is False
alert2.tags = {'plugin_instance': "Hi"}
# Alert2 now doesn't have an int id
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi"}
alert2.tags = {'plugin_instance': "Hi",
'integration_id': 2}
# Now alert2 has an int id but alert1 does not
assert alert_ex.is_same(alert1, alert2) is False
alert1.tags = {'plugin_instance': "Hi",
'integration_id': 2}
alert1.classification = ["node"]
alert1.node_id = "node1"
alert2.classification = ["node"]
alert2.node_id = "node2"
# Node in both classifications but different ids
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification = ["no_longer_the_same"]
# alert1 clas has node, but alert2 does not
assert alert_ex.is_same(alert1, alert2) is False
alert1.classification = ["no_longer_the_same"]
alert2.classification = ["fail"]
# Alert1 clas does not have node, and neither does Alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification = ["node"]
# Alert1 clas does not have node, but alert2 does
assert alert_ex.is_same(alert1, alert2) is False
alert1.classification = ["node"]
alert1.node_id = "node"
alert2.classification = ["node"]
alert2.node_id = "node"
alert1.alert_type = "Type1"
alert2.alert_type = "Type2"
# Alert types are not equal
assert alert_ex.is_same(alert1, alert2) is False
alert2.alert_type = "Type1"
alert2.classification = ["node", "extra"]
# The sets are not equal due to adding an extra element to alert2
assert alert_ex.is_same(alert1, alert2) is False
alert2.classification.remove("extra")
assert alert_ex.is_same(alert1, alert2) is True