def alert_generator():
""" Generates alerts for a given site """
mails = []
sms_numbers = []
rules = Alert_Rule.objects.all()
for rule in rules:
site = rule.site
site_groups = get_groups_with_perms(site)
# Get datapoint and real value
data_point, real_value = get_alert_check_value(rule)
if data_point is not None and real_value is not None:
if check_alert(rule, real_value):
alert_obj = alert_factory(site, rule, data_point)
# if alert_obj is created
if alert_obj is not None:
content = get_alert_content(site, rule, data_point,
real_value, alert_obj)
mails, sms_numbers = get_recipients_for_site(site)
# reporting
logging.debug("Alert triggered sending alerts out %s" %
mails)
alert_obj.emailSent = send_mail("Alert Mail", mails,
content)
alert_obj.smsSent = send_sms(sms_numbers, content)
slack_msg = get_slack_alert_msg("Alert Triggered",
alert_obj)
alert_obj.slackSent = send_alert_slack(
site_groups, slack_msg)
alert_obj.save()
def alert_generator():
""" Generates alerts for a given site """
mails = []
sms_numbers = []
rules = Alert_Rule.objects.all()
for rule in rules:
site = rule.site
site_groups = get_groups_with_perms(site)
# Get datapoint and real value
data_point, real_value = get_alert_check_value(rule)
if data_point is not None and real_value is not None:
if check_alert(rule, real_value):
alert_obj = alert_factory(site, rule, data_point)
# if alert_obj is created
if alert_obj is not None:
content = get_alert_content(site, rule, data_point, real_value, alert_obj)
mails, sms_numbers = get_recipients_for_site(site)
# reporting
alert_obj.emailSent = send_mail("Alert Mail", mails, content)
alert_obj.smsSent = send_sms(sms_numbers, content)
slack_msg = get_slack_alert_msg("Alert Triggered", alert_obj)
alert_obj.slackSent = send_alert_slack(site_groups, slack_msg)
alert_obj.save()
def alert_status_check():
"""
Checks if the alert is still valid and silences it if it is invalid
"""
unsilenced_alerts = get_unsilenced_alerts()
logger.debug("Running alert status check")
for alert in unsilenced_alerts:
site = alert.site
rule = alert.alert
if is_mysql_rule(rule):
latest_data_point_value = get_latest_data_point_value_mysql(
site, rule)
elif is_influx_rule(rule):
latest_data_point_value = get_latest_point_value_influx(site, rule)
else:
raise Exception("Invalid alert Rule")
if check_alert(rule, latest_data_point_value):
logger.debug("Alert is still valid")
else:
# Silencing the alert and generating email content
logger.debug("Alert is not valid, silencing alert")
alert.isSilence = True
alert.save()
data_point, data_point_value = get_alert_check_value(alert.alert)
# Handle no data point getting returned
if not data_point_value:
logger.warning("Now DP found for alert skipping ")
return None
content = get_alert_content(site, rule, data_point,
data_point_value, alert)
mails, sms_numbers = get_recipients_for_site(site)
site_groups = get_groups_with_perms(site)
# Reporting
if mails:
send_mail('Alert Silenced', mails, content)
if sms_numbers:
send_sms(content, sms_numbers)
slack_msg = get_slack_alert_msg("Alert silenced", alert)
send_alert_slack(site_groups, slack_msg)
def alert_status_check():
"""
Checks if the alert is still valid and silences it if it is invalid
"""
unsilenced_alerts = get_unsilenced_alerts()
logger.debug("Running alert status check")
for alert in unsilenced_alerts:
site = alert.site
rule = alert.alert
if is_mysql_rule(rule):
latest_data_point_value = get_latest_data_point_value_mysql(site, rule)
elif is_influx_rule(rule):
latest_data_point_value = get_latest_point_value_influx(site, rule)
else:
raise Exception("Invalid alert Rule")
if check_alert(rule, latest_data_point_value):
logger.debug("Alert is still valid")
else:
# Silencing the alert and generating email content
logger.debug("Alert is not valid, silencing alert")
alert.isSilence = True
alert.save()
data_point, data_point_value = get_alert_check_value(alert.alert)
# Handle no data point getting returned
if not data_point_value:
logger.warning("Now DP found for alert skipping ")
return None
content = get_alert_content(site, rule, data_point, data_point_value, alert)
mails, sms_numbers = get_recipients_for_site(site)
site_groups = get_groups_with_perms(site)
# Reporting
send_mail('Alert Silenced', content, mails)
send_sms(content, sms_numbers)
slack_msg = get_slack_alert_msg("Alert silenced", alert)
send_alert_slack(site_groups, slack_msg)