def send_notify(alertid):
global tokens, hold
try:
for tag in alert[alertid]["tags"]:
if tag.startswith("sms:") or tag.startswith("email:"):
who = who_to_notify(tag)
message = alert[alertid]["summary"]
if tag.startswith("sms:") and tokens[who, "sms"] > 0:
_Lock.acquire()
tokens[who, "sms"] -= 1
_Lock.release()
logging.debug("Taken a sms token from %s, there are only %d left", who, tokens[who, "sms"])
sms_notify(alertid, USERNAME, PASSWORD, owners[who]["mobile"])
elif tokens[who, "sms"] == 0:
logging.error(
"%s run out of sms tokens. Failed to notify %s.", who, alert[alertid]["lastReceiveId"]
)
if tag.startswith("email:") and tokens[who, "email"] > 0:
_Lock.acquire()
tokens[who, "email"] -= 1
_Lock.release()
logging.debug("Taken a email token from %s, there are only %d left", who, tokens[who, "sms"])
email_notify(alertid, owners[who]["email"])
elif tokens[who, "email"] == 0:
logging.error(
"%s run out of email tokens. Failed to notify %s.", who, alert[alertid]["lastReceiveId"]
)
except Exception, e:
logging.error('Notify sending failed for "%s" - %s - %s', alert[alertid]["lastReceiveId"], message, e)
pass
def on_message(self, headers, body):
global alert, hold
logging.debug("Received alert : %s", body)
alertid = json.loads(body)["id"]
alert[alertid] = json.loads(body)
logging.info(
"%s : [%s] %s", alert[alertid]["lastReceiveId"], alert[alertid]["status"], alert[alertid]["summary"]
)
if not should_we_notify(alertid):
logging.debug(
"%s : NOT PAGING for [%s] %s",
alert[alertid]["lastReceiveId"],
alert[alertid]["status"],
alert[alertid]["summary"],
)
del alert[alertid]
return
if alertid in hold:
if alert[alertid]["severity"] == "NORMAL":
logging.info("%s : Dropping NORMAL alert %s", alert[alertid]["lastReceiveId"], alertid)
del hold[alertid]
del alert[alertid]
else:
logging.info("%s : Update alert %s details", alert[alertid]["lastReceiveId"], alertid)
else:
hold[alertid] = time.time() + GRACEPERIOD
logging.info(
"%s : Holding onto alert %s for %s seconds", alert[alertid]["lastReceiveId"], alertid, GRACEPERIOD
)
def email_notify(alertid, email):
MAILING_LIST = email
createTime = datetime.datetime.strptime(alert[alertid]["createTime"], "%Y-%m-%dT%H:%M:%S.%fZ")
createTime = createTime.replace(tzinfo=pytz.utc)
tz = pytz.timezone(TIMEZONE)
localTime = createTime.astimezone(tz)
text = ""
text += "[%s] %s\n" % (alert[alertid]["status"], alert[alertid]["summary"])
text += "Alert Details\n"
text += "Alert ID: %s\n" % (alert[alertid]["id"])
text += "Create Time: %s\n" % (localTime.strftime("%Y/%m/%d %H:%M:%S"))
text += "Resource: %s\n" % (alert[alertid]["resource"])
text += "Environment: %s\n" % (",".join(alert[alertid]["environment"]))
text += "Service: %s\n" % (",".join(alert[alertid]["service"]))
text += "Event Name: %s\n" % (alert[alertid]["event"])
text += "Event Group: %s\n" % (alert[alertid]["group"])
text += "Event Value: %s\n" % (alert[alertid]["value"])
text += "Severity: %s -> %s\n" % (alert[alertid]["previousSeverity"], alert[alertid]["severity"])
text += "Status: %s\n" % (alert[alertid]["status"])
text += "Text: %s\n" % (alert[alertid]["text"])
if "thresholdInfo" in alert[alertid]:
text += "Threshold Info: %s\n" % (alert[alertid]["thresholdInfo"])
if "duplicateCount" in alert[alertid]:
text += "Duplicate Count: %s\n" % (alert[alertid]["duplicateCount"])
if "moreInfo" in alert[alertid]:
text += "More Info: %s\n" % (alert[alertid]["moreInfo"])
text += "Historical Data\n"
if "graphs" in alert[alertid]:
for g in alert[alertid]["graphs"]:
text += "%s\n" % (g)
text += "Raw Alert\n"
text += "%s\n" % (json.dumps(alert[alertid]))
text += "Generated by %s on %s at %s\n" % (
"alert-notify.py",
os.uname()[1],
datetime.datetime.now().strftime("%a %d %b %H:%M:%S"),
)
logging.debug("Raw Text: %s", text)
html = '<p><table border="0" cellpadding="0" cellspacing="0" width="100%">\n' # table used to center email
html += '<tr><td bgcolor="#ffffff" align="center">\n'
html += '<table border="0" cellpadding="0" cellspacing="0" width="700">\n' # table used to set width of email
html += (
'<tr><td bgcolor="#425470"><p align="center" style="font-size:24px;color:#d9fffd;font-weight:bold;"><strong>[%s] %s</strong></p>\n'
% (alert[alertid]["status"], alert[alertid]["summary"])
)
html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Alert Details</p>\n'
html += "<table>\n"
html += (
'<tr><td><b>Alert ID:</b></td><td><a href="%s/alerta/details.php?id=%s" target="_blank">%s</a></td></tr>\n'
% (API_SERVER, alert[alertid]["id"], alert[alertid]["id"])
)
html += "<tr><td><b>Create Time:</b></td><td>%s</td></tr>\n" % (localTime.strftime("%Y/%m/%d %H:%M:%S"))
html += "<tr><td><b>Resource:</b></td><td>%s</td></tr>\n" % (alert[alertid]["resource"])
html += "<tr><td><b>Environment:</b></td><td>%s</td></tr>\n" % (",".join(alert[alertid]["environment"]))
html += "<tr><td><b>Service:</b></td><td>%s</td></tr>\n" % (",".join(alert[alertid]["service"]))
html += "<tr><td><b>Event Name:</b></td><td>%s</td></tr>\n" % (alert[alertid]["event"])
html += "<tr><td><b>Event Group:</b></td><td>%s</td></tr>\n" % (alert[alertid]["group"])
html += "<tr><td><b>Event Value:</b></td><td>%s</td></tr>\n" % (alert[alertid]["value"])
html += "<tr><td><b>Severity:</b></td><td>%s -> %s</td></tr>\n" % (
alert[alertid]["previousSeverity"],
alert[alertid]["severity"],
)
html += "<tr><td><b>Status:</b></td><td>%s</td></tr>\n" % (alert[alertid]["status"])
html += "<tr><td><b>Text:</b></td><td>%s</td></tr>\n" % (alert[alertid]["text"])
if "thresholdInfo" in alert[alertid]:
html += "<tr><td><b>Threshold Info:</b></td><td>%s</td></tr>\n" % (alert[alertid]["thresholdInfo"])
if "duplicateCount" in alert[alertid]:
html += "<tr><td><b>Duplicate Count:</b></td><td>%s</td></tr>\n" % (alert[alertid]["duplicateCount"])
if "moreInfo" in alert[alertid]:
html += '<tr><td><b>More Info:</b></td><td><a href="%s">ganglia</a></td></tr>\n' % (alert[alertid]["moreInfo"])
html += "</table>\n"
html += "</td></tr>\n"
html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Historical Data</p>\n'
if "graphs" in alert[alertid]:
graph_cid = dict()
for g in alert[alertid]["graphs"]:
graph_cid[g] = str(uuid.uuid4())
html += '<tr><td><img src="cid:' + graph_cid[g] + '"></td></tr>\n'
html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Raw Alert</p>\n'
html += '<tr><td><p align="left" style="font-family: \'Courier New\', Courier, monospace">%s</p></td></tr>\n' % (
json.dumps(alert[alertid])
)
html += "<tr><td>Generated by %s on %s at %s</td></tr>\n" % (
"alert-mailer.py",
os.uname()[1],
datetime.datetime.now().strftime("%a %d %b %H:%M:%S"),
)
html += "</table>"
html += "</td></tr></table>"
html += "</td></tr></table>"
logging.debug("HTML Text %s", html)
msg_root = MIMEMultipart("related")
msg_root["Subject"] = "[%s] %s" % (alert[alertid]["status"], alert[alertid]["summary"])
msg_root["From"] = ALERTER_MAIL
msg_root["To"] = MAILING_LIST
msg_root.preamble = "This is a multi-part message in MIME format."
msg_alt = MIMEMultipart("alternative")
msg_root.attach(msg_alt)
msg_text = MIMEText(text, "plain")
msg_alt.attach(msg_text)
msg_html = MIMEText(html, "html")
msg_alt.attach(msg_html)
if "graphs" in alert[alertid]:
msg_img = dict()
for g in alert[alertid]["graphs"]:
try:
image = urllib2.urlopen(g).read()
msg_img[g] = MIMEImage(image)
logging.debug("graph cid %s", graph_cid[g])
msg_img[g].add_header("Content-ID", "<" + graph_cid[g] + ">")
msg_root.attach(msg_img[g])
except:
pass
try:
logging.info("%s : Send email to %s", alert[alertid]["lastReceiveId"], MAILING_LIST)
s = smtplib.SMTP(SMTP_SERVER)
s.sendmail(ALERTER_MAIL, MAILING_LIST, msg_root.as_string())
s.quit()
except smtplib.SMTPException, e:
logging.error("%s : Sendmail failed - %s", alert[alertid]["lastReceiveId"], e)
