def send_notify(alertid): global tokens, hold try: for tag in alert[alertid]["tags"]: if tag.startswith("sms:") or tag.startswith("email:"): who = who_to_notify(tag) message = alert[alertid]["summary"] if tag.startswith("sms:") and tokens[who, "sms"] > 0: _Lock.acquire() tokens[who, "sms"] -= 1 _Lock.release() logging.debug("Taken a sms token from %s, there are only %d left", who, tokens[who, "sms"]) sms_notify(alertid, USERNAME, PASSWORD, owners[who]["mobile"]) elif tokens[who, "sms"] == 0: logging.error( "%s run out of sms tokens. Failed to notify %s.", who, alert[alertid]["lastReceiveId"] ) if tag.startswith("email:") and tokens[who, "email"] > 0: _Lock.acquire() tokens[who, "email"] -= 1 _Lock.release() logging.debug("Taken a email token from %s, there are only %d left", who, tokens[who, "sms"]) email_notify(alertid, owners[who]["email"]) elif tokens[who, "email"] == 0: logging.error( "%s run out of email tokens. Failed to notify %s.", who, alert[alertid]["lastReceiveId"] ) except Exception, e: logging.error('Notify sending failed for "%s" - %s - %s', alert[alertid]["lastReceiveId"], message, e) pass
def on_message(self, headers, body): global alert, hold logging.debug("Received alert : %s", body) alertid = json.loads(body)["id"] alert[alertid] = json.loads(body) logging.info( "%s : [%s] %s", alert[alertid]["lastReceiveId"], alert[alertid]["status"], alert[alertid]["summary"] ) if not should_we_notify(alertid): logging.debug( "%s : NOT PAGING for [%s] %s", alert[alertid]["lastReceiveId"], alert[alertid]["status"], alert[alertid]["summary"], ) del alert[alertid] return if alertid in hold: if alert[alertid]["severity"] == "NORMAL": logging.info("%s : Dropping NORMAL alert %s", alert[alertid]["lastReceiveId"], alertid) del hold[alertid] del alert[alertid] else: logging.info("%s : Update alert %s details", alert[alertid]["lastReceiveId"], alertid) else: hold[alertid] = time.time() + GRACEPERIOD logging.info( "%s : Holding onto alert %s for %s seconds", alert[alertid]["lastReceiveId"], alertid, GRACEPERIOD )
def email_notify(alertid, email): MAILING_LIST = email createTime = datetime.datetime.strptime(alert[alertid]["createTime"], "%Y-%m-%dT%H:%M:%S.%fZ") createTime = createTime.replace(tzinfo=pytz.utc) tz = pytz.timezone(TIMEZONE) localTime = createTime.astimezone(tz) text = "" text += "[%s] %s\n" % (alert[alertid]["status"], alert[alertid]["summary"]) text += "Alert Details\n" text += "Alert ID: %s\n" % (alert[alertid]["id"]) text += "Create Time: %s\n" % (localTime.strftime("%Y/%m/%d %H:%M:%S")) text += "Resource: %s\n" % (alert[alertid]["resource"]) text += "Environment: %s\n" % (",".join(alert[alertid]["environment"])) text += "Service: %s\n" % (",".join(alert[alertid]["service"])) text += "Event Name: %s\n" % (alert[alertid]["event"]) text += "Event Group: %s\n" % (alert[alertid]["group"]) text += "Event Value: %s\n" % (alert[alertid]["value"]) text += "Severity: %s -> %s\n" % (alert[alertid]["previousSeverity"], alert[alertid]["severity"]) text += "Status: %s\n" % (alert[alertid]["status"]) text += "Text: %s\n" % (alert[alertid]["text"]) if "thresholdInfo" in alert[alertid]: text += "Threshold Info: %s\n" % (alert[alertid]["thresholdInfo"]) if "duplicateCount" in alert[alertid]: text += "Duplicate Count: %s\n" % (alert[alertid]["duplicateCount"]) if "moreInfo" in alert[alertid]: text += "More Info: %s\n" % (alert[alertid]["moreInfo"]) text += "Historical Data\n" if "graphs" in alert[alertid]: for g in alert[alertid]["graphs"]: text += "%s\n" % (g) text += "Raw Alert\n" text += "%s\n" % (json.dumps(alert[alertid])) text += "Generated by %s on %s at %s\n" % ( "alert-notify.py", os.uname()[1], datetime.datetime.now().strftime("%a %d %b %H:%M:%S"), ) logging.debug("Raw Text: %s", text) html = '<p><table border="0" cellpadding="0" cellspacing="0" width="100%">\n' # table used to center email html += '<tr><td bgcolor="#ffffff" align="center">\n' html += '<table border="0" cellpadding="0" cellspacing="0" width="700">\n' # table used to set width of email html += ( '<tr><td bgcolor="#425470"><p align="center" style="font-size:24px;color:#d9fffd;font-weight:bold;"><strong>[%s] %s</strong></p>\n' % (alert[alertid]["status"], alert[alertid]["summary"]) ) html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Alert Details</p>\n' html += "<table>\n" html += ( '<tr><td><b>Alert ID:</b></td><td><a href="%s/alerta/details.php?id=%s" target="_blank">%s</a></td></tr>\n' % (API_SERVER, alert[alertid]["id"], alert[alertid]["id"]) ) html += "<tr><td><b>Create Time:</b></td><td>%s</td></tr>\n" % (localTime.strftime("%Y/%m/%d %H:%M:%S")) html += "<tr><td><b>Resource:</b></td><td>%s</td></tr>\n" % (alert[alertid]["resource"]) html += "<tr><td><b>Environment:</b></td><td>%s</td></tr>\n" % (",".join(alert[alertid]["environment"])) html += "<tr><td><b>Service:</b></td><td>%s</td></tr>\n" % (",".join(alert[alertid]["service"])) html += "<tr><td><b>Event Name:</b></td><td>%s</td></tr>\n" % (alert[alertid]["event"]) html += "<tr><td><b>Event Group:</b></td><td>%s</td></tr>\n" % (alert[alertid]["group"]) html += "<tr><td><b>Event Value:</b></td><td>%s</td></tr>\n" % (alert[alertid]["value"]) html += "<tr><td><b>Severity:</b></td><td>%s -> %s</td></tr>\n" % ( alert[alertid]["previousSeverity"], alert[alertid]["severity"], ) html += "<tr><td><b>Status:</b></td><td>%s</td></tr>\n" % (alert[alertid]["status"]) html += "<tr><td><b>Text:</b></td><td>%s</td></tr>\n" % (alert[alertid]["text"]) if "thresholdInfo" in alert[alertid]: html += "<tr><td><b>Threshold Info:</b></td><td>%s</td></tr>\n" % (alert[alertid]["thresholdInfo"]) if "duplicateCount" in alert[alertid]: html += "<tr><td><b>Duplicate Count:</b></td><td>%s</td></tr>\n" % (alert[alertid]["duplicateCount"]) if "moreInfo" in alert[alertid]: html += '<tr><td><b>More Info:</b></td><td><a href="%s">ganglia</a></td></tr>\n' % (alert[alertid]["moreInfo"]) html += "</table>\n" html += "</td></tr>\n" html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Historical Data</p>\n' if "graphs" in alert[alertid]: graph_cid = dict() for g in alert[alertid]["graphs"]: graph_cid[g] = str(uuid.uuid4()) html += '<tr><td><img src="cid:' + graph_cid[g] + '"></td></tr>\n' html += '<tr><td><p align="left" style="font-size:18px;line-height:22px;color:#c25130;font-weight:bold;">Raw Alert</p>\n' html += '<tr><td><p align="left" style="font-family: \'Courier New\', Courier, monospace">%s</p></td></tr>\n' % ( json.dumps(alert[alertid]) ) html += "<tr><td>Generated by %s on %s at %s</td></tr>\n" % ( "alert-mailer.py", os.uname()[1], datetime.datetime.now().strftime("%a %d %b %H:%M:%S"), ) html += "</table>" html += "</td></tr></table>" html += "</td></tr></table>" logging.debug("HTML Text %s", html) msg_root = MIMEMultipart("related") msg_root["Subject"] = "[%s] %s" % (alert[alertid]["status"], alert[alertid]["summary"]) msg_root["From"] = ALERTER_MAIL msg_root["To"] = MAILING_LIST msg_root.preamble = "This is a multi-part message in MIME format." msg_alt = MIMEMultipart("alternative") msg_root.attach(msg_alt) msg_text = MIMEText(text, "plain") msg_alt.attach(msg_text) msg_html = MIMEText(html, "html") msg_alt.attach(msg_html) if "graphs" in alert[alertid]: msg_img = dict() for g in alert[alertid]["graphs"]: try: image = urllib2.urlopen(g).read() msg_img[g] = MIMEImage(image) logging.debug("graph cid %s", graph_cid[g]) msg_img[g].add_header("Content-ID", "<" + graph_cid[g] + ">") msg_root.attach(msg_img[g]) except: pass try: logging.info("%s : Send email to %s", alert[alertid]["lastReceiveId"], MAILING_LIST) s = smtplib.SMTP(SMTP_SERVER) s.sendmail(ALERTER_MAIL, MAILING_LIST, msg_root.as_string()) s.quit() except smtplib.SMTPException, e: logging.error("%s : Sendmail failed - %s", alert[alertid]["lastReceiveId"], e)