def test_error_severity_with_explicit_summary(self):
# An explicit summary suppresses the 'ERROR: ' prefix.
alertlib.Alert('test message',
severity=logging.ERROR,
summary='a test...').send_to_email('ka-admin')
with disable_google_mail():
alertlib.Alert('test message',
severity=logging.ERROR,
summary='a test...').send_to_email('ka-admin')
self.assertEqual([{
'body': 'test message\n',
'sender': 'alertlib <*****@*****.**>',
'subject': 'a test...',
'to': ['*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: a test...\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'test message\n'),
], self.sent_to_sendmail)
def test_implicit_summary_long_first_line_with_period(self):
message = ('This text is long. It is very very long, '
'I cannot even say how long it will go on for. '
'Probably a long time a long time.\n'
'Finally, a second line!')
alertlib.Alert(message).send_to_email('ka-admin')
with disable_google_mail():
alertlib.Alert(message).send_to_email('ka-admin')
self.assertEqual([{
'body': message + '\n',
'sender': 'alertlib <*****@*****.**>',
'subject': 'This text is long',
'to': ['*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: This text is long\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'%s\n' % message),
], self.sent_to_sendmail)
def test_sender(self):
sender = 'foo$123*bar'
clean_sender = 'foo-123-bar'
alertlib.Alert('test message').send_to_email(
['ka-admin', 'ka-blackhole'], sender=sender)
with disable_google_mail():
alertlib.Alert('test message').send_to_email(
['ka-admin', 'ka-blackhole'], sender=sender)
self.assertEqual([{
'body':
'test message\n',
'sender':
('alertlib <*****@*****.**>' % clean_sender),
'subject':
'test message',
'to': ['*****@*****.**', '*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', [
'*****@*****.**', '*****@*****.**'
], 'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: test message\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected],'
' [email protected]\n\n'
'test message\n' % clean_sender),
], self.sent_to_sendmail)
def test_illegal_hostname(self):
with self.assertRaises(ValueError):
alertlib.Alert('test message').send_to_email(
'*****@*****.**')
with disable_google_mail():
with self.assertRaises(ValueError):
alertlib.Alert('test message').send_to_email(
'*****@*****.**')
def send_timeseries_to_cloudmonitoring(google_project_id, data, dry_run=False):
"""data is a list of 4tuples: (metric-name, metric-labels, value, time)."""
# alertlib is set up to only send one timeseries per call. But we
# want to send all the timeseries in a single call, so we have to
# do some hackery.
timeseries_data = []
alert = alertlib.Alert('gae-dashboard metrics')
old_send_datapoints = alert.send_datapoints_to_stackdriver
alert.send_datapoints_to_stackdriver = lambda timeseries, *a, **kw: (
timeseries_data.extend(timeseries))
# This will put the data into timeseries_data but not actually send it.
try:
for (metric_name, metric_labels, value, time_t) in data:
logging.info("Collecting to send to stackdriver: %s (%s) %s %s",
metric_name, metric_labels, value, time_t)
alert.send_to_stackdriver(metric_name,
value,
metric_labels=metric_labels,
project=google_project_id,
when=time_t)
finally:
alert.send_datapoints_to_stackdriver = old_send_datapoints
# Now we do the actual send.
if timeseries_data and not dry_run:
logging.debug("Sending to stackdriver: %s", timeseries_data)
alert.send_datapoints_to_stackdriver(timeseries_data,
project=google_project_id,
ignore_errors=False)
elif timeseries_data and dry_run:
logging.debug("Would send to stackdriver: %s", timeseries_data)
return len(timeseries_data)
def test_test_mode(self):
alertlib.enter_test_mode()
try:
alertlib.Alert('test message') \
.send_to_hipchat('1s and 0s') \
.send_to_email('ka-admin') \
.send_to_pagerduty('oncall') \
.send_to_logs() \
.send_to_graphite('stats.alerted')
finally:
alertlib.exit_test_mode()
# Should only log, not send to anything
self.assertEqual([], self.sent_to_hipchat)
self.assertEqual([], self.sent_to_google_mail)
self.assertEqual([], self.sent_to_sendmail)
self.assertEqual([], self.sent_to_syslog)
self.assertEqual([], self.sent_to_graphite)
self.assertEqual(
[('alertlib: would send to hipchat room 1s and 0s: '
'test message', ),
("alertlib: would send email to "
"['*****@*****.**'] "
"(from alertlib <*****@*****.**> CC None BCC None): "
"(subject test message) test message", ),
("alertlib: would send pagerduty email to "
"['*****@*****.**'] "
"(subject test message) test message", ),
('alertlib: would send to graphite: stats.alerted 1', )],
self.sent_to_info_log)
def test_no_limiting_with_longer_delay(self):
alert = alertlib.Alert('test message', rate_limit=60)
with self._mock_time(10):
alert.send_to_graphite('stats.test_message', 4)
self._set_time(100)
alert.send_to_graphite('stats.test_message', 4)
self.assertEqual(2, len(self.sent_to_graphite))
def test_default_alert_with_summary(self):
alertlib.Alert('xyz', summary='ABC').send_to_slack('#bot-testing')
actual = json.loads(self.sent_to_slack[0])
self.assertEqual(len(actual['attachments']), 1)
self.assertEqual(actual['attachments'][0]['pretext'], 'ABC')
self.assertEqual(actual['attachments'][0]['text'], 'xyz')
self.assertEqual(actual['attachments'][0]['fallback'], 'ABC\nxyz')
def test_default_options(self):
alertlib.Alert('test message').send_to_slack('#bot-testing')
actual = json.loads(self.sent_to_slack[0])
self.assertEqual(actual['channel'], '#bot-testing')
self.assertEqual(len(actual['attachments']), 1)
self.assertEqual(actual['attachments'][0]['text'], 'test message')
self.assertEqual(actual['attachments'][0]['fallback'], 'test message')
def test_utf8(self):
with disable_google_mail():
alertlib.Alert(u'yo \xf7', summary=u'yep \xf7') \
.send_to_pagerduty('oncall') \
.send_to_email('ka-admin')
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 8bit\n'
'Subject: yep \xc3\xb7\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'yo \xc3\xb7\n'),
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 8bit\n'
'Subject: yep \xc3\xb7\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'yo \xc3\xb7\n'),
], self.sent_to_sendmail)
self.assertEqual([], self.sent_to_google_mail)
def test_limiting_on_different_services(self):
alert = alertlib.Alert('test message', rate_limit=60)
for _ in xrange(100):
alert.send_to_graphite('stats.test_message', 4) \
.send_to_hipchat('1s and 0s')
alert.send_to_logs()
self.assertEqual(1, len(self.sent_to_graphite))
self.assertEqual(1, len(self.sent_to_hipchat))
self.assertEqual(1, len(self.sent_to_syslog))
def test_specified_options(self):
alertlib.Alert('test message').send_to_slack('#bot-testing',
sender='Bob Bot',
icon_emoji=':poop:')
actual = json.loads(self.sent_to_slack[0])
self.assertEqual(actual['channel'], '#bot-testing')
self.assertEqual(actual['username'], 'Bob Bot')
self.assertEqual(actual['icon_emoji'], ':poop:')
self.assertEqual(len(actual['attachments']), 1)
def alert(message, args):
a = alertlib.Alert(message, args.summary, args.severity, html=args.html)
for room in args.hipchat:
a.send_to_hipchat(room, args.color, args.notify, args.chat_sender
or 'AlertiGator')
for channel in args.slack:
a.send_to_slack(channel,
sender=args.chat_sender,
intro=args.slack_intro,
icon_url=args.icon_url,
icon_emoji=args.icon_emoji,
simple_message=args.slack_simple_message,
attachments=json.loads(args.slack_attachments),
thread=args.slack_thread,
as_app=args.as_app)
if args.mail:
a.send_to_email(args.mail, args.cc, args.bcc, args.sender_suffix)
# TODO(jacqueline): The --asana flag is deprecated and all callers
# should be shifted to --bugtracker. Remove support for this tag when
# confirmed that there are no remaining callers using this flag.
for project in args.asana:
a.send_to_asana(project, tags=args.bug_tags, followers=args.cc)
for project in args.bugtracker:
a.send_to_bugtracker(project, labels=args.bug_tags, watchers=args.cc)
if args.pagerduty:
a.send_to_pagerduty(args.pagerduty)
if args.logs:
a.send_to_logs()
for statistic in args.graphite:
a.send_to_graphite(statistic, args.graphite_value, args.graphite_host)
for statistic in args.stackdriver:
statistic_parts = statistic.split('|')
metric_name = statistic_parts[0]
metric_labels = dict(part.split('=') for part in statistic_parts[1:])
a.send_to_stackdriver(metric_name,
args.stackdriver_value,
metric_labels=metric_labels,
project=args.stackdriver_project,
ignore_errors=False)
# subject to change if we decide go the route of having an alert
# be exclusive to just one initiative
for initiative in args.aggregator:
a.send_to_alerta(initiative,
resource=args.aggregator_resource,
event=args.aggregator_event_name,
timeout=args.aggregator_timeout,
resolve=args.aggregator_resolve)
def dos_detect(start, end):
query = QUERY_TEMPLATE.format(fastly_log_tables=_fastly_log_tables(
start, end),
start_timestamp=start.strftime(TS_FORMAT),
end_timestamp=end.strftime(TS_FORMAT),
max_count=(MAX_REQS_SEC * PERIOD))
results = bq_util.call_bq(['query', query], project=BQ_PROJECT)
for row in results:
msg = ALERT_TEMPLATE.format(**row)
alertlib.Alert(msg).send_to_slack(ALERT_CHANNEL)
def test_html(self):
alertlib.Alert('<b>test message</b>', html=True).send_to_hipchat('rm')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'purple',
'from': 'AlertiGator',
'message': '<b>test message</b>',
'message_format': 'html',
'notify': 0,
'room_id': 'rm'
}], self.sent_to_hipchat)
def test_utf8(self):
alertlib.Alert(u'\xf7').send_to_hipchat(u'1s and \xf7s')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'purple',
'from': 'AlertiGator',
'message': '\xc3\xb7',
'message_format': 'text',
'notify': 0,
'room_id': '1s and \xc3\xb7s'
}], self.sent_to_hipchat)
def test_critical_severity(self):
alertlib.Alert('test message', severity=logging.CRITICAL) \
.send_to_hipchat('1s and 0s')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'red',
'from': 'AlertiGator',
'message': 'test message',
'message_format': 'text',
'notify': 1,
'room_id': '1s and 0s'
}], self.sent_to_hipchat)
def test_options(self):
alertlib.Alert('test message') \
.send_to_hipchat('1s and 0s', color='gray', notify=True)
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'gray',
'from': 'AlertiGator',
'message': 'test message',
'message_format': 'text',
'notify': 1,
'room_id': '1s and 0s'
}], self.sent_to_hipchat)
def test_custom_sender(self):
alertlib.Alert('test message') \
.send_to_hipchat('1s and 0s', sender='Notification Newt')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'purple',
'from': 'Notification Newt',
'message': 'test message',
'message_format': 'text',
'notify': 0,
'room_id': '1s and 0s'
}], self.sent_to_hipchat)
def test_no_message_munging_in_html(self):
"""html mode doesn't display emoticons, so no need to munge them."""
alertlib.Alert('(commit 345d8)', html=True).send_to_hipchat('rm')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'purple',
'from': 'AlertiGator',
'message': '(commit 345d8)',
'message_format': 'html',
'notify': 0,
'room_id': 'rm'
}], self.sent_to_hipchat)
def test_debug_severity(self):
alertlib.Alert('test message', severity=logging.DEBUG) \
.send_to_hipchat('1s and 0s')
self.assertEqual([{
'auth_token': '<hipchat token>',
'color': 'gray',
'from': 'AlertiGator',
'message': 'test message',
'message_format': 'text',
'notify': 0,
'room_id': '1s and 0s'
}], self.sent_to_hipchat)
def _send_as_bot(channel, msg, attachments):
"""Send a slack message on behalf of the testimonials bot.
Arguments:
channel: slack channel
msg: text of main slack message (ignored by slack if attachemnts exist)
attachments: slack message attachments, which are used for richly-
formatted messages (see https://api.slack.com/docs/attachments)
"""
alertlib.Alert(msg).send_to_slack(channel,
sender=_TESTIMONIALS_SENDER, icon_emoji=_TESTIMONIALS_EMOJI,
attachments=attachments)
def test_extra_newlines(self):
alertlib.Alert('yo!\n\n\n\n').send_to_email('ka-admin')
with disable_google_mail():
alertlib.Alert('yo!\n\n\n\n').send_to_email('ka-admin')
self.assertEqual([{
'body': 'yo!\n',
'sender': 'alertlib <*****@*****.**>',
'subject': 'yo!',
'to': ['*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: yo!\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'yo!\n'),
], self.sent_to_sendmail)
def _alert(slack_channel,
failures,
test_type,
truncate=10,
num_errors=None,
extra_text=''):
"""Alert with the first truncate failures, adding a header.
If num_errors is not equal to len(failures), you can pass it in.
(This happens when a system prints two error-lines for each file,
for instance.)
failures should be a list of strings with slack links.
If slack_channel is None or the empty string, we suppress alerting to
Slack, and only log.
"""
if not failures:
return
alert_lines = failures[:truncate]
if num_errors is None:
num_errors = len(failures)
if num_errors == 1:
pretext = 'Failed 1 %s' % test_type
else:
pretext = 'Failed %s %ss' % (num_errors, test_type)
if extra_text:
pretext = '%s %s' % (pretext, extra_text)
if len(failures) > truncate:
alert_lines.append('...')
text = '\n'.join(alert for alert in alert_lines)
fallback_text = '%s:\n%s' % (pretext, text)
attachment = {
'fallback': fallback_text,
'pretext': pretext,
'text': text,
'color': 'danger',
}
alert = alertlib.Alert(fallback_text, severity=logging.ERROR)
alert.send_to_logs()
if slack_channel:
alert.send_to_slack(slack_channel,
sender='Testing Turtle',
icon_emoji=':turtle:',
attachments=[attachment])
def test_implicit_summary_first_line(self):
message = 'This text is short\nBut has multiple lines'
alertlib.Alert(message).send_to_email('ka-admin')
with disable_google_mail():
alertlib.Alert(message).send_to_email('ka-admin')
self.assertEqual([{
'body': message + '\n',
'sender': 'alertlib <*****@*****.**>',
'subject': 'This text is short',
'to': ['*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: This text is short\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'%s\n' % message),
], self.sent_to_sendmail)
def test_html_email(self):
alertlib.Alert('<b>fire!</b>', html=True).send_to_email('ka-admin')
with disable_google_mail():
alertlib.Alert('<b>fire!</b>', html=True).send_to_email('ka-admin')
self.assertEqual([{
'body': '<b>fire!</b>\n',
'html': '<b>fire!</b>\n',
'sender': 'alertlib <*****@*****.**>',
'subject': '',
'to': ['*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', ['*****@*****.**'],
'Content-Type: text/html; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: \n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected]\n\n'
'<b>fire!</b>\n'),
], self.sent_to_sendmail)
def test_service_name_to_email(self):
alertlib.Alert('on fire!').send_to_pagerduty(
['The oncall-service, at your service!'])
self.assertEqual([{
'body':
'on fire!\n',
'sender':
'alertlib <*****@*****.**>',
'subject':
'on fire!',
'to':
['theoncall-serviceatyourservice'
'@khan-academy.pagerduty.com']
}], self.sent_to_google_mail)
def test_multiple_recipients(self):
alertlib.Alert('on fire!').send_to_pagerduty(['oncall', 'backup'])
self.assertEqual([{
'body':
'on fire!\n',
'sender':
'alertlib <*****@*****.**>',
'subject':
'on fire!',
'to': [
'*****@*****.**',
'*****@*****.**'
]
}], self.sent_to_google_mail)
def test_cc_and_bcc(self):
alertlib.Alert('test message').send_to_email(
['ka-admin', 'ka-blackhole'],
cc='ka-cc',
bcc=['ka-bcc', 'ka-hidden'])
with disable_google_mail():
alertlib.Alert('test message').send_to_email(
['ka-admin', 'ka-blackhole'],
cc='ka-cc',
bcc=['ka-bcc', 'ka-hidden'])
self.assertEqual(
[{
'body': 'test message\n',
'sender': 'alertlib <*****@*****.**>',
'subject': 'test message',
'to':
['*****@*****.**', '*****@*****.**'],
'cc': ['*****@*****.**'],
'bcc': ['*****@*****.**', '*****@*****.**']
}], self.sent_to_google_mail)
self.assertEqual([
('*****@*****.**', [
'*****@*****.**', '*****@*****.**'
], 'Content-Type: text/plain; charset="us-ascii"\n'
'MIME-Version: 1.0\n'
'Content-Transfer-Encoding: 7bit\n'
'Subject: test message\n'
'From: alertlib <*****@*****.**>\n'
'To: [email protected],'
' [email protected]\n'
'Cc: [email protected]\n'
'Bcc: [email protected],'
' [email protected]\n\n'
'test message\n'),
], self.sent_to_sendmail)
def alert(message, args):
a = alertlib.Alert(message, args.summary, args.severity, html=args.html)
for room in args.hipchat:
a.send_to_hipchat(room, args.color, args.notify, args.chat_sender
or 'AlertiGator')
for channel in args.slack:
a.send_to_slack(channel,
sender=args.chat_sender,
intro=args.slack_intro,
icon_url=args.icon_url,
icon_emoji=args.icon_emoji,
simple_message=args.slack_simple_message,
attachments=json.loads(args.slack_attachments))
if args.mail:
a.send_to_email(args.mail, args.cc, args.bcc, args.sender_suffix)
for project in args.asana:
a.send_to_asana(project, tags=args.asana_tags, followers=args.cc)
if args.pagerduty:
a.send_to_pagerduty(args.pagerduty)
if args.logs:
a.send_to_logs()
for statistic in args.graphite:
a.send_to_graphite(statistic, args.graphite_value, args.graphite_host)
for statistic in args.stackdriver:
statistic_parts = statistic.split('|')
metric_name = statistic_parts[0]
metric_labels = dict(part.split('=') for part in statistic_parts[1:])
a.send_to_stackdriver(metric_name,
args.stackdriver_value,
metric_labels=metric_labels,
project=args.stackdriver_project,
ignore_errors=False)
# subject to change if we decide go the route of having an alert
# be exclusive to just one initiative
for initiative in args.aggregator:
a.send_to_alerta(initiative,
resource=args.aggregator_resource,
event=args.aggregator_event_name,
timeout=args.aggregator_timeout,
resolve=args.aggregator_resolve)