def test_uptime_emails(self):
self._cleanup()
# GLOBAL ALERT
uptime_alert = {
"above_below": "above",
"rule_type": "uptime",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0}
alerts_model.collection.insert(down_alert)
data = {'data': []}
uptime_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test / testprocess is Down')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_health_check_emails(self):
self._cleanup()
health_check_alert = {
"rule_type": "health_check",
"server": self.server_id,
"status": "critical",
"command": "check-http.rb",
"period": 0,
}
alert_id = alerts_model.collection.insert(health_check_alert)
data = [{
u'command': u'check-http.rb',
u'name': u'',
u'exit_code': 2,
}]
formated_check_data = health_checks_results_model.save(data=data, server=self.server)
health_check_alerter.check(data=formated_check_data, server=self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - check-http.rb status is CRITICAL')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_system_emails(self):
self._cleanup()
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_process_emails(self):
self._cleanup()
process_alert = {
"above_below": "above",
"rule_type": "process",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}]}
process_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent()
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - testprocess/CPU > 1% alert (Current value: 2.0%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_health_check_emails(self):
self._cleanup()
health_check_alert = {
"rule_type": "health_check",
"server": self.server_id,
"status": "critical",
"command": "check-http.rb",
"period": 0,
}
alert_id = alerts_model.collection.insert(health_check_alert)
data = [{
u'command': u'check-http.rb',
u'name': u'',
u'exit_code': 2,
}]
formated_check_data = health_checks_results_model.save(data=data, server=self.server)
health_check_alerter.check(data=formated_check_data, server=self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - check-http.rb status is CRITICAL')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_uptime_emails(self):
self._cleanup()
# GLOBAL ALERT
uptime_alert = {
"above_below": "above",
"rule_type": "uptime",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0}
alerts_model.collection.insert(down_alert)
data = {'data': []}
uptime_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test / testprocess is Down')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_process_emails(self):
self._cleanup()
process_alert = {
"above_below": "above",
"rule_type": "process",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}]}
process_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent()
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - testprocess/CPU > 1% alert (Current value: 2.0%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_system_emails(self):
self._cleanup()
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def send_notifications():
notifications_to_send = generate_notifications()
for n in notifications_to_send:
if n.mute != True and n.global_mute != True:
message = generate_message(notification=n)
notify = n.alert.get('notifications', [])
# Collect all emails
emails_list = []
for x in notify:
email = x.get('email')
if email:
emails_list.append(email)
if len(emails_list) > 0:
send_notification_email(notification=n, emails=emails_list)
for provider_auth in notify:
provider = provider_auth.get('provider_id')
if provider == 'pushover':
send_pushover_notification(message=message,
auth=provider_auth)
if provider == 'opsgenie':
send_opsgenie_notification(message=message,
auth=provider_auth)
if provider == 'pagerduty':
send_pagerduty_notification(message=message,
auth=provider_auth)
if provider == 'victorops':
send_victorops_notification(message=message,
auth=provider_auth)
if provider == 'slack':
send_slack_notification(message=message,
auth=provider_auth)
if provider == 'hipchat':
send_hipchat_notification(message=message,
auth=provider_auth)
if provider == 'webhook':
send_webhook_notification(notification=n,
auth=provider_auth,
message=message)
alerts_history_model.mark_as_sent(n.trigger['_id'])
return notifications_to_send # For the remote command execute
def test_plugin_emails(self):
self._cleanup()
plugin = plugin_model.get_or_create(server_id=self.server_id,
name='testplugin')
gauge = plugin_model.get_or_create_gauge_by_name(plugin=plugin,
name='gauge')
plugin_alert = {
"above_below": "above",
"rule_type": "plugin",
"server": self.server_id,
"gauge": gauge['_id'],
"plugin": plugin['_id'],
"account_id": self.account_id,
"key": "testkey",
"period": 0,
"metric_value": 5,
"notifications": self.notifications_list
}
alert_id = alerts_model.collection.insert(plugin_alert)
key_name = '{0}.testkey'.format(gauge['name'])
data = {'gauges': {'bla.test': 1, key_name: 6}}
plugin_alerter.check(data=data, plugin=plugin)
unsent_alerts = alerts_history_model.get_unsent()
for trigger in unsent_alerts['data']:
assert trigger['alert_id'] == alert_id
assert trigger['average_value'] == 6
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(
mail.outbox[0].subject,
'Server: test - testplugin.gauge.testkey > 5 (Current value: 6.0)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def send_notifications():
notifications_to_send = generate_notifications()
for n in notifications_to_send:
if n.mute != True and n.global_mute != True:
message = generate_message(notification=n)
notify = n.alert.get('notifications', [])
# Collect all emails
emails_list = []
for x in notify:
email = x.get('email')
if email:
emails_list.append(email)
if len(emails_list) > 0:
send_notification_email(notification=n, emails=emails_list)
for provider_auth in notify:
provider = provider_auth.get('provider_id')
if provider == 'pushover':
send_pushover_notification(message=message, auth=provider_auth)
if provider == 'opsgenie':
send_opsgenie_notification(message=message, auth=provider_auth)
if provider == 'pagerduty':
send_pagerduty_notification(message=message, auth=provider_auth)
if provider == 'victorops':
send_victorops_notification(message=message, auth=provider_auth)
if provider == 'slack':
send_slack_notification(message=message, auth=provider_auth)
if provider == 'telegram':
send_telegram_notification(message=message, auth=provider_auth)
if provider == 'hipchat':
send_hipchat_notification(message=message, auth=provider_auth)
if provider == 'webhook':
send_webhook_notification(notification=n, auth=provider_auth, message=message)
alerts_history_model.mark_as_sent(n.trigger['_id'])
return notifications_to_send # For the remote command execute
def test_plugin_emails(self):
self._cleanup()
plugin = plugin_model.get_or_create(server_id=self.server_id, name='testplugin')
gauge = plugin_model.get_or_create_gauge_by_name(plugin=plugin, name='gauge')
plugin_alert = {
"above_below": "above",
"rule_type": "plugin",
"server": self.server_id,
"gauge": gauge['_id'],
"plugin": plugin['_id'],
"account_id": self.account_id,
"key": "testkey",
"period": 0,
"metric_value": 5,
"notifications": self.notifications_list
}
alert_id = alerts_model.collection.insert(plugin_alert)
key_name = '{0}.testkey'.format(gauge['name'])
data = {'gauges': {'bla.test': 1, key_name: 6}}
plugin_alerter.check(data=data, plugin=plugin)
unsent_alerts = alerts_history_model.get_unsent()
for trigger in unsent_alerts['data']:
assert trigger['alert_id'] == alert_id
assert trigger['average_value'] == 6
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - testplugin.gauge.testkey > 5 (Current value: 6.0)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def generate_notifications_test(self):
# System alert
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
}
# CPU alert
cpu_alert_dict = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert_dict)
for r in alerts_model.collection.find():
print(r)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
process_alert = {
"above_below": "above",
"rule_type": "process",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
}
cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}] }
process_alerter.check(data, self.server)
uptime_alert = {
"above_below": "above",
"rule_type": "uptime",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
}
down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0}
alerts_model.collection.insert(down_alert)
data = {'data': []}
uptime_alerter.check(data, self.server)
result = generate_notifications()
assert len(result) == 3
# Assert notification dict
system_keys = ['alert', 'server', 'metadata', 'timezone', 'trigger', 'mute', 'global_mute']
process_keys = list(system_keys)
process_keys.append('process')
for r in result:
rule_type = r.alert['rule_type']
if rule_type in ['uptime', 'process']:
assert set(r.__dict__.keys()) == set(process_keys)
else:
assert set(r.__dict__.keys()) == set(system_keys)
for r in result:
message = generate_message(notification=r)
if r.alert['rule_type'] == 'process':
assert message == 'Server:testserver/testprocess CPU > 1% for 0 seconds (Current value: 2.0%)'
elif r.alert['rule_type'] == 'system':
assert message == 'Server:testserver CPU>1% for 0 seconds (Current value: 1.3%)'
elif r.alert['rule_type'] == 'uptime':
assert message == 'testprocess on testserver is Down'