def test_system_emails(self):
self._cleanup()
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_system_emails(self):
self._cleanup()
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
"notifications": self.notifications_list
}
# CPU alert
cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
eq_(unsent_alerts['data'].count(), 1)
notifications = generate_notifications()
for n in notifications:
send_notification_email(notification=n, emails=self.emails)
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)')
self.assertEqual(mail.outbox[0].to, ['*****@*****.**'])
self._cleanup()
def test_system_check(self):
self._cleanup()
# System alert
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
}
# CPU alert
cpu_alert_dict = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alert_id = alerts_model.collection.insert(cpu_alert_dict)
rules = alerts_model.get_alerts(type='system', server=self.server)
eq_(len(rules), 1)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id)
for trigger in unsent_alerts['data']:
assert trigger['alert_id'] == alert_id
eq_(unsent_alerts['data'].count(), 1)
self._cleanup()
def test_global_check(self):
self._cleanup()
# GLOBAL ALERT
global_alert = {
"above_below": "above",
"rule_type": "global",
"server": "all",
"account_id": self.account_id,
"period": 0,
}
# CPU alert
cpu_alert_dict = {
**global_alert, 'metric': 'CPU',
'metric_value': 1,
'metric_type': "%"
}
alert_id = alerts_model.collection.insert(cpu_alert_dict)
global_rules = alerts_model.get_global_alerts(
account_id=self.account_id)
eq_(len(global_rules), 1)
data = {
u'cpu': {
u'system': u'1.30',
u'idle': u'98.70',
u'user': u'0.00',
u'steal': u'0.00',
u'nice': u'0.00'
}
}
server_alerter.check(data, self.server)
unsent_alerts = alerts_history_model.get_unsent(
server_id=self.server_id)
for trigger in unsent_alerts['data']:
assert trigger['alert_id'] == alert_id
eq_(unsent_alerts['data'].count(), 1)
def save_data_to_backend(self, data=None, server=None):
if server is None:
return
time_now = unix_utc_now()
date_now = datetime.utcnow()
expires_days = server.get('keep_data', 30)
if settings.KEEP_DATA is not None:
expires_days = settings.KEEP_DATA
expires_at = date_now + timedelta(days=expires_days)
system_data = data.get('system')
process_data = data.get('processes')
plugin_data = data.get('plugins')
checks_data = data.get('checks')
telegraf_data = data.get('series')
if telegraf_data:
formated_data = plugin_model.format_telegraf_to_amon(data=telegraf_data)
if len(formated_data) > 0:
for name, d in formated_data.items():
plugin = plugin_model.save_data(
server=server,
name=name,
data=d,
time=time_now,
expires_at=expires_at
)
if system_data:
system_model.save_data(
server=server,
data=system_data.copy(),
time=time_now,
expires_at=expires_at
)
server_alerter.check(data=system_data, server=server)
if process_data:
data = process_model.save_data(
server=server,
data=process_data,
time=time_now,
expires_at=expires_at
)
process_alerter.check(data=data, server=server)
uptime_alerter.check(data=data, server=server)
if plugin_data:
formated_data = plugin_model.flatten_plugin_data(data=plugin_data)
for name, data in formated_data.items():
plugin = plugin_model.save_data(
server=server,
name=name,
data=data,
time=time_now,
expires_at=expires_at
)
plugin_alerter.check(data=data, plugin=plugin, server=server)
if checks_data:
formated_check_data = health_checks_results_model.save(data=checks_data, server=server)
health_check_alerter.check(data=formated_check_data, server=server)
def generate_notifications_test(self):
# System alert
system_alert = {
"above_below": "above",
"rule_type": "system",
"server": self.server_id,
"account_id": self.account_id,
"period": 0,
}
# CPU alert
cpu_alert_dict = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert_dict)
for r in alerts_model.collection.find():
print(r)
data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}}
server_alerter.check(data, self.server)
process_alert = {
"above_below": "above",
"rule_type": "process",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
}
cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"}
alerts_model.collection.insert(cpu_alert)
data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}] }
process_alerter.check(data, self.server)
uptime_alert = {
"above_below": "above",
"rule_type": "uptime",
"server": self.server_id,
"process": self.process_id,
"account_id": self.account_id,
"period": 0,
}
down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0}
alerts_model.collection.insert(down_alert)
data = {'data': []}
uptime_alerter.check(data, self.server)
result = generate_notifications()
assert len(result) == 3
# Assert notification dict
system_keys = ['alert', 'server', 'metadata', 'timezone', 'trigger', 'mute', 'global_mute']
process_keys = list(system_keys)
process_keys.append('process')
for r in result:
rule_type = r.alert['rule_type']
if rule_type in ['uptime', 'process']:
assert set(r.__dict__.keys()) == set(process_keys)
else:
assert set(r.__dict__.keys()) == set(system_keys)
for r in result:
message = generate_message(notification=r)
if r.alert['rule_type'] == 'process':
assert message == 'Server:testserver/testprocess CPU > 1% for 0 seconds (Current value: 2.0%)'
elif r.alert['rule_type'] == 'system':
assert message == 'Server:testserver CPU>1% for 0 seconds (Current value: 1.3%)'
elif r.alert['rule_type'] == 'uptime':
assert message == 'testprocess on testserver is Down'
def save_data_to_backend(self, data=None, server=None):
if server is None:
return
time_now = unix_utc_now()
date_now = datetime.utcnow()
expires_days = server.get('keep_data', 30)
if settings.KEEP_DATA is not None:
expires_days = settings.KEEP_DATA
expires_at = date_now + timedelta(days=expires_days)
system_data = data.get('system')
process_data = data.get('processes')
plugin_data = data.get('plugins')
checks_data = data.get('checks')
telegraf_data = data.get('series')
if telegraf_data:
formated_data = plugin_model.format_telegraf_to_amon(
data=telegraf_data)
if len(formated_data) > 0:
for name, d in formated_data.items():
plugin = plugin_model.save_data(server=server,
name=name,
data=d,
time=time_now,
expires_at=expires_at)
if system_data:
system_model.save_data(server=server,
data=system_data.copy(),
time=time_now,
expires_at=expires_at)
server_alerter.check(data=system_data, server=server)
if process_data:
data = process_model.save_data(server=server,
data=process_data,
time=time_now,
expires_at=expires_at)
process_alerter.check(data=data, server=server)
uptime_alerter.check(data=data, server=server)
if plugin_data:
formated_data = plugin_model.flatten_plugin_data(data=plugin_data)
for name, data in formated_data.items():
plugin = plugin_model.save_data(server=server,
name=name,
data=data,
time=time_now,
expires_at=expires_at)
plugin_alerter.check(data=data, plugin=plugin, server=server)
if checks_data:
formated_check_data = health_checks_results_model.save(
data=checks_data, server=server)
health_check_alerter.check(data=formated_check_data, server=server)