def test_uptime_emails(self): self._cleanup() # GLOBAL ALERT uptime_alert = { "above_below": "above", "rule_type": "uptime", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0} alerts_model.collection.insert(down_alert) data = {'data': []} uptime_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test / testprocess is Down') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_health_check_emails(self): self._cleanup() health_check_alert = { "rule_type": "health_check", "server": self.server_id, "status": "critical", "command": "check-http.rb", "period": 0, } alert_id = alerts_model.collection.insert(health_check_alert) data = [{ u'command': u'check-http.rb', u'name': u'', u'exit_code': 2, }] formated_check_data = health_checks_results_model.save(data=data, server=self.server) health_check_alerter.check(data=formated_check_data, server=self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - check-http.rb status is CRITICAL') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_system_emails(self): self._cleanup() system_alert = { "above_below": "above", "rule_type": "system", "server": self.server_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } # CPU alert cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert) data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}} server_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_process_emails(self): self._cleanup() process_alert = { "above_below": "above", "rule_type": "process", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } # CPU alert cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert) data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}]} process_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent() eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - testprocess/CPU > 1% alert (Current value: 2.0%)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_health_check_emails(self): self._cleanup() health_check_alert = { "rule_type": "health_check", "server": self.server_id, "status": "critical", "command": "check-http.rb", "period": 0, } alert_id = alerts_model.collection.insert(health_check_alert) data = [{ u'command': u'check-http.rb', u'name': u'', u'exit_code': 2, }] formated_check_data = health_checks_results_model.save(data=data, server=self.server) health_check_alerter.check(data=formated_check_data, server=self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - check-http.rb status is CRITICAL') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_uptime_emails(self): self._cleanup() # GLOBAL ALERT uptime_alert = { "above_below": "above", "rule_type": "uptime", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0} alerts_model.collection.insert(down_alert) data = {'data': []} uptime_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test / testprocess is Down') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_process_emails(self): self._cleanup() process_alert = { "above_below": "above", "rule_type": "process", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } # CPU alert cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert) data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}]} process_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent() eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - testprocess/CPU > 1% alert (Current value: 2.0%)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def test_system_emails(self): self._cleanup() system_alert = { "above_below": "above", "rule_type": "system", "server": self.server_id, "account_id": self.account_id, "period": 0, "notifications": self.notifications_list } # CPU alert cpu_alert = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert) data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}} server_alerter.check(data, self.server) unsent_alerts = alerts_history_model.get_unsent(server_id=self.server_id) eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - CPU > 1% alert (Current value: 1.3%)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def send_notifications(): notifications_to_send = generate_notifications() for n in notifications_to_send: if n.mute != True and n.global_mute != True: message = generate_message(notification=n) notify = n.alert.get('notifications', []) # Collect all emails emails_list = [] for x in notify: email = x.get('email') if email: emails_list.append(email) if len(emails_list) > 0: send_notification_email(notification=n, emails=emails_list) for provider_auth in notify: provider = provider_auth.get('provider_id') if provider == 'pushover': send_pushover_notification(message=message, auth=provider_auth) if provider == 'opsgenie': send_opsgenie_notification(message=message, auth=provider_auth) if provider == 'pagerduty': send_pagerduty_notification(message=message, auth=provider_auth) if provider == 'victorops': send_victorops_notification(message=message, auth=provider_auth) if provider == 'slack': send_slack_notification(message=message, auth=provider_auth) if provider == 'hipchat': send_hipchat_notification(message=message, auth=provider_auth) if provider == 'webhook': send_webhook_notification(notification=n, auth=provider_auth, message=message) alerts_history_model.mark_as_sent(n.trigger['_id']) return notifications_to_send # For the remote command execute
def test_plugin_emails(self): self._cleanup() plugin = plugin_model.get_or_create(server_id=self.server_id, name='testplugin') gauge = plugin_model.get_or_create_gauge_by_name(plugin=plugin, name='gauge') plugin_alert = { "above_below": "above", "rule_type": "plugin", "server": self.server_id, "gauge": gauge['_id'], "plugin": plugin['_id'], "account_id": self.account_id, "key": "testkey", "period": 0, "metric_value": 5, "notifications": self.notifications_list } alert_id = alerts_model.collection.insert(plugin_alert) key_name = '{0}.testkey'.format(gauge['name']) data = {'gauges': {'bla.test': 1, key_name: 6}} plugin_alerter.check(data=data, plugin=plugin) unsent_alerts = alerts_history_model.get_unsent() for trigger in unsent_alerts['data']: assert trigger['alert_id'] == alert_id assert trigger['average_value'] == 6 eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual( mail.outbox[0].subject, 'Server: test - testplugin.gauge.testkey > 5 (Current value: 6.0)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def send_notifications(): notifications_to_send = generate_notifications() for n in notifications_to_send: if n.mute != True and n.global_mute != True: message = generate_message(notification=n) notify = n.alert.get('notifications', []) # Collect all emails emails_list = [] for x in notify: email = x.get('email') if email: emails_list.append(email) if len(emails_list) > 0: send_notification_email(notification=n, emails=emails_list) for provider_auth in notify: provider = provider_auth.get('provider_id') if provider == 'pushover': send_pushover_notification(message=message, auth=provider_auth) if provider == 'opsgenie': send_opsgenie_notification(message=message, auth=provider_auth) if provider == 'pagerduty': send_pagerduty_notification(message=message, auth=provider_auth) if provider == 'victorops': send_victorops_notification(message=message, auth=provider_auth) if provider == 'slack': send_slack_notification(message=message, auth=provider_auth) if provider == 'telegram': send_telegram_notification(message=message, auth=provider_auth) if provider == 'hipchat': send_hipchat_notification(message=message, auth=provider_auth) if provider == 'webhook': send_webhook_notification(notification=n, auth=provider_auth, message=message) alerts_history_model.mark_as_sent(n.trigger['_id']) return notifications_to_send # For the remote command execute
def test_plugin_emails(self): self._cleanup() plugin = plugin_model.get_or_create(server_id=self.server_id, name='testplugin') gauge = plugin_model.get_or_create_gauge_by_name(plugin=plugin, name='gauge') plugin_alert = { "above_below": "above", "rule_type": "plugin", "server": self.server_id, "gauge": gauge['_id'], "plugin": plugin['_id'], "account_id": self.account_id, "key": "testkey", "period": 0, "metric_value": 5, "notifications": self.notifications_list } alert_id = alerts_model.collection.insert(plugin_alert) key_name = '{0}.testkey'.format(gauge['name']) data = {'gauges': {'bla.test': 1, key_name: 6}} plugin_alerter.check(data=data, plugin=plugin) unsent_alerts = alerts_history_model.get_unsent() for trigger in unsent_alerts['data']: assert trigger['alert_id'] == alert_id assert trigger['average_value'] == 6 eq_(unsent_alerts['data'].count(), 1) notifications = generate_notifications() for n in notifications: send_notification_email(notification=n, emails=self.emails) self.assertEqual(len(mail.outbox), 1) self.assertEqual(mail.outbox[0].subject, 'Server: test - testplugin.gauge.testkey > 5 (Current value: 6.0)') self.assertEqual(mail.outbox[0].to, ['*****@*****.**']) self._cleanup()
def generate_notifications_test(self): # System alert system_alert = { "above_below": "above", "rule_type": "system", "server": self.server_id, "account_id": self.account_id, "period": 0, } # CPU alert cpu_alert_dict = {**system_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert_dict) for r in alerts_model.collection.find(): print(r) data = {u'cpu': {u'system': u'1.30', u'idle': u'98.70', u'user': u'0.00', u'steal': u'0.00', u'nice': u'0.00'}} server_alerter.check(data, self.server) process_alert = { "above_below": "above", "rule_type": "process", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, } cpu_alert = {**process_alert, 'metric': 'CPU', 'metric_value': 1, 'metric_type': "%"} alerts_model.collection.insert(cpu_alert) data = {'data': [{'p': self.process_id, 'c': 2, 'm': 254.0}] } process_alerter.check(data, self.server) uptime_alert = { "above_below": "above", "rule_type": "uptime", "server": self.server_id, "process": self.process_id, "account_id": self.account_id, "period": 0, } down_alert = {**uptime_alert, 'metric': 'Down', 'metric_value': 0} alerts_model.collection.insert(down_alert) data = {'data': []} uptime_alerter.check(data, self.server) result = generate_notifications() assert len(result) == 3 # Assert notification dict system_keys = ['alert', 'server', 'metadata', 'timezone', 'trigger', 'mute', 'global_mute'] process_keys = list(system_keys) process_keys.append('process') for r in result: rule_type = r.alert['rule_type'] if rule_type in ['uptime', 'process']: assert set(r.__dict__.keys()) == set(process_keys) else: assert set(r.__dict__.keys()) == set(system_keys) for r in result: message = generate_message(notification=r) if r.alert['rule_type'] == 'process': assert message == 'Server:testserver/testprocess CPU > 1% for 0 seconds (Current value: 2.0%)' elif r.alert['rule_type'] == 'system': assert message == 'Server:testserver CPU>1% for 0 seconds (Current value: 1.3%)' elif r.alert['rule_type'] == 'uptime': assert message == 'testprocess on testserver is Down'