def construct_notification_object(db_repo, notification_json): try: notification = Notification(notification_json['id'], notification_json['type'], notification_json['name'], notification_json['address'], notification_json['period'], notification_json['retry_count'], notification_json['raw_alarm']) # Grab notification method from database to see if it was changed stored_notification = grab_stored_notification_method(db_repo, notification.id) # Notification method was deleted if stored_notification is None: log.debug("Notification method {0} was deleted from database. " "Will stop sending.".format(notification.id)) return None # Update notification method with most up to date values else: notification.name = stored_notification[0] notification.type = stored_notification[1] notification.address = stored_notification[2] notification.period = stored_notification[3] return notification except exceptions.DatabaseException: log.warn("Error querying mysql for notification method. " "Using currently cached method.") return notification except Exception as e: log.warn("Error when attempting to construct notification {0}".format(e)) return None
def test_statsd(self, mock_log, mock_smtp, mock_email): mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_email.EmailNotifier = self._goodSendStub config_dict = { 'email': self.email_config, 'webhook': { 'address': 'xyz.com' }, 'pagerduty': { 'address': 'abc' } } notifiers.init(self.statsd) notifiers.config(config_dict) notifications = [] notifications.append( Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifications.append( Notification(1, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifications.append( Notification(2, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifiers.send_notifications(notifications) self.assertEqual(self.statsd.timer.timer_calls['email_time_start'], 3) self.assertEqual(self.statsd.timer.timer_calls['email_time_stop'], 3) self.assertEqual(self.statsd.counter.counter, 3)
def test_two_valid_notifications(self): alarm_dict = { "tenantId": "0", "alarmDefinitionId": "0", "alarmId": "1", "alarmName": "test Alarm", "oldState": "OK", "newState": "ALARM", "stateChangeReason": "I am alarming!", "timestamp": time.time() * 1000, "actionsEnabled": 1, "metrics": "cpu_util", "severity": "LOW", "link": "http://some-place.com", "lifecycleState": "OPEN" } alarm = self._create_raw_alarm(0, 5, alarm_dict) sql_response = [['test notification', 'EMAIL', '*****@*****.**', 0], ['test notification2', 'EMAIL', '*****@*****.**', 0]] notifications, partition, offset = self._run_alarm_processor( alarm, sql_response) test_notification = Notification('email', 0, 5, 'test notification', '*****@*****.**', 0, 0, alarm_dict) test_notification2 = Notification('email', 0, 5, 'test notification2', '*****@*****.**', 0, 0, alarm_dict) self.assertEqual(notifications, [test_notification, test_notification2]) self.assertEqual(partition, 0) self.assertEqual(offset, 5)
def test_smtp_sendmail_failed_connection_once(self, mock_smtp): """Email that fails on smtp_connect once """ metrics = [] metric_data = { 'name': 'cpu.percent', 'dimensions': { 'hostname': 'foo1', 'service': 'bar1' } } metrics.append(metric_data) metric_data = { 'name': 'cpu.percent', 'dimensions': { 'hostname': 'foo2', 'service': 'bar2' } } metrics.append(metric_data) mock_log = mock.MagicMock() mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.debug = self.trap.append mock_log.info = self.trap.append mock_log.exception = self.trap.append mock_smtp.SMTP.return_value = mock_smtp mock_smtp.sendmail.side_effect = [smtplib.SMTPServerDisconnected, None] # There has to be a better way to preserve exception definitions when # we're mocking access to a library mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected mock_smtp.SMTPException = smtplib.SMTPException email = email_notifier.EmailNotifier(mock_log) email.config() alarm_dict = alarm(metrics) notification = Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm_dict) self.trap.append(email.send_notification(notification)) self.assertIn( "SMTP server disconnected. Will reconnect and retry message.", self.trap) self.assertIn( "Sent email to %s, notification %s" % (notification.address, notification.to_json()), self.trap)
def notify(self, http_func, mock_requests): mock_log = mock.MagicMock() mock_log.warn = self.trap.put mock_log.error = self.trap.put mock_log.exception = self.trap.put mock_requests.post = http_func pagerduty = pagerduty_notifier.PagerdutyNotifier(mock_log) pagerduty.config(self.pagerduty_config) metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metric.append(metric_data) alarm_dict = alarm(metric) notification = Notification(0, 'pagerduty', 'pagerduty notification', 'ABCDEF', 0, 0, alarm_dict) self.trap.put(pagerduty.send_notification(notification))
def test_send_notification_unconfigured(self, mock_log, mock_smtp, mock_email): mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.info = self.trap.append mock_email.EmailNotifier = self._sendExceptionStub config_dict = { 'email': self.email_config, 'webhook': { 'address': 'xyz.com' } } notifiers.init(self.statsd) notifiers.config(config_dict) self.assertIn("No config data for type: pagerduty", self.trap) notifications = [] notifications.append( Notification(0, 'pagerduty', 'pagerduty notification', '*****@*****.**', 0, 0, alarm({}))) sent, failed, invalid = notifiers.send_notifications(notifications) self.assertEqual(sent, []) self.assertEqual(failed, []) self.assertEqual(len(invalid), 1) self.assertIn( "attempting to send unconfigured notification: pagerduty", self.trap)
def test_send_notification_failure(self, mock_log, mock_smtp, mock_email): mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.exception = self.trap.append mock_email.EmailNotifier = self._sendFailureStub config_dict = { 'email': self.email_config, 'webhook': { 'address': 'xyz.com' }, 'pagerduty': { 'address': 'abc' } } notifiers.init(self.statsd) notifiers.config(config_dict) notifications = [] notifications.append( Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) sent, failed, invalid = notifiers.send_notifications(notifications) self.assertEqual(sent, []) self.assertEqual(len(failed), 1) self.assertEqual(invalid, [])
def test_send_notification_exception(self, mock_log, mock_smtp, mock_email): mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.exception = self.trap.append mock_email.EmailNotifier = self._sendExceptionStub config_dict = { 'email': self.email_config, 'webhook': { 'address': 'xyz.com' }, 'pagerduty': { 'address': 'abc' } } notifiers.init(self.statsd) notifiers.config(config_dict) notifications = [] notifications.append( Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifiers.send_notifications(notifications) self.assertIn("send_notification exception for email", self.trap)
def test_valid_notification(self): """Test a valid notification, being put onto the notification_queue """ alarm_dict = { "tenantId": "0", "alarmDefinitionId": "0", "alarmId": "1", "alarmName": "test Alarm", "oldState": "OK", "newState": "ALARM", "stateChangeReason": "I am alarming!", "timestamp": time.time() * 1000, "actionsEnabled": 1, "metrics": "cpu_util" } alarm = self._create_raw_alarm(0, 4, alarm_dict) sql_response = [['test notification', 'EMAIL', '*****@*****.**']] notifications, partition, offset = self._run_alarm_processor( alarm, sql_response) test_notification = Notification('email', 0, 4, 'test notification', '*****@*****.**', 0, alarm_dict) self.assertEqual(notifications, [test_notification]) self.assertEqual(partition, 0) self.assertEqual(offset, 4)
def test_smtp_sendmail_smtp_None(self, mock_smtp): """Email that fails on smtp_connect twice """ metrics = [] metric_data = { 'name': 'cpu.percent', 'dimensions': { 'hostname': 'foo1', 'service': 'bar1' } } metrics.append(metric_data) metric_data = { 'name': 'cpu.percent', 'dimensions': { 'hostname': 'foo2', 'service': 'bar2' } } metrics.append(metric_data) mock_log = mock.MagicMock() mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.debug = self.trap.append mock_log.info = self.trap.append mock_log.exception = self.trap.append mock_smtp.SMTP.return_value = None mock_smtp.SMTP.side_effect = [socket.error, socket.error, socket.error] mock_smtp.sendmail.side_effect = [ smtplib.SMTPServerDisconnected, smtplib.SMTPServerDisconnected ] # There has to be a better way to preserve exception definitions when # we're mocking access to a library mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected mock_smtp.SMTPException = smtplib.SMTPException email = email_notifier.EmailNotifier(mock_log) email.config() del self.trap[:] alarm_dict = alarm(metrics) notification = Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm_dict) email_result = email.send_notification(notification) self.assertFalse(email_result) self.assertIn("Connecting to Email Server my.smtp.server", self.trap)
def _build_notification(self, alarm): db_time = self._statsd.get_timer() with db_time.time('config_db_time'): alarms_actions = self._db_repo.fetch_notifications(alarm) return [ Notification(alarms_action[0], alarms_action[1], alarms_action[2], alarms_action[3], alarms_action[4], 0, alarm) for alarms_action in alarms_actions ]
def test_invalid_notification(self): """Verify invalid notification type is rejected. """ alarm_dict = {"tenantId": "0", "alarmId": "0", "alarmName": "test Alarm", "oldState": "OK", "newState": "ALARM", "stateChangeReason": "I am alarming!", "timestamp": time.time(), "metrics": "cpu_util", "severity": "LOW", "link": "http://some-place.com", "lifecycleState": "OPEN"} invalid_notification = Notification(0, 'invalid', 'test notification', '*****@*****.**', 0, 0, alarm_dict) self._start_processor([invalid_notification]) self.assertIn('attempting to send unconfigured notification: invalid', self.trap)
def test_send_notification_correct(self, mock_log, mock_smtp, mock_email, mock_time): mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_email.EmailNotifier = self._goodSendStub mock_time.time.return_value = 42 config_dict = { 'email': self.email_config, 'webhook': { 'address': 'xyz.com' }, 'pagerduty': { 'address': 'abc' } } notifiers.init(self.statsd) notifiers.config(config_dict) notifications = [] notifications.append( Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifications.append( Notification(1, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) notifications.append( Notification(2, 'email', 'email notification', '*****@*****.**', 0, 0, alarm({}))) sent, failed, invalid = notifiers.send_notifications(notifications) self.assertEqual(len(sent), 3) self.assertEqual(failed, []) self.assertEqual(invalid, []) for n in sent: self.assertEqual(n.notification_timestamp, 42)
def to_notification(self, raw_alarm): """Check the notification setting for this project in mysql then create the appropriate notification """ failed_parse_count = self._statsd.get_counter(name='alarms_failed_parse_count') no_notification_count = self._statsd.get_counter(name='alarms_no_notification_count') notification_count = self._statsd.get_counter(name='created_count') db_time = self._statsd.get_timer() cur = self._mysql.cursor() partition = raw_alarm[0] offset = raw_alarm[1].offset try: alarm = self._parse_alarm(raw_alarm[1].message.value) except Exception as e: # This is general because of a lack of json exception base class failed_parse_count += 1 log.exception("Invalid Alarm format skipping partition %d, offset %d\nError%s" % (partition, offset, e)) return [], partition, offset log.debug("Read alarm from alarms sent_queue. Partition %d, Offset %d, alarm data %s" % (partition, offset, alarm)) if not self._alarm_is_valid(alarm): no_notification_count += 1 return [], partition, offset try: with db_time.time('config_db_time'): cur.execute("""SELECT name, type, address FROM alarm_action as aa JOIN notification_method as nm ON aa.action_id = nm.id WHERE aa.alarm_definition_id = %s and aa.alarm_state = %s""", [alarm['alarmDefinitionId'], alarm['newState']]) except MySQLdb.Error: log.exception('Mysql Error') raise notifications = [Notification(row[1].lower(), partition, offset, row[0], row[2], 0, alarm) for row in cur] if len(notifications) == 0: no_notification_count += 1 log.debug('No notifications found for this alarm, partition %d, offset %d, alarm data %s' % (partition, offset, alarm)) return [], partition, offset else: notification_count += len(notifications) return notifications, partition, offset
def construct_notification_object(db_repo, notification_json): try: notification = Notification( notification_json['id'], notification_json['type'], notification_json['name'], notification_json['address'], notification_json['period'], notification_json['retry_count'], notification_json['raw_alarm']) # Grab notification method from database to see if it was changed stored_notification = grab_stored_notification_method( db_repo, notification.id) # Notification method was deleted if stored_notification is None: LOG.debug("Notification method {0} was deleted from database. " "Will stop sending.".format(notification.id)) return None # Update notification method with most up to date values else: notification.name = stored_notification[0] notification.type = stored_notification[1] notification.address = stored_notification[2] notification.period = stored_notification[3] return notification except exceptions.DatabaseException: LOG.warn("Error querying mysql for notification method. " "Using currently cached method.") return notification except Exception as e: LOG.warn( "Error when attempting to construct notification {0}".format(e)) return None
def test_smtp_sendmail_failed_connection_once(self, mock_smtp): """Email that fails on smtp_connect once """ metrics = [] metric_data = {'name': 'cpu.percent', 'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metrics.append(metric_data) metric_data = {'name': 'cpu.percent', 'dimensions': {'hostname': 'foo2', 'service': 'bar2'}} metrics.append(metric_data) mock_log = mock.MagicMock() mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.debug = self.trap.append mock_log.info = self.trap.append mock_log.exception = self.trap.append mock_smtp.SMTP.return_value = mock_smtp mock_smtp.sendmail.side_effect = [smtplib.SMTPServerDisconnected, None] # There has to be a better way to preserve exception definitions when # we're mocking access to a library mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected mock_smtp.SMTPException = smtplib.SMTPException email = email_notifier.EmailNotifier(mock_log) email.config() alarm_dict = alarm(metrics) notification = Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm_dict) self.trap.append(email.send_notification(notification)) self.assertIn("SMTP server disconnected. Will reconnect and retry message.", self.trap) self.assertIn("Sent email to %s, notification %s" % (notification.address, notification.to_json()), self.trap)
def notify(self, smtp_stub, metric, mock_smtp): mock_smtp.SMTP = smtp_stub mock_log = mock.MagicMock() mock_log.warn = self.trap.append mock_log.error = self.trap.append email = email_notifier.EmailNotifier(mock_log) alarm_dict = alarm(metric) notification = Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0, alarm_dict) self.trap.append(email.send_notification(notification))
def email_setup(self, metric): alarm_dict = { "tenantId": "0", "alarmId": "0", "alarmName": "test Alarm", "oldState": "OK", "newState": "ALARM", "stateChangeReason": "I am alarming!", "timestamp": time.time(), "metrics": metric } notification = Notification('email', 0, 1, 'email notification', '*****@*****.**', 0, alarm_dict) self._start_processor([notification])
def test_smtp_sendmail_failed_connection_once_then_email(self, mock_smtp): """Email that fails on smtp_connect once then email """ metrics = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metrics.append(metric_data) metric_data = {'dimensions': {'hostname': 'foo2', 'service': 'bar2'}} metrics.append(metric_data) mock_log = mock.MagicMock() mock_log.warn = self.trap.append mock_log.error = self.trap.append mock_log.debug = self.trap.append mock_log.info = self.trap.append mock_log.exception = self.trap.append mock_smtp.SMTP.return_value = mock_smtp mock_smtp.sendmail.side_effect = [ smtplib.SMTPServerDisconnected, smtplib.SMTPException ] # There has to be a better way to preserve exception definitions when # we're mocking access to a library mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected mock_smtp.SMTPException = smtplib.SMTPException email = email_notifier.EmailNotifier(mock_log) email.config(self.email_config) alarm_dict = alarm(metrics) notification = Notification('email', 0, 1, 'email notification', '*****@*****.**', 0, alarm_dict) self.trap.append(email.send_notification(notification)) self.assertIn( "SMTP server disconnected. Will reconnect and retry message.", self.trap) self.assertIn("Error sending Email Notification", self.trap) self.assertNotIn("Unable to connect to email server.", self.trap)
def notify(self, http_func, mock_requests): mock_log = mock.MagicMock() mock_log.warn = self.trap.put mock_log.error = self.trap.put mock_log.exception = self.trap.put mock_requests.post = http_func webhook = webhook_notifier.WebhookNotifier(mock_log) webhook.config(self.webhook_config) metric = [] metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}} metric.append(metric_data) alarm_dict = alarm(metric) notification = Notification('webhook', 0, 1, 'webhook notification', 'http://mock:3333/', 0, 0, alarm_dict) self.trap.put(webhook.send_notification(notification))