Esempio n. 1
0
def construct_notification_object(db_repo, notification_json):
    try:
        notification = Notification(notification_json['id'],
                                    notification_json['type'],
                                    notification_json['name'],
                                    notification_json['address'],
                                    notification_json['period'],
                                    notification_json['retry_count'],
                                    notification_json['raw_alarm'])
        # Grab notification method from database to see if it was changed
        stored_notification = grab_stored_notification_method(db_repo, notification.id)
        # Notification method was deleted
        if stored_notification is None:
            log.debug("Notification method {0} was deleted from database. "
                      "Will stop sending.".format(notification.id))
            return None
        # Update notification method with most up to date values
        else:
            notification.name = stored_notification[0]
            notification.type = stored_notification[1]
            notification.address = stored_notification[2]
            notification.period = stored_notification[3]
            return notification
    except exceptions.DatabaseException:
        log.warn("Error querying mysql for notification method. "
                 "Using currently cached method.")
        return notification
    except Exception as e:
        log.warn("Error when attempting to construct notification {0}".format(e))
        return None
Esempio n. 2
0
    def test_statsd(self, mock_log, mock_smtp, mock_email):
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append

        mock_email.EmailNotifier = self._goodSendStub

        config_dict = {
            'email': self.email_config,
            'webhook': {
                'address': 'xyz.com'
            },
            'pagerduty': {
                'address': 'abc'
            }
        }

        notifiers.init(self.statsd)
        notifiers.config(config_dict)

        notifications = []
        notifications.append(
            Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0,
                         alarm({})))
        notifications.append(
            Notification(1, 'email', 'email notification', '*****@*****.**', 0,
                         0, alarm({})))
        notifications.append(
            Notification(2, 'email', 'email notification', '*****@*****.**', 0,
                         0, alarm({})))

        notifiers.send_notifications(notifications)

        self.assertEqual(self.statsd.timer.timer_calls['email_time_start'], 3)
        self.assertEqual(self.statsd.timer.timer_calls['email_time_stop'], 3)
        self.assertEqual(self.statsd.counter.counter, 3)
Esempio n. 3
0
    def test_two_valid_notifications(self):
        alarm_dict = {
            "tenantId": "0",
            "alarmDefinitionId": "0",
            "alarmId": "1",
            "alarmName": "test Alarm",
            "oldState": "OK",
            "newState": "ALARM",
            "stateChangeReason": "I am alarming!",
            "timestamp": time.time() * 1000,
            "actionsEnabled": 1,
            "metrics": "cpu_util",
            "severity": "LOW",
            "link": "http://some-place.com",
            "lifecycleState": "OPEN"
        }

        alarm = self._create_raw_alarm(0, 5, alarm_dict)

        sql_response = [['test notification', 'EMAIL', '*****@*****.**', 0],
                        ['test notification2', 'EMAIL', '*****@*****.**', 0]]
        notifications, partition, offset = self._run_alarm_processor(
            alarm, sql_response)

        test_notification = Notification('email', 0, 5, 'test notification',
                                         '*****@*****.**', 0, 0, alarm_dict)
        test_notification2 = Notification('email', 0, 5, 'test notification2',
                                          '*****@*****.**', 0, 0, alarm_dict)

        self.assertEqual(notifications,
                         [test_notification, test_notification2])
        self.assertEqual(partition, 0)
        self.assertEqual(offset, 5)
    def test_smtp_sendmail_failed_connection_once(self, mock_smtp):
        """Email that fails on smtp_connect once
        """

        metrics = []
        metric_data = {
            'name': 'cpu.percent',
            'dimensions': {
                'hostname': 'foo1',
                'service': 'bar1'
            }
        }
        metrics.append(metric_data)
        metric_data = {
            'name': 'cpu.percent',
            'dimensions': {
                'hostname': 'foo2',
                'service': 'bar2'
            }
        }
        metrics.append(metric_data)

        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.debug = self.trap.append
        mock_log.info = self.trap.append
        mock_log.exception = self.trap.append

        mock_smtp.SMTP.return_value = mock_smtp
        mock_smtp.sendmail.side_effect = [smtplib.SMTPServerDisconnected, None]

        # There has to be a better way to preserve exception definitions when
        # we're mocking access to a library
        mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected
        mock_smtp.SMTPException = smtplib.SMTPException

        email = email_notifier.EmailNotifier(mock_log)

        email.config()

        alarm_dict = alarm(metrics)

        notification = Notification(0, 'email', 'email notification',
                                    '*****@*****.**', 0, 0, alarm_dict)

        self.trap.append(email.send_notification(notification))

        self.assertIn(
            "SMTP server disconnected. Will reconnect and retry message.",
            self.trap)
        self.assertIn(
            "Sent email to %s, notification %s" %
            (notification.address, notification.to_json()), self.trap)
    def notify(self, http_func, mock_requests):
        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.put
        mock_log.error = self.trap.put
        mock_log.exception = self.trap.put

        mock_requests.post = http_func

        pagerduty = pagerduty_notifier.PagerdutyNotifier(mock_log)

        pagerduty.config(self.pagerduty_config)

        metric = []
        metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}}
        metric.append(metric_data)

        alarm_dict = alarm(metric)

        notification = Notification(0,
                                    'pagerduty',
                                    'pagerduty notification',
                                    'ABCDEF',
                                    0,
                                    0,
                                    alarm_dict)

        self.trap.put(pagerduty.send_notification(notification))
Esempio n. 6
0
    def test_send_notification_unconfigured(self, mock_log, mock_smtp,
                                            mock_email):
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.info = self.trap.append

        mock_email.EmailNotifier = self._sendExceptionStub

        config_dict = {
            'email': self.email_config,
            'webhook': {
                'address': 'xyz.com'
            }
        }

        notifiers.init(self.statsd)
        notifiers.config(config_dict)

        self.assertIn("No config data for type: pagerduty", self.trap)

        notifications = []
        notifications.append(
            Notification(0, 'pagerduty', 'pagerduty notification',
                         '*****@*****.**', 0, 0, alarm({})))

        sent, failed, invalid = notifiers.send_notifications(notifications)

        self.assertEqual(sent, [])
        self.assertEqual(failed, [])
        self.assertEqual(len(invalid), 1)

        self.assertIn(
            "attempting to send unconfigured notification: pagerduty",
            self.trap)
Esempio n. 7
0
    def test_send_notification_failure(self, mock_log, mock_smtp, mock_email):
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.exception = self.trap.append

        mock_email.EmailNotifier = self._sendFailureStub

        config_dict = {
            'email': self.email_config,
            'webhook': {
                'address': 'xyz.com'
            },
            'pagerduty': {
                'address': 'abc'
            }
        }

        notifiers.init(self.statsd)
        notifiers.config(config_dict)

        notifications = []
        notifications.append(
            Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0,
                         alarm({})))

        sent, failed, invalid = notifiers.send_notifications(notifications)

        self.assertEqual(sent, [])
        self.assertEqual(len(failed), 1)
        self.assertEqual(invalid, [])
Esempio n. 8
0
    def test_send_notification_exception(self, mock_log, mock_smtp,
                                         mock_email):
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.exception = self.trap.append

        mock_email.EmailNotifier = self._sendExceptionStub

        config_dict = {
            'email': self.email_config,
            'webhook': {
                'address': 'xyz.com'
            },
            'pagerduty': {
                'address': 'abc'
            }
        }

        notifiers.init(self.statsd)
        notifiers.config(config_dict)

        notifications = []
        notifications.append(
            Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0,
                         alarm({})))

        notifiers.send_notifications(notifications)

        self.assertIn("send_notification exception for email", self.trap)
Esempio n. 9
0
    def test_valid_notification(self):
        """Test a valid notification, being put onto the notification_queue
        """
        alarm_dict = {
            "tenantId": "0",
            "alarmDefinitionId": "0",
            "alarmId": "1",
            "alarmName": "test Alarm",
            "oldState": "OK",
            "newState": "ALARM",
            "stateChangeReason": "I am alarming!",
            "timestamp": time.time() * 1000,
            "actionsEnabled": 1,
            "metrics": "cpu_util"
        }
        alarm = self._create_raw_alarm(0, 4, alarm_dict)

        sql_response = [['test notification', 'EMAIL', '*****@*****.**']]
        notifications, partition, offset = self._run_alarm_processor(
            alarm, sql_response)

        test_notification = Notification('email', 0, 4, 'test notification',
                                         '*****@*****.**', 0, alarm_dict)

        self.assertEqual(notifications, [test_notification])
        self.assertEqual(partition, 0)
        self.assertEqual(offset, 4)
    def test_smtp_sendmail_smtp_None(self, mock_smtp):
        """Email that fails on smtp_connect twice
        """

        metrics = []
        metric_data = {
            'name': 'cpu.percent',
            'dimensions': {
                'hostname': 'foo1',
                'service': 'bar1'
            }
        }
        metrics.append(metric_data)
        metric_data = {
            'name': 'cpu.percent',
            'dimensions': {
                'hostname': 'foo2',
                'service': 'bar2'
            }
        }
        metrics.append(metric_data)

        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.debug = self.trap.append
        mock_log.info = self.trap.append
        mock_log.exception = self.trap.append

        mock_smtp.SMTP.return_value = None
        mock_smtp.SMTP.side_effect = [socket.error, socket.error, socket.error]

        mock_smtp.sendmail.side_effect = [
            smtplib.SMTPServerDisconnected, smtplib.SMTPServerDisconnected
        ]

        # There has to be a better way to preserve exception definitions when
        # we're mocking access to a library
        mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected
        mock_smtp.SMTPException = smtplib.SMTPException

        email = email_notifier.EmailNotifier(mock_log)

        email.config()

        del self.trap[:]

        alarm_dict = alarm(metrics)

        notification = Notification(0, 'email', 'email notification',
                                    '*****@*****.**', 0, 0, alarm_dict)

        email_result = email.send_notification(notification)

        self.assertFalse(email_result)
        self.assertIn("Connecting to Email Server my.smtp.server", self.trap)
    def _build_notification(self, alarm):
        db_time = self._statsd.get_timer()

        with db_time.time('config_db_time'):
            alarms_actions = self._db_repo.fetch_notifications(alarm)

        return [
            Notification(alarms_action[0], alarms_action[1], alarms_action[2],
                         alarms_action[3], alarms_action[4], 0, alarm)
            for alarms_action in alarms_actions
        ]
    def test_invalid_notification(self):
        """Verify invalid notification type is rejected.
        """
        alarm_dict = {"tenantId": "0", "alarmId": "0", "alarmName": "test Alarm", "oldState": "OK", "newState": "ALARM",
                      "stateChangeReason": "I am alarming!", "timestamp": time.time(), "metrics": "cpu_util",
                      "severity": "LOW", "link": "http://some-place.com", "lifecycleState": "OPEN"}
        invalid_notification = Notification(0, 'invalid', 'test notification', '*****@*****.**', 0, 0, alarm_dict)

        self._start_processor([invalid_notification])

        self.assertIn('attempting to send unconfigured notification: invalid', self.trap)
Esempio n. 13
0
    def test_send_notification_correct(self, mock_log, mock_smtp, mock_email,
                                       mock_time):
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append

        mock_email.EmailNotifier = self._goodSendStub

        mock_time.time.return_value = 42

        config_dict = {
            'email': self.email_config,
            'webhook': {
                'address': 'xyz.com'
            },
            'pagerduty': {
                'address': 'abc'
            }
        }

        notifiers.init(self.statsd)
        notifiers.config(config_dict)

        notifications = []
        notifications.append(
            Notification(0, 'email', 'email notification', '*****@*****.**', 0, 0,
                         alarm({})))
        notifications.append(
            Notification(1, 'email', 'email notification', '*****@*****.**', 0,
                         0, alarm({})))
        notifications.append(
            Notification(2, 'email', 'email notification', '*****@*****.**', 0,
                         0, alarm({})))

        sent, failed, invalid = notifiers.send_notifications(notifications)

        self.assertEqual(len(sent), 3)
        self.assertEqual(failed, [])
        self.assertEqual(invalid, [])

        for n in sent:
            self.assertEqual(n.notification_timestamp, 42)
Esempio n. 14
0
    def to_notification(self, raw_alarm):
        """Check the notification setting for this project in mysql then create the appropriate notification
        """
        failed_parse_count = self._statsd.get_counter(name='alarms_failed_parse_count')
        no_notification_count = self._statsd.get_counter(name='alarms_no_notification_count')
        notification_count = self._statsd.get_counter(name='created_count')
        db_time = self._statsd.get_timer()

        cur = self._mysql.cursor()

        partition = raw_alarm[0]
        offset = raw_alarm[1].offset
        try:
            alarm = self._parse_alarm(raw_alarm[1].message.value)
        except Exception as e:  # This is general because of a lack of json exception base class
            failed_parse_count += 1
            log.exception("Invalid Alarm format skipping partition %d, offset %d\nError%s" % (partition, offset, e))
            return [], partition, offset

        log.debug("Read alarm from alarms sent_queue. Partition %d, Offset %d, alarm data %s"
                  % (partition, offset, alarm))

        if not self._alarm_is_valid(alarm):
            no_notification_count += 1
            return [], partition, offset

        try:
            with db_time.time('config_db_time'):
                cur.execute("""SELECT name, type, address
                               FROM alarm_action as aa
                               JOIN notification_method as nm ON aa.action_id = nm.id
                               WHERE aa.alarm_definition_id = %s and aa.alarm_state = %s""",
                            [alarm['alarmDefinitionId'], alarm['newState']])
        except MySQLdb.Error:
            log.exception('Mysql Error')
            raise

        notifications = [Notification(row[1].lower(),
                                      partition,
                                      offset,
                                      row[0],
                                      row[2],
                                      0,
                                      alarm) for row in cur]

        if len(notifications) == 0:
            no_notification_count += 1
            log.debug('No notifications found for this alarm, partition %d, offset %d, alarm data %s'
                      % (partition, offset, alarm))
            return [], partition, offset
        else:
            notification_count += len(notifications)
            return notifications, partition, offset
Esempio n. 15
0
def construct_notification_object(db_repo, notification_json):
    try:
        notification = Notification(
            notification_json['id'], notification_json['type'],
            notification_json['name'], notification_json['address'],
            notification_json['period'], notification_json['retry_count'],
            notification_json['raw_alarm'])
        # Grab notification method from database to see if it was changed
        stored_notification = grab_stored_notification_method(
            db_repo, notification.id)
        # Notification method was deleted
        if stored_notification is None:
            LOG.debug("Notification method {0} was deleted from database. "
                      "Will stop sending.".format(notification.id))
            return None
        # Update notification method with most up to date values
        else:
            notification.name = stored_notification[0]
            notification.type = stored_notification[1]
            notification.address = stored_notification[2]
            notification.period = stored_notification[3]
            return notification
    except exceptions.DatabaseException:
        LOG.warn("Error querying mysql for notification method. "
                 "Using currently cached method.")
        return notification
    except Exception as e:
        LOG.warn(
            "Error when attempting to construct notification {0}".format(e))
        return None
    def test_smtp_sendmail_failed_connection_once(self, mock_smtp):
        """Email that fails on smtp_connect once
        """

        metrics = []
        metric_data = {'name': 'cpu.percent', 'dimensions': {'hostname': 'foo1', 'service': 'bar1'}}
        metrics.append(metric_data)
        metric_data = {'name': 'cpu.percent', 'dimensions': {'hostname': 'foo2', 'service': 'bar2'}}
        metrics.append(metric_data)

        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.debug = self.trap.append
        mock_log.info = self.trap.append
        mock_log.exception = self.trap.append

        mock_smtp.SMTP.return_value = mock_smtp
        mock_smtp.sendmail.side_effect = [smtplib.SMTPServerDisconnected, None]

        # There has to be a better way to preserve exception definitions when
        # we're mocking access to a library
        mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected
        mock_smtp.SMTPException = smtplib.SMTPException

        email = email_notifier.EmailNotifier(mock_log)

        email.config()

        alarm_dict = alarm(metrics)

        notification = Notification(0, 'email', 'email notification',
                                    '*****@*****.**', 0, 0, alarm_dict)

        self.trap.append(email.send_notification(notification))

        self.assertIn("SMTP server disconnected. Will reconnect and retry message.", self.trap)
        self.assertIn("Sent email to %s, notification %s"
                      % (notification.address, notification.to_json()), self.trap)
    def notify(self, smtp_stub, metric, mock_smtp):
        mock_smtp.SMTP = smtp_stub

        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append

        email = email_notifier.EmailNotifier(mock_log)

        alarm_dict = alarm(metric)

        notification = Notification(0, 'email', 'email notification',
                                    '*****@*****.**', 0, 0, alarm_dict)

        self.trap.append(email.send_notification(notification))
Esempio n. 18
0
    def email_setup(self, metric):
        alarm_dict = {
            "tenantId": "0",
            "alarmId": "0",
            "alarmName": "test Alarm",
            "oldState": "OK",
            "newState": "ALARM",
            "stateChangeReason": "I am alarming!",
            "timestamp": time.time(),
            "metrics": metric
        }

        notification = Notification('email', 0, 1, 'email notification',
                                    '*****@*****.**', 0, alarm_dict)

        self._start_processor([notification])
    def test_smtp_sendmail_failed_connection_once_then_email(self, mock_smtp):
        """Email that fails on smtp_connect once then email
        """

        metrics = []
        metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}}
        metrics.append(metric_data)
        metric_data = {'dimensions': {'hostname': 'foo2', 'service': 'bar2'}}
        metrics.append(metric_data)

        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.append
        mock_log.error = self.trap.append
        mock_log.debug = self.trap.append
        mock_log.info = self.trap.append
        mock_log.exception = self.trap.append

        mock_smtp.SMTP.return_value = mock_smtp

        mock_smtp.sendmail.side_effect = [
            smtplib.SMTPServerDisconnected, smtplib.SMTPException
        ]

        # There has to be a better way to preserve exception definitions when
        # we're mocking access to a library
        mock_smtp.SMTPServerDisconnected = smtplib.SMTPServerDisconnected
        mock_smtp.SMTPException = smtplib.SMTPException

        email = email_notifier.EmailNotifier(mock_log)

        email.config(self.email_config)

        alarm_dict = alarm(metrics)

        notification = Notification('email', 0, 1, 'email notification',
                                    '*****@*****.**', 0, alarm_dict)

        self.trap.append(email.send_notification(notification))

        self.assertIn(
            "SMTP server disconnected. Will reconnect and retry message.",
            self.trap)
        self.assertIn("Error sending Email Notification", self.trap)
        self.assertNotIn("Unable to connect to email server.", self.trap)
    def notify(self, http_func, mock_requests):
        mock_log = mock.MagicMock()
        mock_log.warn = self.trap.put
        mock_log.error = self.trap.put
        mock_log.exception = self.trap.put

        mock_requests.post = http_func

        webhook = webhook_notifier.WebhookNotifier(mock_log)

        webhook.config(self.webhook_config)

        metric = []
        metric_data = {'dimensions': {'hostname': 'foo1', 'service': 'bar1'}}
        metric.append(metric_data)

        alarm_dict = alarm(metric)

        notification = Notification('webhook', 0, 1, 'webhook notification',
                                    'http://mock:3333/', 0, 0, alarm_dict)

        self.trap.put(webhook.send_notification(notification))