async def notify(log_level: int = None) -> NoReturn:
    """Notify our users periodically of the number of red metrics."""
    logging.getLogger().setLevel(log_level or logging.ERROR)
    sleep_duration = int(os.environ.get("NOTIFIER_SLEEP_DURATION", 60))
    api_version = "v3"
    reports_url = (
        f"http://{os.environ.get('SERVER_HOST', 'localhost')}:"
        f"{os.environ.get('SERVER_PORT', '5001')}/api/{api_version}/reports")
    data_model = await retrieve_data_model(api_version)
    most_recent_measurement_seen = datetime.max.replace(tzinfo=timezone.utc)
    outbox = Outbox()
    notification_finder = NotificationFinder(data_model)
    while True:
        record_health()
        logging.info("Determining notifications...")
        try:
            async with aiohttp.ClientSession(raise_for_status=True,
                                             trust_env=True) as session:
                response = await session.get(reports_url)
                json = await response.json()
        except Exception as reason:  # pylint: disable=broad-except
            logging.error("Could not get reports from %s: %s", reports_url,
                          reason)
            json = dict(reports=[])
        notifications = notification_finder.get_notifications(
            json, most_recent_measurement_seen)
        outbox.add_notifications(notifications)
        outbox.send_notifications()
        most_recent_measurement_seen = most_recent_measurement_timestamp(json)
        logging.info("Sleeping %.1f seconds...", sleep_duration)
        await asyncio.sleep(sleep_duration)
Exemple #2
0
 def setUp(self):
     """Set variables for the tests."""
     self.old_timestamp = "2019-03-01T00:00:00+00:00"
     self.new_timestamp = "2020-03-01T00:00:00+00:00"
     self.metric_uuid = "metric_uuid"
     self.red_metric = self.metric(
         status="target_not_met",
         recent_measurements=[],
         status_start=str(datetime.fromisoformat(datetime.min.isoformat())),
     )
     self.notification_finder = NotificationFinder(self.data_model)
     self.metric_with_recent_measurements = self.metric(
         recent_measurements=[
             dict(start=self.old_timestamp,
                  end=self.old_timestamp,
                  count=dict(status="target_not_met", value="10")),
             dict(
                 start=self.old_timestamp,
                 end=self.new_timestamp,
                 count=dict(
                     status="target_not_met",
                     value="0",
                     status_start=self.old_timestamp,
                 ),
             ),
         ],
         status_start=self.old_timestamp,
     )
Exemple #3
0
class LongUnchangedTests(Base):
    """Unit tests for the 'status long unchanged' notification strategy."""
    def setUp(self):
        """Override to set up a metric fixture."""
        self.old_timestamp = "2019-02-01T00:00:00+00:00"
        self.new_timestamp = "2020-02-01T00:00:00+00:00"
        self.notification_finder = NotificationFinder(self.data_model)
        self.red_metric = self.metric(
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp, end=self.new_timestamp)
            ],
            status_start=self.old_timestamp,
        )

    def test_long_unchanged_status(self):
        """Test that a metric is notable if its status has been the same for 3 weeks."""
        now = datetime.now()
        self.red_metric["status_start"] = str(now -
                                              timedelta(days=21, hours=1))
        self.assertEqual(
            "status_long_unchanged",
            self.notification_finder.get_notification(self.red_metric,
                                                      "red_metric", now))

    def test_long_unchanged_status_already_notified(self):
        """Test that a metric is not notable if a notification has already been generated for the long status."""
        now = datetime.now()
        self.red_metric["status_start"] = str(now -
                                              timedelta(days=21, hours=1))
        self.notification_finder.already_notified.append("red_metric")
        self.assertEqual(
            None,
            self.notification_finder.get_notification(self.red_metric,
                                                      "red_metric", now))

    def test_too_long_unchanged_status(self):
        """Test that a metric isn't notable if its status has been the same for more than 3 weeks."""
        now = datetime.now()
        self.red_metric["status_start"] = str(now - timedelta(days=100))
        self.assertEqual(
            None,
            self.notification_finder.get_notification(self.red_metric,
                                                      "red_metric", now))

    def test_short_unchanged_status(self):
        """Test that a metric isn't notable if its current status has changed in the last 3 weeks."""
        now = datetime.now()
        self.red_metric["status_start"] = str(now -
                                              timedelta(days=20, hours=23))
        self.assertEqual(
            None,
            self.notification_finder.get_notification(self.red_metric,
                                                      "red_metric", now))
Exemple #4
0
 def setUp(self):
     """Override to set up a metric fixture."""
     self.old_timestamp = "2019-02-01T00:00:00+00:00"
     self.new_timestamp = "2020-02-01T00:00:00+00:00"
     self.notification_finder = NotificationFinder(self.data_model)
     self.red_metric = self.metric(
         status="target_not_met",
         recent_measurements=[
             dict(start=self.old_timestamp, end=self.new_timestamp)
         ],
         status_start=self.old_timestamp,
     )
Exemple #5
0
 def setUp(self):
     """."""
     self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
     self.old_timestamp = "2019-02-01T00:23:59+59:00"
     self.new_timestamp = "2020-02-01T00:23:59+59:00"
     self.notification_finder = NotificationFinder(self.data_model)
     count = dict(status="target_not_met", value="34")
     self.red_metric = self.metric(
         status="target_not_met",
         recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
         status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
     )
Exemple #6
0
 def setUp(self):
     """Set variables for the other testcases."""
     self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
     self.old_timestamp = "2019-01-01T00:23:59+59:00"
     self.new_timestamp = "2020-01-01T00:23:59+59:00"
     self.report_url = "https://report1"
     self.white_metric_status = "unknown"
     self.notification_finder = NotificationFinder(self.data_model)
     count = dict(status="target_not_met", value="10")
     self.red_metric = self.metric(
         status="target_not_met",
         recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
         status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
     )
Exemple #7
0
class CheckIfMetricIsNotableTestCase(Base):
    """Unit tests for the check_if_metric_is_notable method."""
    def setUp(self):
        """Set variables for the tests."""
        self.old_timestamp = "2019-03-01T00:00:00+00:00"
        self.new_timestamp = "2020-03-01T00:00:00+00:00"
        self.metric_uuid = "metric_uuid"
        self.red_metric = self.metric(
            status="target_not_met",
            recent_measurements=[],
            status_start=str(datetime.fromisoformat(datetime.min.isoformat())),
        )
        self.notification_finder = NotificationFinder(self.data_model)
        self.metric_with_recent_measurements = self.metric(
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.old_timestamp,
                     count=dict(status="target_not_met", value="10")),
                dict(
                    start=self.old_timestamp,
                    end=self.new_timestamp,
                    count=dict(
                        status="target_not_met",
                        value="0",
                        status_start=self.old_timestamp,
                    ),
                ),
            ],
            status_start=self.old_timestamp,
        )

    def test_metric_not_notable_if_no_recent_measurements(self):
        """Test that a metric is not notable if it does not contain any recent measurements."""
        self.assertIsNone(
            self.notification_finder.get_notification(
                self.red_metric, self.metric_uuid,
                datetime.min.replace(tzinfo=timezone.utc)))

    def test_metric_is_notable_because_status_changed(self):
        """Test that a metric is notable because the status changed."""
        metric = self.metric(recent_measurements=[
            dict(start=self.old_timestamp,
                 end=self.old_timestamp,
                 count=dict(status="target_not_met", value="10")),
            dict(start=self.old_timestamp,
                 end=self.new_timestamp,
                 count=dict(status="target_met", value="0")),
        ])
        self.assertEqual(
            "status_changed",
            self.notification_finder.get_notification(
                metric, self.metric_uuid,
                datetime.min.replace(tzinfo=timezone.utc)),
        )

    def test_metric_already_notified_is_removed_because_status_changed(self):
        """Test that the metric is removed from already notified when the status changes."""
        metric = self.metric(recent_measurements=[
            dict(start=self.old_timestamp,
                 end=self.old_timestamp,
                 count=dict(status="target_not_met", value="10")),
            dict(start=self.old_timestamp,
                 end=self.new_timestamp,
                 count=dict(status="target_met", value="0")),
        ])
        self.notification_finder.already_notified.append(self.metric_uuid)
        self.notification_finder.get_notification(
            metric, self.metric_uuid,
            datetime.min.replace(tzinfo=timezone.utc))
        self.assertEqual(self.notification_finder.already_notified, [])

    def test_metric_is_notable_because_status_unchanged_3weeks(self):
        """Test that a metric is notable for long unchanged status."""
        self.assertEqual(
            "status_long_unchanged",
            self.notification_finder.get_notification(
                self.metric_with_recent_measurements,
                self.metric_uuid,
                datetime.fromisoformat(self.old_timestamp) +
                timedelta(days=21, hours=12),
            ),
        )

    def test_metric_not_notable_if_already_notified(self):
        """Test that a metric isn't notable for long unchanged status if a notification has already been generated."""
        self.notification_finder.already_notified.append(self.metric_uuid)
        self.assertIsNone(
            self.notification_finder.get_notification(
                self.metric_with_recent_measurements,
                self.metric_uuid,
                datetime.fromisoformat(self.old_timestamp) +
                timedelta(days=21, hours=12),
            ))
Exemple #8
0
class StrategiesTests(Base):
    """Unit tests for the 'amount of new red metrics per report' notification strategy."""
    def setUp(self):
        """Set variables for the other testcases."""
        self.most_recent_measurement_seen = datetime.min.replace(
            tzinfo=timezone.utc)
        self.old_timestamp = "2019-01-01T00:00:00+00:00"
        self.new_timestamp = "2020-01-01T00:00:00+00:00"
        self.report_url = "https://report1"
        self.white_metric_status = "unknown"
        self.notification_finder = NotificationFinder(self.data_model)
        count = dict(status="target_not_met", value="10")
        self.red_metric = self.metric(
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=count)
            ],
            status_start=str(self.most_recent_measurement_seen),
        )

    def test_no_reports(self):
        """Test that there is nothing to notify when there are no reports."""
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             dict(reports=[]),
                             self.most_recent_measurement_seen))

    def test_no_red_metrics(self):
        """Test that there is nothing to notify when there are no red metrics."""
        count = dict(status="target_met", value="0")
        green_metric = self.metric(recent_measurements=[
            dict(start=self.old_timestamp, end=self.new_timestamp, count=count)
        ])
        subject1 = dict(metrics=dict(metric1=green_metric))
        report1 = dict(report_uuid="report1",
                       title="report_title",
                       subjects=dict(subject1=subject1))
        reports_json = dict(reports=[report1])
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             reports_json, self.most_recent_measurement_seen))

    def test_old_red_metric(self):
        """Test that there is nothing to notify if the red metric was already red."""
        subject1 = dict(metrics=dict(metric1=self.red_metric))
        report1 = dict(report_uuid="report1",
                       title="Title",
                       subjects=dict(subject1=subject1))
        reports_json = dict(reports=[report1])
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             reports_json, self.most_recent_measurement_seen))

    def test_red_metric_without_recent_measurements(self):
        """Test that there is nothing to notify if the red metric has no recent measurements."""
        red_metric1 = self.metric(status="target_not_met")
        red_metric2 = self.metric(status="target_not_met")
        subject1 = dict(metrics=dict(metric1=red_metric1, metric2=red_metric2))
        report1 = dict(report_uuid="report1",
                       title="Title",
                       subjects=dict(subject1=subject1))
        reports_json = dict(reports=[report1])
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             reports_json, self.most_recent_measurement_seen))

    def test_new_red_metric(self):
        """Test that a metric that has become red is included."""
        old_count = dict(status="target_met", value="5")
        new_count = dict(status="target_not_met", value="10")
        red_metric = self.metric(
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=old_count),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     count=new_count),
            ],
        )
        subject1 = dict(metrics=dict(metric1=red_metric))
        report1 = dict(
            title="Title",
            report_uuid="report1",
            subjects=dict(subject1=subject1),
            notification_destinations=dict(uuid1=dict(name="destination1")),
        )
        reports_json = dict(reports=[report1])
        result = self.notification_finder.get_notifications(
            reports_json, self.most_recent_measurement_seen)
        self.assertEqual(
            ["metric1", "red (target not met)"],
            [
                result[0].metrics[0].metric_name,
                result[0].metrics[0].new_metric_status
            ],
        )

    def test_new_red_metric_without_count_scale(self):
        """Test that a metric that doesn't have a count scale that became red is included."""
        old_percentage = dict(status="target_met", value="5")
        new_percentage = dict(status="target_not_met", value="10")
        red_metric = self.metric(
            name="",
            scale="percentage",
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     percentage=old_percentage),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     percentage=new_percentage),
            ],
        )
        subject1 = dict(metrics=dict(metric1=red_metric))
        report1 = dict(
            title="Title",
            report_uuid="report1",
            subjects=dict(subject1=subject1),
            notification_destinations=dict(name="destination1"),
        )
        reports_json = dict(reports=[report1])
        result = self.notification_finder.get_notifications(
            reports_json, self.most_recent_measurement_seen)[0].metrics
        self.assertEqual(
            ["Tests"],
            [result[0].metric_name],
        )

    def test_recently_changed_metric_status(self):
        """Test that a metric that turns white is added."""
        metric = self.metric(
            status=self.white_metric_status,
            recent_measurements=[
                dict(start=self.new_timestamp,
                     count=dict(status="target_met")),
                dict(start=self.old_timestamp,
                     count=dict(status=self.white_metric_status)),
            ],
        )
        self.assertTrue(
            self.notification_finder.status_changed(
                metric, self.most_recent_measurement_seen))

    def test_new_measurement_same_status(self):
        """Test that a metric that was already white isn't added."""
        metric = self.metric(
            status=self.white_metric_status,
            recent_measurements=[
                dict(start=self.new_timestamp,
                     count=dict(status=self.white_metric_status)),
                dict(start=self.old_timestamp,
                     count=dict(status=self.white_metric_status)),
            ],
        )
        self.assertFalse(
            self.notification_finder.status_changed(
                metric, self.most_recent_measurement_seen))

    def test_no_change_due_to_only_one_measurement(self):
        """Test that metrics with only one measurement (and therefore no changes in value) aren't added."""
        metric = self.metric(
            status=self.white_metric_status,
            recent_measurements=[
                dict(start=self.old_timestamp,
                     count=dict(status=self.white_metric_status))
            ],
        )
        self.assertFalse(
            self.notification_finder.status_changed(
                metric, self.most_recent_measurement_seen))

    def test_no_change_due_to_no_measurements(self):
        """Test that metrics without measurements (and therefore no changes in value) aren't added."""
        metric = self.metric(status=self.white_metric_status)
        self.assertFalse(
            self.notification_finder.status_changed(
                metric, self.most_recent_measurement_seen))

    def test_multiple_reports_with_same_destination(self):
        """Test that the correct metrics are notified when multiple reports notify the same destination."""
        old_count = dict(status="target_met", value="5")
        new_count = dict(status="target_not_met", value="10")

        red_metric1 = self.metric(
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=old_count),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     count=new_count),
            ],
        )
        subject1 = dict(metrics=dict(metric1=red_metric1))
        report1 = dict(
            title="Title",
            report_uuid="report1",
            webhook="webhook",
            subjects=dict(subject1=subject1),
            notification_destinations=dict(
                uuid1=dict(url=self.report_url, name="destination1")),
        )

        red_metric2 = self.metric(
            name="metric2",
            status="target_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=old_count),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     count=new_count),
            ],
        )
        subject2 = dict(metrics=dict(metric1=red_metric2))
        report2 = dict(
            title="Title",
            report_uuid="report2",
            webhook="webhook",
            subjects=dict(subject1=subject2),
            notification_destinations=dict(
                uuid1=dict(url="https://report2", name="destination2")),
        )
        result = []
        reports_json = dict(reports=[report1, report2])
        for notification in self.notification_finder.get_notifications(
                reports_json, self.most_recent_measurement_seen):
            result.append(notification.metrics)
        self.assertEqual(["metric1", "metric2"],
                         [result[0][0].metric_name, result[1][0].metric_name])

    def test_no_notification_destinations_configured(self):
        """Test that no notification is to be sent if there are no configurations in notification destinations."""
        old_count = dict(status="target_met", value="5")
        new_count = dict(status="target_not_met", value="10")
        red_metric = self.metric(
            name="metric1",
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=old_count),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     count=new_count),
            ],
        )
        subject1 = dict(metrics=dict(metric1=red_metric))
        report = dict(
            title="Title",
            report_uuid="report1",
            webhook="webhook",
            subjects=dict(subject1=subject1),
            notification_destinations={},
        )
        report_json = dict(reports=[report])
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             report_json, self.most_recent_measurement_seen))

    def test_no_notification_destinations_in_json(self):
        """Test that no notification is to be sent if notification destinations do not exist in the data."""
        old_count = dict(status="target_met", value="5")
        new_count = dict(status="target_not_met", value="10")
        red_metric = self.metric(
            name="metric1",
            status="target_not_met",
            recent_measurements=[
                dict(start=self.old_timestamp,
                     end=self.new_timestamp,
                     count=old_count),
                dict(start=self.new_timestamp,
                     end=self.new_timestamp,
                     count=new_count),
            ],
        )
        subject1 = dict(metrics=dict(metric1=red_metric))
        report = dict(title="Title",
                      report_uuid="report1",
                      webhook="webhook",
                      subjects=dict(subject1=subject1))
        report_json = dict(reports=[report])
        self.assertEqual([],
                         self.notification_finder.get_notifications(
                             report_json, self.most_recent_measurement_seen))
Exemple #9
0
class LongUnchangedTestCase(Base):
    """Unit tests for the 'status long unchanged' notification strategy."""

    def setUp(self):
        """."""
        self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
        self.old_timestamp = "2019-02-01T00:23:59+59:00"
        self.new_timestamp = "2020-02-01T00:23:59+59:00"
        self.notification_finder = NotificationFinder(self.data_model)
        count = dict(status="target_not_met", value="34")
        self.red_metric = self.metric(
            status="target_not_met",
            recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
            status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
        )

    @staticmethod
    def metric(name="metric1", status="target_met", scale="count", recent_measurements=None, status_start=None):
        """Create a metric."""
        return dict(
            name=name,
            scale=scale,
            status=status,
            type="tests",
            unit="units",
            recent_measurements=recent_measurements or [],
            status_start=status_start or "",
        )

    def test_long_unchanged_status(self):
        """Test that metric is notable if it's status has been the same for 3 weeks."""
        time_since_status_change = datetime.datetime.fromisoformat(
            self.most_recent_measurement_seen
        ) + datetime.timedelta(days=21, hours=1)
        self.assertTrue(
            self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
        )

    def test_long_unchanged_status_already_notified(self):
        """Test that metric is not notable if a notification has already been generated for the long status."""
        time_since_status_change = datetime.datetime.fromisoformat(
            self.most_recent_measurement_seen
        ) + datetime.timedelta(days=21, hours=1)
        self.notification_finder.already_notified.append("red_metric")
        self.assertFalse(
            self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
        )

    def test_too_long_unchanged_status(self):
        """Test that metric isn't notable if it's status has been the same for more than 3 weeks."""
        time_since_status_change = datetime.datetime.fromisoformat(
            self.most_recent_measurement_seen
        ) + datetime.timedelta(days=24, hours=1)
        self.assertFalse(
            self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
        )

    def test_short_unchanged_status(self):
        """Test that metric isn't notable if it's current status has been different in the last 3 weeks."""
        time_since_status_change = datetime.datetime.fromisoformat(
            self.most_recent_measurement_seen
        ) + datetime.timedelta(days=20, hours=23)
        self.assertFalse(
            self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
        )