async def notify(log_level: int = None) -> NoReturn: """Notify our users periodically of the number of red metrics.""" logging.getLogger().setLevel(log_level or logging.ERROR) sleep_duration = int(os.environ.get("NOTIFIER_SLEEP_DURATION", 60)) api_version = "v3" reports_url = ( f"http://{os.environ.get('SERVER_HOST', 'localhost')}:" f"{os.environ.get('SERVER_PORT', '5001')}/api/{api_version}/reports") data_model = await retrieve_data_model(api_version) most_recent_measurement_seen = datetime.max.replace(tzinfo=timezone.utc) outbox = Outbox() notification_finder = NotificationFinder(data_model) while True: record_health() logging.info("Determining notifications...") try: async with aiohttp.ClientSession(raise_for_status=True, trust_env=True) as session: response = await session.get(reports_url) json = await response.json() except Exception as reason: # pylint: disable=broad-except logging.error("Could not get reports from %s: %s", reports_url, reason) json = dict(reports=[]) notifications = notification_finder.get_notifications( json, most_recent_measurement_seen) outbox.add_notifications(notifications) outbox.send_notifications() most_recent_measurement_seen = most_recent_measurement_timestamp(json) logging.info("Sleeping %.1f seconds...", sleep_duration) await asyncio.sleep(sleep_duration)
class StrategiesTests(Base): """Unit tests for the 'amount of new red metrics per report' notification strategy.""" def setUp(self): """Set variables for the other testcases.""" self.most_recent_measurement_seen = datetime.min.replace( tzinfo=timezone.utc) self.old_timestamp = "2019-01-01T00:00:00+00:00" self.new_timestamp = "2020-01-01T00:00:00+00:00" self.report_url = "https://report1" self.white_metric_status = "unknown" self.notification_finder = NotificationFinder(self.data_model) count = dict(status="target_not_met", value="10") self.red_metric = self.metric( status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=count) ], status_start=str(self.most_recent_measurement_seen), ) def test_no_reports(self): """Test that there is nothing to notify when there are no reports.""" self.assertEqual([], self.notification_finder.get_notifications( dict(reports=[]), self.most_recent_measurement_seen)) def test_no_red_metrics(self): """Test that there is nothing to notify when there are no red metrics.""" count = dict(status="target_met", value="0") green_metric = self.metric(recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=count) ]) subject1 = dict(metrics=dict(metric1=green_metric)) report1 = dict(report_uuid="report1", title="report_title", subjects=dict(subject1=subject1)) reports_json = dict(reports=[report1]) self.assertEqual([], self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen)) def test_old_red_metric(self): """Test that there is nothing to notify if the red metric was already red.""" subject1 = dict(metrics=dict(metric1=self.red_metric)) report1 = dict(report_uuid="report1", title="Title", subjects=dict(subject1=subject1)) reports_json = dict(reports=[report1]) self.assertEqual([], self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen)) def test_red_metric_without_recent_measurements(self): """Test that there is nothing to notify if the red metric has no recent measurements.""" red_metric1 = self.metric(status="target_not_met") red_metric2 = self.metric(status="target_not_met") subject1 = dict(metrics=dict(metric1=red_metric1, metric2=red_metric2)) report1 = dict(report_uuid="report1", title="Title", subjects=dict(subject1=subject1)) reports_json = dict(reports=[report1]) self.assertEqual([], self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen)) def test_new_red_metric(self): """Test that a metric that has become red is included.""" old_count = dict(status="target_met", value="5") new_count = dict(status="target_not_met", value="10") red_metric = self.metric( status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=old_count), dict(start=self.new_timestamp, end=self.new_timestamp, count=new_count), ], ) subject1 = dict(metrics=dict(metric1=red_metric)) report1 = dict( title="Title", report_uuid="report1", subjects=dict(subject1=subject1), notification_destinations=dict(uuid1=dict(name="destination1")), ) reports_json = dict(reports=[report1]) result = self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen) self.assertEqual( ["metric1", "red (target not met)"], [ result[0].metrics[0].metric_name, result[0].metrics[0].new_metric_status ], ) def test_new_red_metric_without_count_scale(self): """Test that a metric that doesn't have a count scale that became red is included.""" old_percentage = dict(status="target_met", value="5") new_percentage = dict(status="target_not_met", value="10") red_metric = self.metric( name="", scale="percentage", status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, percentage=old_percentage), dict(start=self.new_timestamp, end=self.new_timestamp, percentage=new_percentage), ], ) subject1 = dict(metrics=dict(metric1=red_metric)) report1 = dict( title="Title", report_uuid="report1", subjects=dict(subject1=subject1), notification_destinations=dict(name="destination1"), ) reports_json = dict(reports=[report1]) result = self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen)[0].metrics self.assertEqual( ["Tests"], [result[0].metric_name], ) def test_recently_changed_metric_status(self): """Test that a metric that turns white is added.""" metric = self.metric( status=self.white_metric_status, recent_measurements=[ dict(start=self.new_timestamp, count=dict(status="target_met")), dict(start=self.old_timestamp, count=dict(status=self.white_metric_status)), ], ) self.assertTrue( self.notification_finder.status_changed( metric, self.most_recent_measurement_seen)) def test_new_measurement_same_status(self): """Test that a metric that was already white isn't added.""" metric = self.metric( status=self.white_metric_status, recent_measurements=[ dict(start=self.new_timestamp, count=dict(status=self.white_metric_status)), dict(start=self.old_timestamp, count=dict(status=self.white_metric_status)), ], ) self.assertFalse( self.notification_finder.status_changed( metric, self.most_recent_measurement_seen)) def test_no_change_due_to_only_one_measurement(self): """Test that metrics with only one measurement (and therefore no changes in value) aren't added.""" metric = self.metric( status=self.white_metric_status, recent_measurements=[ dict(start=self.old_timestamp, count=dict(status=self.white_metric_status)) ], ) self.assertFalse( self.notification_finder.status_changed( metric, self.most_recent_measurement_seen)) def test_no_change_due_to_no_measurements(self): """Test that metrics without measurements (and therefore no changes in value) aren't added.""" metric = self.metric(status=self.white_metric_status) self.assertFalse( self.notification_finder.status_changed( metric, self.most_recent_measurement_seen)) def test_multiple_reports_with_same_destination(self): """Test that the correct metrics are notified when multiple reports notify the same destination.""" old_count = dict(status="target_met", value="5") new_count = dict(status="target_not_met", value="10") red_metric1 = self.metric( status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=old_count), dict(start=self.new_timestamp, end=self.new_timestamp, count=new_count), ], ) subject1 = dict(metrics=dict(metric1=red_metric1)) report1 = dict( title="Title", report_uuid="report1", webhook="webhook", subjects=dict(subject1=subject1), notification_destinations=dict( uuid1=dict(url=self.report_url, name="destination1")), ) red_metric2 = self.metric( name="metric2", status="target_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=old_count), dict(start=self.new_timestamp, end=self.new_timestamp, count=new_count), ], ) subject2 = dict(metrics=dict(metric1=red_metric2)) report2 = dict( title="Title", report_uuid="report2", webhook="webhook", subjects=dict(subject1=subject2), notification_destinations=dict( uuid1=dict(url="https://report2", name="destination2")), ) result = [] reports_json = dict(reports=[report1, report2]) for notification in self.notification_finder.get_notifications( reports_json, self.most_recent_measurement_seen): result.append(notification.metrics) self.assertEqual(["metric1", "metric2"], [result[0][0].metric_name, result[1][0].metric_name]) def test_no_notification_destinations_configured(self): """Test that no notification is to be sent if there are no configurations in notification destinations.""" old_count = dict(status="target_met", value="5") new_count = dict(status="target_not_met", value="10") red_metric = self.metric( name="metric1", status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=old_count), dict(start=self.new_timestamp, end=self.new_timestamp, count=new_count), ], ) subject1 = dict(metrics=dict(metric1=red_metric)) report = dict( title="Title", report_uuid="report1", webhook="webhook", subjects=dict(subject1=subject1), notification_destinations={}, ) report_json = dict(reports=[report]) self.assertEqual([], self.notification_finder.get_notifications( report_json, self.most_recent_measurement_seen)) def test_no_notification_destinations_in_json(self): """Test that no notification is to be sent if notification destinations do not exist in the data.""" old_count = dict(status="target_met", value="5") new_count = dict(status="target_not_met", value="10") red_metric = self.metric( name="metric1", status="target_not_met", recent_measurements=[ dict(start=self.old_timestamp, end=self.new_timestamp, count=old_count), dict(start=self.new_timestamp, end=self.new_timestamp, count=new_count), ], ) subject1 = dict(metrics=dict(metric1=red_metric)) report = dict(title="Title", report_uuid="report1", webhook="webhook", subjects=dict(subject1=subject1)) report_json = dict(reports=[report]) self.assertEqual([], self.notification_finder.get_notifications( report_json, self.most_recent_measurement_seen))