async def notify(log_level: int = None) -> NoReturn:
"""Notify our users periodically of the number of red metrics."""
logging.getLogger().setLevel(log_level or logging.ERROR)
sleep_duration = int(os.environ.get("NOTIFIER_SLEEP_DURATION", 60))
api_version = "v3"
reports_url = (
f"http://{os.environ.get('SERVER_HOST', 'localhost')}:"
f"{os.environ.get('SERVER_PORT', '5001')}/api/{api_version}/reports")
data_model = await retrieve_data_model(api_version)
most_recent_measurement_seen = datetime.max.replace(tzinfo=timezone.utc)
outbox = Outbox()
notification_finder = NotificationFinder(data_model)
while True:
record_health()
logging.info("Determining notifications...")
try:
async with aiohttp.ClientSession(raise_for_status=True,
trust_env=True) as session:
response = await session.get(reports_url)
json = await response.json()
except Exception as reason: # pylint: disable=broad-except
logging.error("Could not get reports from %s: %s", reports_url,
reason)
json = dict(reports=[])
notifications = notification_finder.get_notifications(
json, most_recent_measurement_seen)
outbox.add_notifications(notifications)
outbox.send_notifications()
most_recent_measurement_seen = most_recent_measurement_timestamp(json)
logging.info("Sleeping %.1f seconds...", sleep_duration)
await asyncio.sleep(sleep_duration)
class StrategiesTests(Base):
"""Unit tests for the 'amount of new red metrics per report' notification strategy."""
def setUp(self):
"""Set variables for the other testcases."""
self.most_recent_measurement_seen = datetime.min.replace(
tzinfo=timezone.utc)
self.old_timestamp = "2019-01-01T00:00:00+00:00"
self.new_timestamp = "2020-01-01T00:00:00+00:00"
self.report_url = "https://report1"
self.white_metric_status = "unknown"
self.notification_finder = NotificationFinder(self.data_model)
count = dict(status="target_not_met", value="10")
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=count)
],
status_start=str(self.most_recent_measurement_seen),
)
def test_no_reports(self):
"""Test that there is nothing to notify when there are no reports."""
self.assertEqual([],
self.notification_finder.get_notifications(
dict(reports=[]),
self.most_recent_measurement_seen))
def test_no_red_metrics(self):
"""Test that there is nothing to notify when there are no red metrics."""
count = dict(status="target_met", value="0")
green_metric = self.metric(recent_measurements=[
dict(start=self.old_timestamp, end=self.new_timestamp, count=count)
])
subject1 = dict(metrics=dict(metric1=green_metric))
report1 = dict(report_uuid="report1",
title="report_title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_old_red_metric(self):
"""Test that there is nothing to notify if the red metric was already red."""
subject1 = dict(metrics=dict(metric1=self.red_metric))
report1 = dict(report_uuid="report1",
title="Title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_red_metric_without_recent_measurements(self):
"""Test that there is nothing to notify if the red metric has no recent measurements."""
red_metric1 = self.metric(status="target_not_met")
red_metric2 = self.metric(status="target_not_met")
subject1 = dict(metrics=dict(metric1=red_metric1, metric2=red_metric2))
report1 = dict(report_uuid="report1",
title="Title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_new_red_metric(self):
"""Test that a metric that has become red is included."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report1 = dict(
title="Title",
report_uuid="report1",
subjects=dict(subject1=subject1),
notification_destinations=dict(uuid1=dict(name="destination1")),
)
reports_json = dict(reports=[report1])
result = self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen)
self.assertEqual(
["metric1", "red (target not met)"],
[
result[0].metrics[0].metric_name,
result[0].metrics[0].new_metric_status
],
)
def test_new_red_metric_without_count_scale(self):
"""Test that a metric that doesn't have a count scale that became red is included."""
old_percentage = dict(status="target_met", value="5")
new_percentage = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="",
scale="percentage",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
percentage=old_percentage),
dict(start=self.new_timestamp,
end=self.new_timestamp,
percentage=new_percentage),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report1 = dict(
title="Title",
report_uuid="report1",
subjects=dict(subject1=subject1),
notification_destinations=dict(name="destination1"),
)
reports_json = dict(reports=[report1])
result = self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen)[0].metrics
self.assertEqual(
["Tests"],
[result[0].metric_name],
)
def test_recently_changed_metric_status(self):
"""Test that a metric that turns white is added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.new_timestamp,
count=dict(status="target_met")),
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status)),
],
)
self.assertTrue(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_new_measurement_same_status(self):
"""Test that a metric that was already white isn't added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.new_timestamp,
count=dict(status=self.white_metric_status)),
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status)),
],
)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_no_change_due_to_only_one_measurement(self):
"""Test that metrics with only one measurement (and therefore no changes in value) aren't added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status))
],
)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_no_change_due_to_no_measurements(self):
"""Test that metrics without measurements (and therefore no changes in value) aren't added."""
metric = self.metric(status=self.white_metric_status)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_multiple_reports_with_same_destination(self):
"""Test that the correct metrics are notified when multiple reports notify the same destination."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric1 = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric1))
report1 = dict(
title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1),
notification_destinations=dict(
uuid1=dict(url=self.report_url, name="destination1")),
)
red_metric2 = self.metric(
name="metric2",
status="target_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject2 = dict(metrics=dict(metric1=red_metric2))
report2 = dict(
title="Title",
report_uuid="report2",
webhook="webhook",
subjects=dict(subject1=subject2),
notification_destinations=dict(
uuid1=dict(url="https://report2", name="destination2")),
)
result = []
reports_json = dict(reports=[report1, report2])
for notification in self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen):
result.append(notification.metrics)
self.assertEqual(["metric1", "metric2"],
[result[0][0].metric_name, result[1][0].metric_name])
def test_no_notification_destinations_configured(self):
"""Test that no notification is to be sent if there are no configurations in notification destinations."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="metric1",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report = dict(
title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1),
notification_destinations={},
)
report_json = dict(reports=[report])
self.assertEqual([],
self.notification_finder.get_notifications(
report_json, self.most_recent_measurement_seen))
def test_no_notification_destinations_in_json(self):
"""Test that no notification is to be sent if notification destinations do not exist in the data."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="metric1",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report = dict(title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1))
report_json = dict(reports=[report])
self.assertEqual([],
self.notification_finder.get_notifications(
report_json, self.most_recent_measurement_seen))