async def notify(log_level: int = None) -> NoReturn:
"""Notify our users periodically of the number of red metrics."""
logging.getLogger().setLevel(log_level or logging.ERROR)
sleep_duration = int(os.environ.get("NOTIFIER_SLEEP_DURATION", 60))
api_version = "v3"
reports_url = (
f"http://{os.environ.get('SERVER_HOST', 'localhost')}:"
f"{os.environ.get('SERVER_PORT', '5001')}/api/{api_version}/reports")
data_model = await retrieve_data_model(api_version)
most_recent_measurement_seen = datetime.max.replace(tzinfo=timezone.utc)
outbox = Outbox()
notification_finder = NotificationFinder(data_model)
while True:
record_health()
logging.info("Determining notifications...")
try:
async with aiohttp.ClientSession(raise_for_status=True,
trust_env=True) as session:
response = await session.get(reports_url)
json = await response.json()
except Exception as reason: # pylint: disable=broad-except
logging.error("Could not get reports from %s: %s", reports_url,
reason)
json = dict(reports=[])
notifications = notification_finder.get_notifications(
json, most_recent_measurement_seen)
outbox.add_notifications(notifications)
outbox.send_notifications()
most_recent_measurement_seen = most_recent_measurement_timestamp(json)
logging.info("Sleeping %.1f seconds...", sleep_duration)
await asyncio.sleep(sleep_duration)
def setUp(self):
"""Set variables for the tests."""
self.old_timestamp = "2019-03-01T00:00:00+00:00"
self.new_timestamp = "2020-03-01T00:00:00+00:00"
self.metric_uuid = "metric_uuid"
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[],
status_start=str(datetime.fromisoformat(datetime.min.isoformat())),
)
self.notification_finder = NotificationFinder(self.data_model)
self.metric_with_recent_measurements = self.metric(
recent_measurements=[
dict(start=self.old_timestamp,
end=self.old_timestamp,
count=dict(status="target_not_met", value="10")),
dict(
start=self.old_timestamp,
end=self.new_timestamp,
count=dict(
status="target_not_met",
value="0",
status_start=self.old_timestamp,
),
),
],
status_start=self.old_timestamp,
)
class LongUnchangedTests(Base):
"""Unit tests for the 'status long unchanged' notification strategy."""
def setUp(self):
"""Override to set up a metric fixture."""
self.old_timestamp = "2019-02-01T00:00:00+00:00"
self.new_timestamp = "2020-02-01T00:00:00+00:00"
self.notification_finder = NotificationFinder(self.data_model)
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp, end=self.new_timestamp)
],
status_start=self.old_timestamp,
)
def test_long_unchanged_status(self):
"""Test that a metric is notable if its status has been the same for 3 weeks."""
now = datetime.now()
self.red_metric["status_start"] = str(now -
timedelta(days=21, hours=1))
self.assertEqual(
"status_long_unchanged",
self.notification_finder.get_notification(self.red_metric,
"red_metric", now))
def test_long_unchanged_status_already_notified(self):
"""Test that a metric is not notable if a notification has already been generated for the long status."""
now = datetime.now()
self.red_metric["status_start"] = str(now -
timedelta(days=21, hours=1))
self.notification_finder.already_notified.append("red_metric")
self.assertEqual(
None,
self.notification_finder.get_notification(self.red_metric,
"red_metric", now))
def test_too_long_unchanged_status(self):
"""Test that a metric isn't notable if its status has been the same for more than 3 weeks."""
now = datetime.now()
self.red_metric["status_start"] = str(now - timedelta(days=100))
self.assertEqual(
None,
self.notification_finder.get_notification(self.red_metric,
"red_metric", now))
def test_short_unchanged_status(self):
"""Test that a metric isn't notable if its current status has changed in the last 3 weeks."""
now = datetime.now()
self.red_metric["status_start"] = str(now -
timedelta(days=20, hours=23))
self.assertEqual(
None,
self.notification_finder.get_notification(self.red_metric,
"red_metric", now))
def setUp(self):
"""Override to set up a metric fixture."""
self.old_timestamp = "2019-02-01T00:00:00+00:00"
self.new_timestamp = "2020-02-01T00:00:00+00:00"
self.notification_finder = NotificationFinder(self.data_model)
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp, end=self.new_timestamp)
],
status_start=self.old_timestamp,
)
def setUp(self):
"""."""
self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
self.old_timestamp = "2019-02-01T00:23:59+59:00"
self.new_timestamp = "2020-02-01T00:23:59+59:00"
self.notification_finder = NotificationFinder(self.data_model)
count = dict(status="target_not_met", value="34")
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
)
def setUp(self):
"""Set variables for the other testcases."""
self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
self.old_timestamp = "2019-01-01T00:23:59+59:00"
self.new_timestamp = "2020-01-01T00:23:59+59:00"
self.report_url = "https://report1"
self.white_metric_status = "unknown"
self.notification_finder = NotificationFinder(self.data_model)
count = dict(status="target_not_met", value="10")
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
)
class CheckIfMetricIsNotableTestCase(Base):
"""Unit tests for the check_if_metric_is_notable method."""
def setUp(self):
"""Set variables for the tests."""
self.old_timestamp = "2019-03-01T00:00:00+00:00"
self.new_timestamp = "2020-03-01T00:00:00+00:00"
self.metric_uuid = "metric_uuid"
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[],
status_start=str(datetime.fromisoformat(datetime.min.isoformat())),
)
self.notification_finder = NotificationFinder(self.data_model)
self.metric_with_recent_measurements = self.metric(
recent_measurements=[
dict(start=self.old_timestamp,
end=self.old_timestamp,
count=dict(status="target_not_met", value="10")),
dict(
start=self.old_timestamp,
end=self.new_timestamp,
count=dict(
status="target_not_met",
value="0",
status_start=self.old_timestamp,
),
),
],
status_start=self.old_timestamp,
)
def test_metric_not_notable_if_no_recent_measurements(self):
"""Test that a metric is not notable if it does not contain any recent measurements."""
self.assertIsNone(
self.notification_finder.get_notification(
self.red_metric, self.metric_uuid,
datetime.min.replace(tzinfo=timezone.utc)))
def test_metric_is_notable_because_status_changed(self):
"""Test that a metric is notable because the status changed."""
metric = self.metric(recent_measurements=[
dict(start=self.old_timestamp,
end=self.old_timestamp,
count=dict(status="target_not_met", value="10")),
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=dict(status="target_met", value="0")),
])
self.assertEqual(
"status_changed",
self.notification_finder.get_notification(
metric, self.metric_uuid,
datetime.min.replace(tzinfo=timezone.utc)),
)
def test_metric_already_notified_is_removed_because_status_changed(self):
"""Test that the metric is removed from already notified when the status changes."""
metric = self.metric(recent_measurements=[
dict(start=self.old_timestamp,
end=self.old_timestamp,
count=dict(status="target_not_met", value="10")),
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=dict(status="target_met", value="0")),
])
self.notification_finder.already_notified.append(self.metric_uuid)
self.notification_finder.get_notification(
metric, self.metric_uuid,
datetime.min.replace(tzinfo=timezone.utc))
self.assertEqual(self.notification_finder.already_notified, [])
def test_metric_is_notable_because_status_unchanged_3weeks(self):
"""Test that a metric is notable for long unchanged status."""
self.assertEqual(
"status_long_unchanged",
self.notification_finder.get_notification(
self.metric_with_recent_measurements,
self.metric_uuid,
datetime.fromisoformat(self.old_timestamp) +
timedelta(days=21, hours=12),
),
)
def test_metric_not_notable_if_already_notified(self):
"""Test that a metric isn't notable for long unchanged status if a notification has already been generated."""
self.notification_finder.already_notified.append(self.metric_uuid)
self.assertIsNone(
self.notification_finder.get_notification(
self.metric_with_recent_measurements,
self.metric_uuid,
datetime.fromisoformat(self.old_timestamp) +
timedelta(days=21, hours=12),
))
class StrategiesTests(Base):
"""Unit tests for the 'amount of new red metrics per report' notification strategy."""
def setUp(self):
"""Set variables for the other testcases."""
self.most_recent_measurement_seen = datetime.min.replace(
tzinfo=timezone.utc)
self.old_timestamp = "2019-01-01T00:00:00+00:00"
self.new_timestamp = "2020-01-01T00:00:00+00:00"
self.report_url = "https://report1"
self.white_metric_status = "unknown"
self.notification_finder = NotificationFinder(self.data_model)
count = dict(status="target_not_met", value="10")
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=count)
],
status_start=str(self.most_recent_measurement_seen),
)
def test_no_reports(self):
"""Test that there is nothing to notify when there are no reports."""
self.assertEqual([],
self.notification_finder.get_notifications(
dict(reports=[]),
self.most_recent_measurement_seen))
def test_no_red_metrics(self):
"""Test that there is nothing to notify when there are no red metrics."""
count = dict(status="target_met", value="0")
green_metric = self.metric(recent_measurements=[
dict(start=self.old_timestamp, end=self.new_timestamp, count=count)
])
subject1 = dict(metrics=dict(metric1=green_metric))
report1 = dict(report_uuid="report1",
title="report_title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_old_red_metric(self):
"""Test that there is nothing to notify if the red metric was already red."""
subject1 = dict(metrics=dict(metric1=self.red_metric))
report1 = dict(report_uuid="report1",
title="Title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_red_metric_without_recent_measurements(self):
"""Test that there is nothing to notify if the red metric has no recent measurements."""
red_metric1 = self.metric(status="target_not_met")
red_metric2 = self.metric(status="target_not_met")
subject1 = dict(metrics=dict(metric1=red_metric1, metric2=red_metric2))
report1 = dict(report_uuid="report1",
title="Title",
subjects=dict(subject1=subject1))
reports_json = dict(reports=[report1])
self.assertEqual([],
self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen))
def test_new_red_metric(self):
"""Test that a metric that has become red is included."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report1 = dict(
title="Title",
report_uuid="report1",
subjects=dict(subject1=subject1),
notification_destinations=dict(uuid1=dict(name="destination1")),
)
reports_json = dict(reports=[report1])
result = self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen)
self.assertEqual(
["metric1", "red (target not met)"],
[
result[0].metrics[0].metric_name,
result[0].metrics[0].new_metric_status
],
)
def test_new_red_metric_without_count_scale(self):
"""Test that a metric that doesn't have a count scale that became red is included."""
old_percentage = dict(status="target_met", value="5")
new_percentage = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="",
scale="percentage",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
percentage=old_percentage),
dict(start=self.new_timestamp,
end=self.new_timestamp,
percentage=new_percentage),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report1 = dict(
title="Title",
report_uuid="report1",
subjects=dict(subject1=subject1),
notification_destinations=dict(name="destination1"),
)
reports_json = dict(reports=[report1])
result = self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen)[0].metrics
self.assertEqual(
["Tests"],
[result[0].metric_name],
)
def test_recently_changed_metric_status(self):
"""Test that a metric that turns white is added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.new_timestamp,
count=dict(status="target_met")),
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status)),
],
)
self.assertTrue(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_new_measurement_same_status(self):
"""Test that a metric that was already white isn't added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.new_timestamp,
count=dict(status=self.white_metric_status)),
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status)),
],
)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_no_change_due_to_only_one_measurement(self):
"""Test that metrics with only one measurement (and therefore no changes in value) aren't added."""
metric = self.metric(
status=self.white_metric_status,
recent_measurements=[
dict(start=self.old_timestamp,
count=dict(status=self.white_metric_status))
],
)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_no_change_due_to_no_measurements(self):
"""Test that metrics without measurements (and therefore no changes in value) aren't added."""
metric = self.metric(status=self.white_metric_status)
self.assertFalse(
self.notification_finder.status_changed(
metric, self.most_recent_measurement_seen))
def test_multiple_reports_with_same_destination(self):
"""Test that the correct metrics are notified when multiple reports notify the same destination."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric1 = self.metric(
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric1))
report1 = dict(
title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1),
notification_destinations=dict(
uuid1=dict(url=self.report_url, name="destination1")),
)
red_metric2 = self.metric(
name="metric2",
status="target_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject2 = dict(metrics=dict(metric1=red_metric2))
report2 = dict(
title="Title",
report_uuid="report2",
webhook="webhook",
subjects=dict(subject1=subject2),
notification_destinations=dict(
uuid1=dict(url="https://report2", name="destination2")),
)
result = []
reports_json = dict(reports=[report1, report2])
for notification in self.notification_finder.get_notifications(
reports_json, self.most_recent_measurement_seen):
result.append(notification.metrics)
self.assertEqual(["metric1", "metric2"],
[result[0][0].metric_name, result[1][0].metric_name])
def test_no_notification_destinations_configured(self):
"""Test that no notification is to be sent if there are no configurations in notification destinations."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="metric1",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report = dict(
title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1),
notification_destinations={},
)
report_json = dict(reports=[report])
self.assertEqual([],
self.notification_finder.get_notifications(
report_json, self.most_recent_measurement_seen))
def test_no_notification_destinations_in_json(self):
"""Test that no notification is to be sent if notification destinations do not exist in the data."""
old_count = dict(status="target_met", value="5")
new_count = dict(status="target_not_met", value="10")
red_metric = self.metric(
name="metric1",
status="target_not_met",
recent_measurements=[
dict(start=self.old_timestamp,
end=self.new_timestamp,
count=old_count),
dict(start=self.new_timestamp,
end=self.new_timestamp,
count=new_count),
],
)
subject1 = dict(metrics=dict(metric1=red_metric))
report = dict(title="Title",
report_uuid="report1",
webhook="webhook",
subjects=dict(subject1=subject1))
report_json = dict(reports=[report])
self.assertEqual([],
self.notification_finder.get_notifications(
report_json, self.most_recent_measurement_seen))
class LongUnchangedTestCase(Base):
"""Unit tests for the 'status long unchanged' notification strategy."""
def setUp(self):
"""."""
self.most_recent_measurement_seen = datetime.datetime.min.isoformat()
self.old_timestamp = "2019-02-01T00:23:59+59:00"
self.new_timestamp = "2020-02-01T00:23:59+59:00"
self.notification_finder = NotificationFinder(self.data_model)
count = dict(status="target_not_met", value="34")
self.red_metric = self.metric(
status="target_not_met",
recent_measurements=[dict(start=self.old_timestamp, end=self.new_timestamp, count=count)],
status_start=str(datetime.datetime.fromisoformat(self.most_recent_measurement_seen)),
)
@staticmethod
def metric(name="metric1", status="target_met", scale="count", recent_measurements=None, status_start=None):
"""Create a metric."""
return dict(
name=name,
scale=scale,
status=status,
type="tests",
unit="units",
recent_measurements=recent_measurements or [],
status_start=status_start or "",
)
def test_long_unchanged_status(self):
"""Test that metric is notable if it's status has been the same for 3 weeks."""
time_since_status_change = datetime.datetime.fromisoformat(
self.most_recent_measurement_seen
) + datetime.timedelta(days=21, hours=1)
self.assertTrue(
self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
)
def test_long_unchanged_status_already_notified(self):
"""Test that metric is not notable if a notification has already been generated for the long status."""
time_since_status_change = datetime.datetime.fromisoformat(
self.most_recent_measurement_seen
) + datetime.timedelta(days=21, hours=1)
self.notification_finder.already_notified.append("red_metric")
self.assertFalse(
self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
)
def test_too_long_unchanged_status(self):
"""Test that metric isn't notable if it's status has been the same for more than 3 weeks."""
time_since_status_change = datetime.datetime.fromisoformat(
self.most_recent_measurement_seen
) + datetime.timedelta(days=24, hours=1)
self.assertFalse(
self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
)
def test_short_unchanged_status(self):
"""Test that metric isn't notable if it's current status has been different in the last 3 weeks."""
time_since_status_change = datetime.datetime.fromisoformat(
self.most_recent_measurement_seen
) + datetime.timedelta(days=20, hours=23)
self.assertFalse(
self.notification_finder.long_unchanged_status(self.red_metric, "red_metric", str(time_since_status_change))
)