def test_backfilling_gracefully_handles_invalid_json_contexts_without_blowing_up(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
broken_context_str, # Note: parametrizes the test
):
record_ready_for_processing.context = broken_context_str
record_ready_for_processing.save()
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary)
try:
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
except (JSONDecodeError, KeyError, Job.DoesNotExist, Push.DoesNotExist):
pytest.fail()
record_ready_for_processing.refresh_from_db()
assert record_ready_for_processing.status == BackfillRecord.FAILED
assert not has_changed(sherlock_settings)
def test_records_change_to_ready_for_processing(
test_perf_alert,
create_record,
record_from_mature_report,
report_maintainer_mock,
backfill_tool_mock,
secretary,
sherlock_settings,
):
# create new report with records - the report will not be mature
create_record(test_perf_alert)
preliminary_records = BackfillRecord.objects.filter(
status=BackfillRecord.PRELIMINARY)
ready_records = BackfillRecord.objects.filter(
status=BackfillRecord.READY_FOR_PROCESSING)
frozen_reports = BackfillReport.objects.filter(frozen=True)
assert preliminary_records.count() == 2
assert ready_records.count() == 0
assert frozen_reports.count() == 0
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
)
sherlock.sheriff(since=EPOCH,
frameworks=['raptor', 'talos'],
repositories=['autoland'])
assert preliminary_records.count() == 1
assert ready_records.count() == 1
assert frozen_reports.count() == 1
def test_email_is_still_sent_if_context_is_too_corrupt_to_be_actionable(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
broken_context_str,
tc_notify_mock
# Note: parametrizes the test
):
record_ready_for_processing.context = broken_context_str
record_ready_for_processing.save()
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
)
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
assert BackfillNotificationRecord.objects.count() == 1
call_command('report_backfill_outcome')
assert BackfillNotificationRecord.objects.count() == 0
def test_no_email_is_sent_for_untargeted_alerts(
self,
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
notify_client_mock,
framework,
repository,
):
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
notify_client_mock,
email_writer=self.email_writer_mock(),
)
sherlock.sheriff(
since=EPOCH,
frameworks=[framework],
repositories=[repository],
)
record_ready_for_processing.refresh_from_db()
assert notify_client_mock.email.call_count == 0
def test_email_is_still_sent_if_context_is_too_corrupt_to_be_actionable(
self,
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
notify_client_mock,
broken_context_str,
# Note: parametrizes the test
):
record_ready_for_processing.context = broken_context_str
record_ready_for_processing.save()
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
notify_client_mock,
)
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
assert notify_client_mock.email.call_count == 1
def test_no_email_is_sent_for_untargeted_alerts(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
framework,
repository,
tc_notify_mock,
):
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
)
sherlock.sheriff(
since=EPOCH,
frameworks=[framework],
repositories=[repository],
)
record_ready_for_processing.refresh_from_db()
assert BackfillNotificationRecord.objects.count() == 0
call_command('report_backfill_outcome')
assert BackfillNotificationRecord.objects.count() == 0
def test_db_limits_update_if_backfills_left(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
notify_client_mock,
):
targeted_platform = record_ready_for_processing.platform.platform
initial_backfills = secretary.backfills_left(on_platform=targeted_platform)
assert initial_backfills == json.loads(
sherlock_settings.settings)['limits'][targeted_platform]
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary,
notify_client_mock)
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
record_ready_for_processing.refresh_from_db()
assert record_ready_for_processing.status == BackfillRecord.BACKFILLED
assert (initial_backfills -
4) == secretary.backfills_left(on_platform=targeted_platform)
def test_records_remain_unchanged_if_no_backfills_left(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
empty_sheriff_settings,
):
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary)
sherlock._backfill(['test_talos'],
[test_settings.TREEHERDER_TEST_REPOSITORY_NAME])
assert not has_changed(record_ready_for_processing)
def test_records_and_db_limits_remain_unchanged_if_no_records_suitable_for_backfill(
report_maintainer_mock,
backfill_tool_mock,
secretary,
sherlock_settings,
record_unsuited_for_backfill,
):
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary)
sherlock._backfill(['test_talos'],
[test_settings.TREEHERDER_TEST_REPOSITORY_NAME])
assert not has_changed(record_unsuited_for_backfill)
assert not has_changed(sherlock_settings)
def test_assert_can_run_throws_exception_when_runtime_exceeded(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
):
no_time_left = timedelta(seconds=0)
sherlock_bot = Sherlock(report_maintainer_mock, backfill_tool_mock,
secretary, no_time_left)
with pytest.raises(MaxRuntimeExceeded):
sherlock_bot.assert_can_run()
def test_assert_can_run_doesnt_throw_exception_when_enough_time_left(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
):
enough_time_left = timedelta(minutes=10)
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary,
enough_time_left)
try:
sherlock.assert_can_run()
except MaxRuntimeExceeded:
pytest.fail()
def sherlock_factory(days_to_lookup: timedelta) -> Sherlock:
report_maintainer = __report_maintainer_factory(days_to_lookup)
backfill_tool = backfill_tool_factory()
secretary = Secretary()
notify_client = notify_client_factory()
return Sherlock(report_maintainer, backfill_tool, secretary, notify_client)
def test_records_and_db_limits_remain_unchanged_if_runtime_exceeded(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
):
no_time_left = timedelta(seconds=0)
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary,
no_time_left)
try:
sherlock.sheriff(since=EPOCH,
frameworks=['raptor', 'talos'],
repositories=['autoland'])
except MaxRuntimeExceeded:
pass
assert not has_changed(record_ready_for_processing)
assert not has_changed(sherlock_settings)
def test_no_email_is_sent_if_runtime_exceeded(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
tc_notify_mock,
):
no_time_left = timedelta(seconds=0)
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, no_time_left)
try:
sherlock.sheriff(since=EPOCH, frameworks=['raptor', 'talos'], repositories=['autoland'])
except MaxRuntimeExceeded:
pass
assert BackfillNotificationRecord.objects.count() == 0
call_command('report_backfill_outcome')
assert BackfillNotificationRecord.objects.count() == 0
def test_no_email_is_sent_if_runtime_exceeded(
self,
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
notify_client_mock,
):
no_time_left = timedelta(seconds=0)
sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock,
secretary, notify_client_mock, no_time_left)
try:
sherlock.sheriff(since=EPOCH,
frameworks=['raptor', 'talos'],
repositories=['autoland'])
except MaxRuntimeExceeded:
pass
assert notify_client_mock.email.call_count == 0
def test_email_is_sent_after_successful_backfills(
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
tc_notify_mock,
):
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
)
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
record_ready_for_processing.refresh_from_db()
assert BackfillNotificationRecord.objects.count() == 1
call_command('report_backfill_outcome')
assert BackfillNotificationRecord.objects.count() == 0
def test_email_is_sent_after_successful_backfills(
self,
report_maintainer_mock,
backfill_tool_mock,
secretary,
record_ready_for_processing,
sherlock_settings,
notify_client_mock,
):
sherlock = Sherlock(
report_maintainer_mock,
backfill_tool_mock,
secretary,
notify_client_mock,
email_writer=self.email_writer_mock(),
)
sherlock.sheriff(
since=EPOCH,
frameworks=['test_talos'],
repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME],
)
record_ready_for_processing.refresh_from_db()
assert notify_client_mock.email.call_count == 1
