def test_backfilling_gracefully_handles_invalid_json_contexts_without_blowing_up( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, broken_context_str, # Note: parametrizes the test ): record_ready_for_processing.context = broken_context_str record_ready_for_processing.save() sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary) try: sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) except (JSONDecodeError, KeyError, Job.DoesNotExist, Push.DoesNotExist): pytest.fail() record_ready_for_processing.refresh_from_db() assert record_ready_for_processing.status == BackfillRecord.FAILED assert not has_changed(sherlock_settings)
def test_records_change_to_ready_for_processing( test_perf_alert, create_record, record_from_mature_report, report_maintainer_mock, backfill_tool_mock, secretary, sherlock_settings, ): # create new report with records - the report will not be mature create_record(test_perf_alert) preliminary_records = BackfillRecord.objects.filter( status=BackfillRecord.PRELIMINARY) ready_records = BackfillRecord.objects.filter( status=BackfillRecord.READY_FOR_PROCESSING) frozen_reports = BackfillReport.objects.filter(frozen=True) assert preliminary_records.count() == 2 assert ready_records.count() == 0 assert frozen_reports.count() == 0 sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, ) sherlock.sheriff(since=EPOCH, frameworks=['raptor', 'talos'], repositories=['autoland']) assert preliminary_records.count() == 1 assert ready_records.count() == 1 assert frozen_reports.count() == 1
def test_email_is_still_sent_if_context_is_too_corrupt_to_be_actionable( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, broken_context_str, tc_notify_mock # Note: parametrizes the test ): record_ready_for_processing.context = broken_context_str record_ready_for_processing.save() sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, ) sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) assert BackfillNotificationRecord.objects.count() == 1 call_command('report_backfill_outcome') assert BackfillNotificationRecord.objects.count() == 0
def test_no_email_is_sent_for_untargeted_alerts( self, report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, notify_client_mock, framework, repository, ): sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, notify_client_mock, email_writer=self.email_writer_mock(), ) sherlock.sheriff( since=EPOCH, frameworks=[framework], repositories=[repository], ) record_ready_for_processing.refresh_from_db() assert notify_client_mock.email.call_count == 0
def test_email_is_still_sent_if_context_is_too_corrupt_to_be_actionable( self, report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, notify_client_mock, broken_context_str, # Note: parametrizes the test ): record_ready_for_processing.context = broken_context_str record_ready_for_processing.save() sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, notify_client_mock, ) sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) assert notify_client_mock.email.call_count == 1
def test_no_email_is_sent_for_untargeted_alerts( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, framework, repository, tc_notify_mock, ): sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, ) sherlock.sheriff( since=EPOCH, frameworks=[framework], repositories=[repository], ) record_ready_for_processing.refresh_from_db() assert BackfillNotificationRecord.objects.count() == 0 call_command('report_backfill_outcome') assert BackfillNotificationRecord.objects.count() == 0
def test_db_limits_update_if_backfills_left( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, notify_client_mock, ): targeted_platform = record_ready_for_processing.platform.platform initial_backfills = secretary.backfills_left(on_platform=targeted_platform) assert initial_backfills == json.loads( sherlock_settings.settings)['limits'][targeted_platform] sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, notify_client_mock) sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) record_ready_for_processing.refresh_from_db() assert record_ready_for_processing.status == BackfillRecord.BACKFILLED assert (initial_backfills - 4) == secretary.backfills_left(on_platform=targeted_platform)
def test_records_remain_unchanged_if_no_backfills_left( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, empty_sheriff_settings, ): sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary) sherlock._backfill(['test_talos'], [test_settings.TREEHERDER_TEST_REPOSITORY_NAME]) assert not has_changed(record_ready_for_processing)
def test_records_and_db_limits_remain_unchanged_if_no_records_suitable_for_backfill( report_maintainer_mock, backfill_tool_mock, secretary, sherlock_settings, record_unsuited_for_backfill, ): sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary) sherlock._backfill(['test_talos'], [test_settings.TREEHERDER_TEST_REPOSITORY_NAME]) assert not has_changed(record_unsuited_for_backfill) assert not has_changed(sherlock_settings)
def test_assert_can_run_throws_exception_when_runtime_exceeded( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, ): no_time_left = timedelta(seconds=0) sherlock_bot = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, no_time_left) with pytest.raises(MaxRuntimeExceeded): sherlock_bot.assert_can_run()
def test_assert_can_run_doesnt_throw_exception_when_enough_time_left( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, ): enough_time_left = timedelta(minutes=10) sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, enough_time_left) try: sherlock.assert_can_run() except MaxRuntimeExceeded: pytest.fail()
def sherlock_factory(days_to_lookup: timedelta) -> Sherlock: report_maintainer = __report_maintainer_factory(days_to_lookup) backfill_tool = backfill_tool_factory() secretary = Secretary() notify_client = notify_client_factory() return Sherlock(report_maintainer, backfill_tool, secretary, notify_client)
def test_records_and_db_limits_remain_unchanged_if_runtime_exceeded( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, ): no_time_left = timedelta(seconds=0) sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, no_time_left) try: sherlock.sheriff(since=EPOCH, frameworks=['raptor', 'talos'], repositories=['autoland']) except MaxRuntimeExceeded: pass assert not has_changed(record_ready_for_processing) assert not has_changed(sherlock_settings)
def test_no_email_is_sent_if_runtime_exceeded( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, tc_notify_mock, ): no_time_left = timedelta(seconds=0) sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, no_time_left) try: sherlock.sheriff(since=EPOCH, frameworks=['raptor', 'talos'], repositories=['autoland']) except MaxRuntimeExceeded: pass assert BackfillNotificationRecord.objects.count() == 0 call_command('report_backfill_outcome') assert BackfillNotificationRecord.objects.count() == 0
def test_no_email_is_sent_if_runtime_exceeded( self, report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, notify_client_mock, ): no_time_left = timedelta(seconds=0) sherlock = Sherlock(report_maintainer_mock, backfill_tool_mock, secretary, notify_client_mock, no_time_left) try: sherlock.sheriff(since=EPOCH, frameworks=['raptor', 'talos'], repositories=['autoland']) except MaxRuntimeExceeded: pass assert notify_client_mock.email.call_count == 0
def test_email_is_sent_after_successful_backfills( report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, tc_notify_mock, ): sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, ) sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) record_ready_for_processing.refresh_from_db() assert BackfillNotificationRecord.objects.count() == 1 call_command('report_backfill_outcome') assert BackfillNotificationRecord.objects.count() == 0
def test_email_is_sent_after_successful_backfills( self, report_maintainer_mock, backfill_tool_mock, secretary, record_ready_for_processing, sherlock_settings, notify_client_mock, ): sherlock = Sherlock( report_maintainer_mock, backfill_tool_mock, secretary, notify_client_mock, email_writer=self.email_writer_mock(), ) sherlock.sheriff( since=EPOCH, frameworks=['test_talos'], repositories=[test_settings.TREEHERDER_TEST_REPOSITORY_NAME], ) record_ready_for_processing.refresh_from_db() assert notify_client_mock.email.call_count == 1