def testGenerateCommitIDsForBoundingTargetsQueryGit(self, mock_revisions): data_points = [ DataPoint.Create(commit_position=1010, git_hash='r1010'), DataPoint.Create(commit_position=1000, git_hash='r1000'), ] mock_revisions.return_value = { 1003: 'r1003', 1004: 'r1004', 1005: 'r1005', 1006: 'r1006', 1007: 'r1007', 1008: 'r1008', 1009: 'r1009', 1010: 'r1010' } lower_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '', '', '', '', '', '', 1003, None) upper_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '', '', '', '', '', '', 1008, None) lower_bound_commit_id = CommitID(commit_position=1003, revision='r1003') upper_bound_commit_id = CommitID(commit_position=1008, revision='r1008') self.assertEqual( (lower_bound_commit_id, upper_bound_commit_id), next_commit_position_utils.GenerateCommitIDsForBoundingTargets( data_points, lower_bound_target, upper_bound_target)) mock_revisions.assert_called_once_with('r1010', 1010, 1003)
def testGetLowestUpperBoundBuildNumber(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(build_number=123, commit_position=1000), DataPoint.Create(build_number=122, commit_position=990) ] self.assertEqual(122, analysis.GetLowestUpperBoundBuildNumber(900))
def testUpdateSuspectedBuild(self, mock_build_number): build_number = 120 build_id = 1200 mock_build_number.return_value = build_number lower_bound_commit_position = 90 upper_bound_commit_position = 100 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=upper_bound_commit_position, pass_rate=0.4), DataPoint.Create(commit_position=lower_bound_commit_position, pass_rate=1.0), ] analysis.Save() lower_bound_target = IsolatedTarget.Create( build_id - 1, '', '', 'm', 'b', '', '', '', '', '', '', lower_bound_commit_position, '') upper_bound_target = IsolatedTarget.Create( build_id, '', '', 'm', 'b', '', '', '', '', '', '', upper_bound_commit_position, '') analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target) self.assertEqual(build_id, analysis.suspected_build_id) self.assertEqual(build_number, analysis.suspected_flake_build_number)
def testSplitIndexNotExists(self, mocked_steppiness): data_points = [ DataPoint(build_number=99, pass_rate=0.95), DataPoint(build_number=100, pass_rate=0.9), ] self.assertRaises(AssertionError, confidence._Steppiness, data_points, lambda x: x.build_number, 90) self.assertFalse(mocked_steppiness.called)
def testGetLatestDataPointNoRecentFlakinessPoints(self): expected_data_point = DataPoint.Create(commit_position=1000) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ expected_data_point, DataPoint.Create(commit_position=990) ] self.assertEqual(expected_data_point, analysis.GetLatestDataPoint())
def testNotEnoughData(self, mocked_steppiness): data_points = [ DataPoint(build_number=99, pass_rate=0.95), DataPoint(build_number=100, pass_rate=0.9), ] steppiness = confidence._Steppiness(data_points, lambda x: x.build_number, 100) self.assertEqual(0, steppiness) self.assertFalse(mocked_steppiness.called)
def testCreateAndSave(self): gitiles_project = 'chromium/src' bucket = 'ci' builder_name = 'b' step_name = 's' test_name = 't' git_hash = 'a1b2c3' legacy_master_name = 'm' build_number = 123 build_url = 'build_url' pass_rate = 0.5 task_ids = ['task'] commit_position = 1000 try_job_url = 'try_job_url' iterations = 100 elapsed_seconds = 300 error = None commit_timestamp = datetime(2018, 12, 4) failed_swarming_task_attempts = 0 data_point = DataPoint.CreateAndSave( gitiles_project, bucket, builder_name, step_name, test_name, git_hash, legacy_master_name=legacy_master_name, build_number=build_number, build_url=build_url, pass_rate=pass_rate, task_ids=task_ids, commit_position=commit_position, try_job_url=try_job_url, iterations=iterations, elapsed_seconds=elapsed_seconds, error=error, commit_timestamp=commit_timestamp, failed_swarming_task_attempts=failed_swarming_task_attempts) self.assertEqual( data_point, DataPoint.Get(gitiles_project, bucket, builder_name, step_name, test_name, git_hash)) self.assertEqual(legacy_master_name, data_point.legacy_master_name) self.assertEqual(build_number, data_point.build_number) self.assertEqual(build_url, data_point.build_url) self.assertEqual(pass_rate, data_point.pass_rate) self.assertEqual(task_ids, data_point.task_ids) self.assertEqual(commit_position, data_point.commit_position) self.assertEqual(try_job_url, data_point.try_job_url) self.assertEqual(iterations, data_point.iterations) self.assertEqual(elapsed_seconds, data_point.elapsed_seconds) self.assertEqual(error, data_point.error) self.assertEqual(commit_timestamp, data_point.commit_timestamp) self.assertEqual(failed_swarming_task_attempts, data_point.failed_swarming_task_attempts)
def testPaddingDataPoints(self, mocked_steppiness): data_points = [ DataPoint(build_number=99, pass_rate=-1), DataPoint(build_number=100, pass_rate=0.5), ] mocked_steppiness.side_effect = [1] steppiness = confidence._Steppiness(data_points, lambda x: x.build_number, 100) self.assertEqual(1, steppiness) mocked_steppiness.assert_called_once_with([1, 1, 1, 1, 0.5], 4)
def testShouldUpdateBugForAnalysisNoBugId(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 1, 's', 't') analysis.status = analysis_status.COMPLETED analysis.data_points = [DataPoint(), DataPoint(), DataPoint()] analysis.confidence_in_culprit = 0.9 self.UpdateUnitTestConfigSettings( 'action_settings', {'minimum_confidence_score_to_update_endpoints': 0.6}) self.assertFalse(flake_bug_util.ShouldUpdateBugForAnalysis(analysis))
def testRemoveDataPointWithCommitPosition(self): data_points = [ DataPoint.Create(build_number=100, commit_position=1000), DataPoint.Create(build_number=101, commit_position=1100), DataPoint.Create(build_number=110, commit_position=2000) ] analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = data_points analysis.RemoveDataPointWithCommitPosition(2000) self.assertEqual(data_points[:-1], analysis.data_points)
def testGetPassCount(self): self.assertEqual( 0, DataPoint.Create(pass_rate=0.0, iterations=100).GetPassCount()) self.assertEqual( 10, DataPoint.Create(pass_rate=0.5, iterations=20).GetPassCount()) self.assertEqual( 429, DataPoint.Create(pass_rate=0.9976744186046511, iterations=430).GetPassCount())
def testCalculateCulpritConfidenceScoreIntroducedNewFlakyTest(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.7, commit_position=1000), DataPoint.Create( pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, commit_position=999) ] self.assertEqual( 1.0, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testGetDataPointOfSuspectedBuildNoDataPoint(self): # This scenario should not happen. expected_build_number = 123 unexpected_build_number = 124 data_point = DataPoint() data_point.build_number = expected_build_number analysis = MasterFlakeAnalysis.Create('m', 'b', 125, 's', 't') analysis.suspected_flake_build_number = unexpected_build_number analysis.data_points.append(data_point) self.assertIsNone(analysis.GetDataPointOfSuspectedBuild())
def testGetDataPointOfSuspectedBuild(self): expected_build_number = 123 data_point = DataPoint() data_point.build_number = expected_build_number analysis = MasterFlakeAnalysis.Create('m', 'b', 125, 's', 't') analysis.suspected_flake_build_number = expected_build_number analysis.data_points.append(data_point) suspected_data_point = analysis.GetDataPointOfSuspectedBuild() self.assertEqual(expected_build_number, suspected_data_point.build_number)
def testCommitPosition(self, mocked_steppiness): data_points = [ DataPoint(commit_position=90, pass_rate=1), DataPoint(commit_position=94, pass_rate=1), DataPoint(commit_position=97, pass_rate=1), DataPoint(commit_position=99, pass_rate=1), DataPoint(commit_position=100, pass_rate=1), ] mocked_steppiness.side_effect = [0] steppiness = confidence.SteppinessForCommitPosition(data_points, 99) self.assertEqual(0, steppiness) mocked_steppiness.assert_called_once_with([1, 1, 1, 1, 1], 3)
def testGetLatestRegressionRangeNoUpperBoundMultipleDataPoints(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='rev90') ] self.assertEqual( CommitIDRange(lower=None, upper=CommitID(commit_position=90, revision='rev90')), analysis.GetLatestRegressionRange())
def testNewlyAddedTest(self): culprit_commit_position = 11 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create( pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, commit_position=culprit_commit_position - 1), DataPoint.Create( pass_rate=0.5, commit_position=culprit_commit_position), ] self.assertTrue( culprit_util.CulpritAddedNewFlakyTest(analysis, culprit_commit_position))
def testIdentifySuspectedRangesFailedToGetBlame(self, _, mock_test_location): mock_test_location.return_value = TestLocation(file='a/b.cc', line=1) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash='r1000', pass_rate=0.5), DataPoint.Create(commit_position=997, git_hash='r997', pass_rate=1.0) ] analysis.Save() self.assertEqual([], heuristic_analysis.IdentifySuspectedRevisions(analysis))
def testShouldTakeAutoAction(self): culprit_commit_position = 1000 culprit = FlakeCulprit.Create('chromium', 'r', culprit_commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.data_points = [ DataPoint.Create(commit_position=culprit_commit_position, pass_rate=0.5), DataPoint.Create(commit_position=culprit_commit_position - 1, pass_rate=1.0) ] self.assertTrue( flake_analysis_util.ShouldTakeAutoAction(analysis, False))
def testGetNextCommitIdFromHeuristicResultsExistingDataPoints(self): suspect = FlakeCulprit.Create('repo', 'revision', 1000) suspect.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe()) analysis.data_points = [ DataPoint.Create(commit_position=999, pass_rate=1.0), DataPoint.Create(commit_position=1000, pass_rate=1.0) ] analysis.put() self.assertIsNone( next_commit_position_utils.GetNextCommitIdFromHeuristicResults( analysis.key.urlsafe()))
def testCalculateCulpritConfidenceScoreFallbackToSteppiness(self, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.7, iterations=20, commit_position=1000), DataPoint.Create(pass_rate=1.0, iterations=400, commit_position=999), ] self.assertEqual( .6, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testLookbackAlgorithmExponentialLookback(self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), ] # Step size 10, rounded up to the next square == 16. next_commit_id = CommitID(commit_position=74, revision='r74') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r90', 90, 74)
def testGetLatestRegressionRange(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=91, pass_rate=0.9, git_hash='rev91'), DataPoint.Create(commit_position=90, pass_rate=1.0, git_hash='rev90'), ] self.assertEqual( CommitIDRange(lower=CommitID(commit_position=90, revision='rev90'), upper=CommitID(commit_position=91, revision='rev91')), analysis.GetLatestRegressionRange())
def testUpdateSuspectedBuildRegressionRangeTooWide(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=100), DataPoint.Create(commit_position=80), ] analysis.Save() lower_bound_target = IsolatedTarget.Create(999, '', '', 'm', 'b', '', '', '', '', '', '', 90, '') upper_bound_target = IsolatedTarget.Create(1000, '', '', 'm', 'b', '', '', '', '', '', '', 100, '') analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target) self.assertIsNone(analysis.suspected_flake_build_number)
def testLookbackAlgorithmBisectWhenTestDoesNotExist(self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create( commit_position=50, pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, git_hash='r50'), ] next_commit_id = CommitID(commit_position=75, revision='r75') self.assertEqual( (next_commit_id, None), # 100 flaky, 50 non-existent. Bisect. lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmWithRegressionRangeRestartExponential( self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=1.0, git_hash='r90'), ] # 100 stable, 90 flaky. Restart search from 99. next_commit_id = CommitID(commit_position=99, revision='r99') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r100', 100, 99)
def testSaveFlakeCulpritsForSuspectedRevisionsExistingCulprit( self, mocked_commit_position, mocked_fn): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' suspected_revision = 'r1' suspect_commit_position = 995 suspected_revisions = [suspected_revision] mocked_commit_position.return_value = suspect_commit_position analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash=suspected_revision) ] suspect = FlakeCulprit.Create('chromium', suspected_revision, suspect_commit_position) suspect.url = 'url' suspect.put() analysis.suspect_urlsafe_keys = [suspect.key.urlsafe()] analysis.Save() mocked_fn.return_value = None heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions( analysis.key.urlsafe(), suspected_revisions) analysis = MasterFlakeAnalysis.GetVersion( master_name, builder_name, build_number, step_name, test_name) self.assertIn(suspect.key.urlsafe(), analysis.suspect_urlsafe_keys)
def testSaveFlakeCulpritsForSuspectedRevisionsNoChangeLog( self, mocked_commit_position, mocked_commit_info): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' suspected_revision = 'r1' suspected_commit_position = 995 suspected_revisions = [suspected_revision] mocked_commit_position.return_value = suspected_commit_position analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash=suspected_revision) ] analysis.Save() mocked_commit_info.return_value = None heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions( analysis.key.urlsafe(), suspected_revisions) suspect = FlakeCulprit.Get('chromium', suspected_revision) self.assertIsNone(suspect)
def testGetNextCommitIDFromBuildRangeReturnCalculated(self): calculated_commit_id = CommitID(commit_position=1005, revision='r1005') lower = CommitID(commit_position=1000, revision='r1000') upper = CommitID(commit_position=1010, revision='r1010') build_range = CommitIDRange(lower=lower, upper=upper) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1010), # Already has both. DataPoint.Create(commit_position=1000), ] self.assertEqual( calculated_commit_id, next_commit_position_utils.GetNextCommitIDFromBuildRange( analysis, build_range, calculated_commit_id))
def testLookbackAlgorithmCulpritFoundExistingTest(self): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=71, pass_rate=0.5, git_hash='r71'), DataPoint.Create(commit_position=70, pass_rate=1.0, git_hash='r70'), ] # 70 stable, 71 flaky. 71 must be the culprit. culprit_commit_id = CommitID(commit_position=71, revision='r71') self.assertEqual( (None, culprit_commit_id), lookback_algorithm._DetermineNextCommitPosition(data_points))