def testGetLowestUpperBoundBuildNumber(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(build_number=123, commit_position=1000), DataPoint.Create(build_number=122, commit_position=990) ] self.assertEqual(122, analysis.GetLowestUpperBoundBuildNumber(900))
def testUpdateSuspectedBuild(self, mock_build_number): build_number = 120 build_id = 1200 mock_build_number.return_value = build_number lower_bound_commit_position = 90 upper_bound_commit_position = 100 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=upper_bound_commit_position, pass_rate=0.4), DataPoint.Create(commit_position=lower_bound_commit_position, pass_rate=1.0), ] analysis.Save() lower_bound_target = IsolatedTarget.Create( build_id - 1, '', '', 'm', 'b', '', '', '', '', '', '', lower_bound_commit_position, '') upper_bound_target = IsolatedTarget.Create( build_id, '', '', 'm', 'b', '', '', '', '', '', '', upper_bound_commit_position, '') analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target) self.assertEqual(build_id, analysis.suspected_build_id) self.assertEqual(build_number, analysis.suspected_flake_build_number)
def testGenerateCommitIDsForBoundingTargetsQueryGit(self, mock_revisions): data_points = [ DataPoint.Create(commit_position=1010, git_hash='r1010'), DataPoint.Create(commit_position=1000, git_hash='r1000'), ] mock_revisions.return_value = { 1003: 'r1003', 1004: 'r1004', 1005: 'r1005', 1006: 'r1006', 1007: 'r1007', 1008: 'r1008', 1009: 'r1009', 1010: 'r1010' } lower_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '', '', '', '', '', '', 1003, None) upper_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '', '', '', '', '', '', 1008, None) lower_bound_commit_id = CommitID(commit_position=1003, revision='r1003') upper_bound_commit_id = CommitID(commit_position=1008, revision='r1008') self.assertEqual( (lower_bound_commit_id, upper_bound_commit_id), next_commit_position_utils.GenerateCommitIDsForBoundingTargets( data_points, lower_bound_target, upper_bound_target)) mock_revisions.assert_called_once_with('r1010', 1010, 1003)
def testGetLatestDataPointNoRecentFlakinessPoints(self): expected_data_point = DataPoint.Create(commit_position=1000) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ expected_data_point, DataPoint.Create(commit_position=990) ] self.assertEqual(expected_data_point, analysis.GetLatestDataPoint())
def testGetPassCount(self): self.assertEqual( 0, DataPoint.Create(pass_rate=0.0, iterations=100).GetPassCount()) self.assertEqual( 10, DataPoint.Create(pass_rate=0.5, iterations=20).GetPassCount()) self.assertEqual( 429, DataPoint.Create(pass_rate=0.9976744186046511, iterations=430).GetPassCount())
def testRemoveDataPointWithCommitPosition(self): data_points = [ DataPoint.Create(build_number=100, commit_position=1000), DataPoint.Create(build_number=101, commit_position=1100), DataPoint.Create(build_number=110, commit_position=2000) ] analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = data_points analysis.RemoveDataPointWithCommitPosition(2000) self.assertEqual(data_points[:-1], analysis.data_points)
def testCalculateCulpritConfidenceScoreIntroducedNewFlakyTest(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.7, commit_position=1000), DataPoint.Create( pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, commit_position=999) ] self.assertEqual( 1.0, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testGetLatestRegressionRangeNoUpperBoundMultipleDataPoints(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='rev90') ] self.assertEqual( CommitIDRange(lower=None, upper=CommitID(commit_position=90, revision='rev90')), analysis.GetLatestRegressionRange())
def testIdentifySuspectedRangesFailedToGetBlame(self, _, mock_test_location): mock_test_location.return_value = TestLocation(file='a/b.cc', line=1) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash='r1000', pass_rate=0.5), DataPoint.Create(commit_position=997, git_hash='r997', pass_rate=1.0) ] analysis.Save() self.assertEqual([], heuristic_analysis.IdentifySuspectedRevisions(analysis))
def testNewlyAddedTest(self): culprit_commit_position = 11 analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create( pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, commit_position=culprit_commit_position - 1), DataPoint.Create( pass_rate=0.5, commit_position=culprit_commit_position), ] self.assertTrue( culprit_util.CulpritAddedNewFlakyTest(analysis, culprit_commit_position))
def testCalculateCulpritConfidenceScoreFallbackToSteppiness(self, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.7, iterations=20, commit_position=1000), DataPoint.Create(pass_rate=1.0, iterations=400, commit_position=999), ] self.assertEqual( .6, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testShouldTakeAutoAction(self): culprit_commit_position = 1000 culprit = FlakeCulprit.Create('chromium', 'r', culprit_commit_position) culprit.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.culprit_urlsafe_key = culprit.key.urlsafe() analysis.data_points = [ DataPoint.Create(commit_position=culprit_commit_position, pass_rate=0.5), DataPoint.Create(commit_position=culprit_commit_position - 1, pass_rate=1.0) ] self.assertTrue( flake_analysis_util.ShouldTakeAutoAction(analysis, False))
def testLookbackAlgorithmExponentialLookback(self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), ] # Step size 10, rounded up to the next square == 16. next_commit_id = CommitID(commit_position=74, revision='r74') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r90', 90, 74)
def testGetNextCommitIdFromHeuristicResultsExistingDataPoints(self): suspect = FlakeCulprit.Create('repo', 'revision', 1000) suspect.put() analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe()) analysis.data_points = [ DataPoint.Create(commit_position=999, pass_rate=1.0), DataPoint.Create(commit_position=1000, pass_rate=1.0) ] analysis.put() self.assertIsNone( next_commit_position_utils.GetNextCommitIdFromHeuristicResults( analysis.key.urlsafe()))
def testUpdateSuspectedBuildRegressionRangeTooWide(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=100), DataPoint.Create(commit_position=80), ] analysis.Save() lower_bound_target = IsolatedTarget.Create(999, '', '', 'm', 'b', '', '', '', '', '', '', 90, '') upper_bound_target = IsolatedTarget.Create(1000, '', '', 'm', 'b', '', '', '', '', '', '', 100, '') analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target) self.assertIsNone(analysis.suspected_flake_build_number)
def testLookbackAlgorithmWithRegressionRangeRestartExponential( self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=1.0, git_hash='r90'), ] # 100 stable, 90 flaky. Restart search from 99. next_commit_id = CommitID(commit_position=99, revision='r99') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r100', 100, 99)
def testLookbackAlgorithmBisectWhenTestDoesNotExist(self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create( commit_position=50, pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, git_hash='r50'), ] next_commit_id = CommitID(commit_position=75, revision='r75') self.assertEqual( (next_commit_id, None), # 100 flaky, 50 non-existent. Bisect. lookback_algorithm._DetermineNextCommitPosition(data_points))
def testGetLatestRegressionRange(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=91, pass_rate=0.9, git_hash='rev91'), DataPoint.Create(commit_position=90, pass_rate=1.0, git_hash='rev90'), ] self.assertEqual( CommitIDRange(lower=CommitID(commit_position=90, revision='rev90'), upper=CommitID(commit_position=91, revision='rev91')), analysis.GetLatestRegressionRange())
def testSaveFlakeCulpritsForSuspectedRevisionsExistingCulprit( self, mocked_commit_position, mocked_fn): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' suspected_revision = 'r1' suspect_commit_position = 995 suspected_revisions = [suspected_revision] mocked_commit_position.return_value = suspect_commit_position analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash=suspected_revision) ] suspect = FlakeCulprit.Create('chromium', suspected_revision, suspect_commit_position) suspect.url = 'url' suspect.put() analysis.suspect_urlsafe_keys = [suspect.key.urlsafe()] analysis.Save() mocked_fn.return_value = None heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions( analysis.key.urlsafe(), suspected_revisions) analysis = MasterFlakeAnalysis.GetVersion( master_name, builder_name, build_number, step_name, test_name) self.assertIn(suspect.key.urlsafe(), analysis.suspect_urlsafe_keys)
def testSaveFlakeCulpritsForSuspectedRevisionsNoChangeLog( self, mocked_commit_position, mocked_commit_info): master_name = 'm' builder_name = 'b' build_number = 123 step_name = 's' test_name = 't' suspected_revision = 'r1' suspected_commit_position = 995 suspected_revisions = [suspected_revision] mocked_commit_position.return_value = suspected_commit_position analysis = MasterFlakeAnalysis.Create(master_name, builder_name, build_number, step_name, test_name) analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash=suspected_revision) ] analysis.Save() mocked_commit_info.return_value = None heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions( analysis.key.urlsafe(), suspected_revisions) suspect = FlakeCulprit.Get('chromium', suspected_revision) self.assertIsNone(suspect)
def testGetNextCommitIDFromBuildRangeReturnCalculated(self): calculated_commit_id = CommitID(commit_position=1005, revision='r1005') lower = CommitID(commit_position=1000, revision='r1000') upper = CommitID(commit_position=1010, revision='r1010') build_range = CommitIDRange(lower=lower, upper=upper) analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1010), # Already has both. DataPoint.Create(commit_position=1000), ] self.assertEqual( calculated_commit_id, next_commit_position_utils.GetNextCommitIDFromBuildRange( analysis, build_range, calculated_commit_id))
def testLookbackAlgorithmCulpritFoundExistingTest(self): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=71, pass_rate=0.5, git_hash='r71'), DataPoint.Create(commit_position=70, pass_rate=1.0, git_hash='r70'), ] # 70 stable, 71 flaky. 71 must be the culprit. culprit_commit_id = CommitID(commit_position=71, revision='r71') self.assertEqual( (None, culprit_commit_id), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testFlakyAtMostRecentlyAnalyzedCommitStillFlaky(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.flakiness_verification_data_points = [ DataPoint.Create(commit_position=1000, pass_rate=0.5) ] self.assertTrue( flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(analysis))
def testFlakyAtMostRecentlyAnalyzedCommitStable(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1000, pass_rate=1.0) ] self.assertFalse( flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(analysis))
def testCalculateCulpritConfidenceScoreIntroducedStableToFlaky(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.7, iterations=10, commit_position=1000), DataPoint.Create(pass_rate=1.0, iterations=400, commit_position=999), DataPoint.Create(pass_rate=1.0, iterations=400, commit_position=996) ] self.assertEqual( .7, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testLookbackAlgorithmWithRegressionRangeContinueExponential( self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), DataPoint.Create(commit_position=70, pass_rate=1.0, git_hash='r70'), ] next_commit_id = CommitID(commit_position=74, revision='r74') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r90', 90, 74)
def testCalculateCulpritConfidenceScoreLowFlakiness(self, *_): analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't') analysis.data_points = [ DataPoint.Create(pass_rate=0.9975, iterations=400, commit_position=1000), DataPoint.Create(pass_rate=1.0, iterations=400, commit_position=999) ] self.assertIsNone( confidence_score_util.CalculateCulpritConfidenceScore( analysis, None)) self.assertEqual( 0.0, confidence_score_util.CalculateCulpritConfidenceScore( analysis, 1000))
def testLookbackAlgorithmRestartExponentialLandsOnExistingDataPoint( self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=32, pass_rate=0.5, git_hash='r32'), DataPoint.Create(commit_position=20, pass_rate=0.5, git_hash='r20'), DataPoint.Create(commit_position=4, pass_rate=1.0, git_hash='r4'), ] next_commit_id = CommitID(commit_position=19, revision='r19') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testUpdateSuspectedBuildUsingBuildInfo(self): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.4), DataPoint.Create(commit_position=90, pass_rate=1.0), ] analysis.Save() lower_bound_build = BuildInfo('m', 'b', 122) lower_bound_build.commit_position = 90 upper_bound_build = BuildInfo('m', 'b', 123) upper_bound_build.commit_position = 100 analysis.UpdateSuspectedBuildUsingBuildInfo(lower_bound_build, upper_bound_build) self.assertEqual(123, analysis.suspected_flake_build_number)
def testIdentifysuspectedRangesNoTestLocation(self, _): analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't') analysis.data_points = [ DataPoint.Create(commit_position=1000, git_hash='r1000') ] analysis.Save() self.assertEqual([], heuristic_analysis.IdentifySuspectedRevisions(analysis))