def testGetLowestUpperBoundBuildNumber(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(build_number=123, commit_position=1000),
DataPoint.Create(build_number=122, commit_position=990)
]
self.assertEqual(122, analysis.GetLowestUpperBoundBuildNumber(900))
def testUpdateSuspectedBuild(self, mock_build_number):
build_number = 120
build_id = 1200
mock_build_number.return_value = build_number
lower_bound_commit_position = 90
upper_bound_commit_position = 100
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=upper_bound_commit_position,
pass_rate=0.4),
DataPoint.Create(commit_position=lower_bound_commit_position,
pass_rate=1.0),
]
analysis.Save()
lower_bound_target = IsolatedTarget.Create(
build_id - 1, '', '', 'm', 'b', '', '', '', '', '', '',
lower_bound_commit_position, '')
upper_bound_target = IsolatedTarget.Create(
build_id, '', '', 'm', 'b', '', '', '', '', '', '',
upper_bound_commit_position, '')
analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target)
self.assertEqual(build_id, analysis.suspected_build_id)
self.assertEqual(build_number, analysis.suspected_flake_build_number)
def testGenerateCommitIDsForBoundingTargetsQueryGit(self, mock_revisions):
data_points = [
DataPoint.Create(commit_position=1010, git_hash='r1010'),
DataPoint.Create(commit_position=1000, git_hash='r1000'),
]
mock_revisions.return_value = {
1003: 'r1003',
1004: 'r1004',
1005: 'r1005',
1006: 'r1006',
1007: 'r1007',
1008: 'r1008',
1009: 'r1009',
1010: 'r1010'
}
lower_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '',
'', '', '', '', '', 1003,
None)
upper_bound_target = IsolatedTarget.Create(67890, '', '', 'm', 'b', '',
'', '', '', '', '', 1008,
None)
lower_bound_commit_id = CommitID(commit_position=1003,
revision='r1003')
upper_bound_commit_id = CommitID(commit_position=1008,
revision='r1008')
self.assertEqual(
(lower_bound_commit_id, upper_bound_commit_id),
next_commit_position_utils.GenerateCommitIDsForBoundingTargets(
data_points, lower_bound_target, upper_bound_target))
mock_revisions.assert_called_once_with('r1010', 1010, 1003)
def testGetLatestDataPointNoRecentFlakinessPoints(self):
expected_data_point = DataPoint.Create(commit_position=1000)
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
expected_data_point,
DataPoint.Create(commit_position=990)
]
self.assertEqual(expected_data_point, analysis.GetLatestDataPoint())
def testGetPassCount(self):
self.assertEqual(
0,
DataPoint.Create(pass_rate=0.0, iterations=100).GetPassCount())
self.assertEqual(
10,
DataPoint.Create(pass_rate=0.5, iterations=20).GetPassCount())
self.assertEqual(
429,
DataPoint.Create(pass_rate=0.9976744186046511,
iterations=430).GetPassCount())
def testRemoveDataPointWithCommitPosition(self):
data_points = [
DataPoint.Create(build_number=100, commit_position=1000),
DataPoint.Create(build_number=101, commit_position=1100),
DataPoint.Create(build_number=110, commit_position=2000)
]
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = data_points
analysis.RemoveDataPointWithCommitPosition(2000)
self.assertEqual(data_points[:-1], analysis.data_points)
def testCalculateCulpritConfidenceScoreIntroducedNewFlakyTest(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't')
analysis.data_points = [
DataPoint.Create(pass_rate=0.7, commit_position=1000),
DataPoint.Create(
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND,
commit_position=999)
]
self.assertEqual(
1.0,
confidence_score_util.CalculateCulpritConfidenceScore(
analysis, 1000))
def testGetLatestRegressionRangeNoUpperBoundMultipleDataPoints(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=100, pass_rate=0.5),
DataPoint.Create(commit_position=90,
pass_rate=0.5,
git_hash='rev90')
]
self.assertEqual(
CommitIDRange(lower=None,
upper=CommitID(commit_position=90,
revision='rev90')),
analysis.GetLatestRegressionRange())
def testIdentifySuspectedRangesFailedToGetBlame(self, _, mock_test_location):
mock_test_location.return_value = TestLocation(file='a/b.cc', line=1)
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=1000, git_hash='r1000', pass_rate=0.5),
DataPoint.Create(commit_position=997, git_hash='r997', pass_rate=1.0)
]
analysis.Save()
self.assertEqual([],
heuristic_analysis.IdentifySuspectedRevisions(analysis))
def testNewlyAddedTest(self):
culprit_commit_position = 11
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND,
commit_position=culprit_commit_position - 1),
DataPoint.Create(
pass_rate=0.5, commit_position=culprit_commit_position),
]
self.assertTrue(
culprit_util.CulpritAddedNewFlakyTest(analysis,
culprit_commit_position))
def testCalculateCulpritConfidenceScoreFallbackToSteppiness(self, _):
analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't')
analysis.data_points = [
DataPoint.Create(pass_rate=0.7,
iterations=20,
commit_position=1000),
DataPoint.Create(pass_rate=1.0,
iterations=400,
commit_position=999),
]
self.assertEqual(
.6,
confidence_score_util.CalculateCulpritConfidenceScore(
analysis, 1000))
def testShouldTakeAutoAction(self):
culprit_commit_position = 1000
culprit = FlakeCulprit.Create('chromium', 'r', culprit_commit_position)
culprit.put()
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.culprit_urlsafe_key = culprit.key.urlsafe()
analysis.data_points = [
DataPoint.Create(commit_position=culprit_commit_position,
pass_rate=0.5),
DataPoint.Create(commit_position=culprit_commit_position - 1,
pass_rate=1.0)
]
self.assertTrue(
flake_analysis_util.ShouldTakeAutoAction(analysis, False))
def testLookbackAlgorithmExponentialLookback(self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=0.5,
git_hash='r90'),
]
# Step size 10, rounded up to the next square == 16.
next_commit_id = CommitID(commit_position=74, revision='r74')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r90', 90, 74)
def testGetNextCommitIdFromHeuristicResultsExistingDataPoints(self):
suspect = FlakeCulprit.Create('repo', 'revision', 1000)
suspect.put()
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.suspect_urlsafe_keys.append(suspect.key.urlsafe())
analysis.data_points = [
DataPoint.Create(commit_position=999, pass_rate=1.0),
DataPoint.Create(commit_position=1000, pass_rate=1.0)
]
analysis.put()
self.assertIsNone(
next_commit_position_utils.GetNextCommitIdFromHeuristicResults(
analysis.key.urlsafe()))
def testUpdateSuspectedBuildRegressionRangeTooWide(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=100),
DataPoint.Create(commit_position=80),
]
analysis.Save()
lower_bound_target = IsolatedTarget.Create(999, '', '', 'm', 'b', '',
'', '', '', '', '', 90, '')
upper_bound_target = IsolatedTarget.Create(1000, '', '', 'm', 'b', '',
'', '', '', '', '', 100, '')
analysis.UpdateSuspectedBuild(lower_bound_target, upper_bound_target)
self.assertIsNone(analysis.suspected_flake_build_number)
def testLookbackAlgorithmWithRegressionRangeRestartExponential(
self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=1.0,
git_hash='r90'),
]
# 100 stable, 90 flaky. Restart search from 99.
next_commit_id = CommitID(commit_position=99, revision='r99')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r100', 100, 99)
def testLookbackAlgorithmBisectWhenTestDoesNotExist(self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(
commit_position=50,
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND,
git_hash='r50'),
]
next_commit_id = CommitID(commit_position=75, revision='r75')
self.assertEqual(
(next_commit_id, None), # 100 flaky, 50 non-existent. Bisect.
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testGetLatestRegressionRange(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=91,
pass_rate=0.9,
git_hash='rev91'),
DataPoint.Create(commit_position=90,
pass_rate=1.0,
git_hash='rev90'),
]
self.assertEqual(
CommitIDRange(lower=CommitID(commit_position=90, revision='rev90'),
upper=CommitID(commit_position=91,
revision='rev91')),
analysis.GetLatestRegressionRange())
def testSaveFlakeCulpritsForSuspectedRevisionsExistingCulprit(
self, mocked_commit_position, mocked_fn):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
suspected_revision = 'r1'
suspect_commit_position = 995
suspected_revisions = [suspected_revision]
mocked_commit_position.return_value = suspect_commit_position
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name, test_name)
analysis.data_points = [
DataPoint.Create(commit_position=1000, git_hash=suspected_revision)
]
suspect = FlakeCulprit.Create('chromium', suspected_revision,
suspect_commit_position)
suspect.url = 'url'
suspect.put()
analysis.suspect_urlsafe_keys = [suspect.key.urlsafe()]
analysis.Save()
mocked_fn.return_value = None
heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions(
analysis.key.urlsafe(), suspected_revisions)
analysis = MasterFlakeAnalysis.GetVersion(
master_name, builder_name, build_number, step_name, test_name)
self.assertIn(suspect.key.urlsafe(), analysis.suspect_urlsafe_keys)
def testSaveFlakeCulpritsForSuspectedRevisionsNoChangeLog(
self, mocked_commit_position, mocked_commit_info):
master_name = 'm'
builder_name = 'b'
build_number = 123
step_name = 's'
test_name = 't'
suspected_revision = 'r1'
suspected_commit_position = 995
suspected_revisions = [suspected_revision]
mocked_commit_position.return_value = suspected_commit_position
analysis = MasterFlakeAnalysis.Create(master_name, builder_name,
build_number, step_name, test_name)
analysis.data_points = [
DataPoint.Create(commit_position=1000, git_hash=suspected_revision)
]
analysis.Save()
mocked_commit_info.return_value = None
heuristic_analysis.SaveFlakeCulpritsForSuspectedRevisions(
analysis.key.urlsafe(), suspected_revisions)
suspect = FlakeCulprit.Get('chromium', suspected_revision)
self.assertIsNone(suspect)
def testGetNextCommitIDFromBuildRangeReturnCalculated(self):
calculated_commit_id = CommitID(commit_position=1005, revision='r1005')
lower = CommitID(commit_position=1000, revision='r1000')
upper = CommitID(commit_position=1010, revision='r1010')
build_range = CommitIDRange(lower=lower, upper=upper)
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=1010), # Already has both.
DataPoint.Create(commit_position=1000),
]
self.assertEqual(
calculated_commit_id,
next_commit_position_utils.GetNextCommitIDFromBuildRange(
analysis, build_range, calculated_commit_id))
def testLookbackAlgorithmCulpritFoundExistingTest(self):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=71, pass_rate=0.5,
git_hash='r71'),
DataPoint.Create(commit_position=70, pass_rate=1.0,
git_hash='r70'),
]
# 70 stable, 71 flaky. 71 must be the culprit.
culprit_commit_id = CommitID(commit_position=71, revision='r71')
self.assertEqual(
(None, culprit_commit_id),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testFlakyAtMostRecentlyAnalyzedCommitStillFlaky(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.flakiness_verification_data_points = [
DataPoint.Create(commit_position=1000, pass_rate=0.5)
]
self.assertTrue(
flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(analysis))
def testFlakyAtMostRecentlyAnalyzedCommitStable(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=1000, pass_rate=1.0)
]
self.assertFalse(
flake_analysis_util.FlakyAtMostRecentlyAnalyzedCommit(analysis))
def testCalculateCulpritConfidenceScoreIntroducedStableToFlaky(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't')
analysis.data_points = [
DataPoint.Create(pass_rate=0.7,
iterations=10,
commit_position=1000),
DataPoint.Create(pass_rate=1.0,
iterations=400,
commit_position=999),
DataPoint.Create(pass_rate=1.0,
iterations=400,
commit_position=996)
]
self.assertEqual(
.7,
confidence_score_util.CalculateCulpritConfidenceScore(
analysis, 1000))
def testLookbackAlgorithmWithRegressionRangeContinueExponential(
self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=0.5,
git_hash='r90'),
DataPoint.Create(commit_position=70, pass_rate=1.0,
git_hash='r70'),
]
next_commit_id = CommitID(commit_position=74, revision='r74')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r90', 90, 74)
def testCalculateCulpritConfidenceScoreLowFlakiness(self, *_):
analysis = MasterFlakeAnalysis.Create('m', 'b', 124, 's', 't')
analysis.data_points = [
DataPoint.Create(pass_rate=0.9975,
iterations=400,
commit_position=1000),
DataPoint.Create(pass_rate=1.0,
iterations=400,
commit_position=999)
]
self.assertIsNone(
confidence_score_util.CalculateCulpritConfidenceScore(
analysis, None))
self.assertEqual(
0.0,
confidence_score_util.CalculateCulpritConfidenceScore(
analysis, 1000))
def testLookbackAlgorithmRestartExponentialLandsOnExistingDataPoint(
self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=32, pass_rate=0.5,
git_hash='r32'),
DataPoint.Create(commit_position=20, pass_rate=0.5,
git_hash='r20'),
DataPoint.Create(commit_position=4, pass_rate=1.0, git_hash='r4'),
]
next_commit_id = CommitID(commit_position=19, revision='r19')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testUpdateSuspectedBuildUsingBuildInfo(self):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=100, pass_rate=0.4),
DataPoint.Create(commit_position=90, pass_rate=1.0),
]
analysis.Save()
lower_bound_build = BuildInfo('m', 'b', 122)
lower_bound_build.commit_position = 90
upper_bound_build = BuildInfo('m', 'b', 123)
upper_bound_build.commit_position = 100
analysis.UpdateSuspectedBuildUsingBuildInfo(lower_bound_build,
upper_bound_build)
self.assertEqual(123, analysis.suspected_flake_build_number)
def testIdentifysuspectedRangesNoTestLocation(self, _):
analysis = MasterFlakeAnalysis.Create('m', 'b', 123, 's', 't')
analysis.data_points = [
DataPoint.Create(commit_position=1000, git_hash='r1000')
]
analysis.Save()
self.assertEqual([],
heuristic_analysis.IdentifySuspectedRevisions(analysis))
