def testLookbackAlgorithmNoCulpritWhenTestDoesNotExist(self):
data_points = [
DataPoint.Create(
commit_position=100,
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND),
]
self.assertEqual(
(None, None), # Test not found from the beginning.
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmSingleFlakyDataPoint(self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100')
]
next_commit_id = CommitID(commit_position=99, revision='r99')
self.assertEqual(
(next_commit_id, None), # Begins with step size 1.
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmExponentialLookback(self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=0.5,
git_hash='r90'),
]
# Step size 10, rounded up to the next square == 16.
next_commit_id = CommitID(commit_position=74, revision='r74')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r90', 90, 74)
def testLookbackAlgorithmWithRegressionRangeRestartExponential(
self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=1.0,
git_hash='r90'),
]
# 100 stable, 90 flaky. Restart search from 99.
next_commit_id = CommitID(commit_position=99, revision='r99')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r100', 100, 99)
def testLookbackAlgorithmBisectWhenTestDoesNotExist(self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(
commit_position=50,
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND,
git_hash='r50'),
]
next_commit_id = CommitID(commit_position=75, revision='r75')
self.assertEqual(
(next_commit_id, None), # 100 flaky, 50 non-existent. Bisect.
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmCulpritFoundExistingTest(self):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=71, pass_rate=0.5,
git_hash='r71'),
DataPoint.Create(commit_position=70, pass_rate=1.0,
git_hash='r70'),
]
# 70 stable, 71 flaky. 71 must be the culprit.
culprit_commit_id = CommitID(commit_position=71, revision='r71')
self.assertEqual(
(None, culprit_commit_id),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmWithRegressionRangeContinueExponential(
self, mock_git):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=0.5,
git_hash='r90'),
DataPoint.Create(commit_position=70, pass_rate=1.0,
git_hash='r70'),
]
next_commit_id = CommitID(commit_position=74, revision='r74')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
mock_git.assert_called_once_with('r90', 90, 74)
def testLookbackAlgorithmRestartExponentialLandsOnExistingDataPoint(
self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=32, pass_rate=0.5,
git_hash='r32'),
DataPoint.Create(commit_position=20, pass_rate=0.5,
git_hash='r20'),
DataPoint.Create(commit_position=4, pass_rate=1.0, git_hash='r4'),
]
next_commit_id = CommitID(commit_position=19, revision='r19')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmWithRegressionRangeRestartExponentialLargeStep(
self, _):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=90, pass_rate=0.5,
git_hash='r90'),
DataPoint.Create(commit_position=20, pass_rate=0.5,
git_hash='r20'),
DataPoint.Create(commit_position=10, pass_rate=1.0,
git_hash='r10'),
]
next_commit_id = CommitID(commit_position=19, revision='r19')
self.assertEqual(
(next_commit_id, None),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmCulpritFoundNewlyAddedTest(self):
data_points = [
DataPoint.Create(commit_position=100,
pass_rate=0.5,
git_hash='r100'),
DataPoint.Create(commit_position=71, pass_rate=0.5,
git_hash='r71'),
DataPoint.Create(
commit_position=70,
pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND,
git_hash='r70')
]
# 70 nonexistent, 71 flaky. 71 must be the culprit.
culprit_commit_id = CommitID(commit_position=71, revision='r71')
self.assertEqual(
(None, culprit_commit_id),
lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmNoCulpritWhenNotReproducible(self):
data_points = [DataPoint.Create(commit_position=100, pass_rate=1.0)]
self.assertEqual(
(None, None), # Flakiness not reproducible from the beginning.
lookback_algorithm._DetermineNextCommitPosition(data_points))