def testLookbackAlgorithmNoCulpritWhenTestDoesNotExist(self): data_points = [ DataPoint.Create( commit_position=100, pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND), ] self.assertEqual( (None, None), # Test not found from the beginning. lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmSingleFlakyDataPoint(self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100') ] next_commit_id = CommitID(commit_position=99, revision='r99') self.assertEqual( (next_commit_id, None), # Begins with step size 1. lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmExponentialLookback(self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), ] # Step size 10, rounded up to the next square == 16. next_commit_id = CommitID(commit_position=74, revision='r74') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r90', 90, 74)
def testLookbackAlgorithmWithRegressionRangeRestartExponential( self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=1.0, git_hash='r90'), ] # 100 stable, 90 flaky. Restart search from 99. next_commit_id = CommitID(commit_position=99, revision='r99') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r100', 100, 99)
def testLookbackAlgorithmBisectWhenTestDoesNotExist(self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create( commit_position=50, pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, git_hash='r50'), ] next_commit_id = CommitID(commit_position=75, revision='r75') self.assertEqual( (next_commit_id, None), # 100 flaky, 50 non-existent. Bisect. lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmCulpritFoundExistingTest(self): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=71, pass_rate=0.5, git_hash='r71'), DataPoint.Create(commit_position=70, pass_rate=1.0, git_hash='r70'), ] # 70 stable, 71 flaky. 71 must be the culprit. culprit_commit_id = CommitID(commit_position=71, revision='r71') self.assertEqual( (None, culprit_commit_id), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmWithRegressionRangeContinueExponential( self, mock_git): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), DataPoint.Create(commit_position=70, pass_rate=1.0, git_hash='r70'), ] next_commit_id = CommitID(commit_position=74, revision='r74') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points)) mock_git.assert_called_once_with('r90', 90, 74)
def testLookbackAlgorithmRestartExponentialLandsOnExistingDataPoint( self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=32, pass_rate=0.5, git_hash='r32'), DataPoint.Create(commit_position=20, pass_rate=0.5, git_hash='r20'), DataPoint.Create(commit_position=4, pass_rate=1.0, git_hash='r4'), ] next_commit_id = CommitID(commit_position=19, revision='r19') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmWithRegressionRangeRestartExponentialLargeStep( self, _): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=90, pass_rate=0.5, git_hash='r90'), DataPoint.Create(commit_position=20, pass_rate=0.5, git_hash='r20'), DataPoint.Create(commit_position=10, pass_rate=1.0, git_hash='r10'), ] next_commit_id = CommitID(commit_position=19, revision='r19') self.assertEqual( (next_commit_id, None), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmCulpritFoundNewlyAddedTest(self): data_points = [ DataPoint.Create(commit_position=100, pass_rate=0.5, git_hash='r100'), DataPoint.Create(commit_position=71, pass_rate=0.5, git_hash='r71'), DataPoint.Create( commit_position=70, pass_rate=flake_constants.PASS_RATE_TEST_NOT_FOUND, git_hash='r70') ] # 70 nonexistent, 71 flaky. 71 must be the culprit. culprit_commit_id = CommitID(commit_position=71, revision='r71') self.assertEqual( (None, culprit_commit_id), lookback_algorithm._DetermineNextCommitPosition(data_points))
def testLookbackAlgorithmNoCulpritWhenNotReproducible(self): data_points = [DataPoint.Create(commit_position=100, pass_rate=1.0)] self.assertEqual( (None, None), # Flakiness not reproducible from the beginning. lookback_algorithm._DetermineNextCommitPosition(data_points))