def MockOutBuildDeployEvaluateForSanityCheck(self): """Mocks out BuildDeployEvaluate behavior for SaintyCheck test. It mocks UpdateCurrentCommit() to emit good and bad commits. And mocks CheckLastEvaluate() to return good and bad commit score. """ commit_info_list = [ common.CommitInfo(sha1=self.GOOD_COMMIT_SHA1, title='good', timestamp=self.GOOD_COMMIT_TIMESTAMP), common.CommitInfo(sha1=self.BAD_COMMIT_SHA1, title='bad', timestamp=self.BAD_COMMIT_TIMESTAMP) ] # This mock is problematic. The side effect should modify "self" when # the member method is called. def UpdateCurrentCommitSideEffect(): self.bisector.current_commit = commit_info_list.pop(0) self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit', side_effect=UpdateCurrentCommitSideEffect) self.PatchObject( evaluator_module.Evaluator, 'CheckLastEvaluate', side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE])
def testBool(self): """Tests CommitInfo's boolean value conversion. Only default(empty) CommitInfo's boolean value is False. """ info1 = common.CommitInfo() self.assertTrue(not info1) self.assertFalse(bool(info1)) info2 = common.CommitInfo(title='something') self.assertTrue(bool(info2)) self.assertFalse(not info2)
def testNotEqual(self): """Tests inequality of two CommitInfo objects.""" info1 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) info2 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2]), label='GOOD', timestamp=100) self.assertNotEqual(info1, info2) self.assertFalse(info1 == info2)
def testEqual(self): """Tests equality of two CommitInfo objects.""" info1 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) info2 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) self.assertEqual(info1, info2) # In Python 2.7, __ne__() doesn't delegates to "not __eq__()" so that the # sanity check is necessary. self.assertFalse(info1 != info2)
def testAssigned(self): """Tests that CommitInfo constrcutor sets up data members correctly.""" info = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1]), label='GOOD', timestamp=100) self.assertEqual( "CommitInfo(sha1='abcdef', title='test', score=Score(values=[1.0]), " "label='GOOD', timestamp=100)", repr(info))
def __init__(self, options, builder, evaluator): """Constructor. Args: options: An argparse.Namespace to hold command line arguments. Should contain: * good: Last known good commit. * bad: Last known bad commit. * remote: DUT (refer lib.commandline.Device). * eval_repeat: Run test for N times. None means 1. * auto_threshold: True to set threshold automatically without prompt. * board: board name of the DUT. * eval_raise_on_error: If set, raises if evaluation failed. * skip_failed_commit: If set, mark the failed commit (build failed / no result) as 'skip' instead of 'bad'. builder: Builder to build/deploy image. Should contain repo_dir. evaluator: Evaluator to get score """ super(GitBisector, self).__init__(options) self.good_commit = options.good self.bad_commit = options.bad self.remote = options.remote self.eval_repeat = options.eval_repeat self.builder = builder self.evaluator = evaluator self.repo_dir = self.builder.repo_dir self.auto_threshold = options.auto_threshold self.board = options.board self.eval_raise_on_error = options.eval_raise_on_error self.skip_failed_commit = options.skip_failed_commit # Initialized in ObtainBisectBoundaryScore(). self.good_commit_info = None self.bad_commit_info = None # Record bisect log (list of CommitInfo). self.bisect_log = [] # If the distance between current commit's score and given good commit is # shorter than threshold, treat the commit as good one. Its default value # comes from evaluator's THRESHOLD constant. If it is None (undefined), # the bisector will prompt user to input threshold after obtaining both # sides' score. self.threshold = evaluator.THRESHOLD # Set auto_threshold to True if the threshold is preset. if self.threshold is not None: self.auto_threshold = True # Init with empty CommitInfo. Will update in UpdateCurrentCommit(). self.current_commit = common.CommitInfo()
def UpdateCurrentCommit(self): """Updates self.current_commit.""" result = self.Git(['show', '--no-patch', '--format=%h %ct %s']) if result.returncode != 0: logging.error('Failed to update current commit info.') return fields = result.output.strip().split() if len(fields) < 3: logging.error('Failed to update current commit info.') return sha1 = fields[0] timestamp = int(fields[1]) title = ' '.join(fields[2:]) self.current_commit = common.CommitInfo(sha1=sha1, title=title, timestamp=timestamp)
class TestGitBisector(cros_test_lib.MockTempDirTestCase): """Tests GitBisector class.""" BOARD = 'samus' TEST_NAME = 'graphics_WebGLAquarium' METRIC = 'avg_fps_1000_fishes/summary/value' REPORT_FILE = 'reports.json' DUT_ADDR = '192.168.1.1' DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR) # Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via # copy.deepcopy() as their users are likely to change the content. GOOD_COMMIT_SHA1 = '44af5c9a5505' GOOD_COMMIT_TIMESTAMP = 1486526594 GOOD_COMMIT_SCORE = common.Score([100]) GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1, timestamp=GOOD_COMMIT_TIMESTAMP, title='good', label='last-known-good ', score=GOOD_COMMIT_SCORE) BAD_COMMIT_SHA1 = '6a163bb66c3e' BAD_COMMIT_TIMESTAMP = 1486530021 BAD_COMMIT_SCORE = common.Score([80]) BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1, timestamp=BAD_COMMIT_TIMESTAMP, title='bad', label='last-known-bad ', score=BAD_COMMIT_SCORE) CULPRIT_COMMIT_SHA1 = '12345abcde' CULPRIT_COMMIT_TIMESTAMP = 1486530000 CULPRIT_COMMIT_SCORE = common.Score([81]) CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1, timestamp=CULPRIT_COMMIT_TIMESTAMP, title='bad', score=CULPRIT_COMMIT_SCORE) THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side. THRESHOLD = 5 # Distance between good score and splitter. REPEAT = 3 def setUp(self): """Sets up test case.""" self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir, board=self.BOARD, reuse_repo=True, good=self.GOOD_COMMIT_SHA1, bad=self.BAD_COMMIT_SHA1, remote=self.DUT, eval_repeat=self.REPEAT, auto_threshold=False, reuse_eval=False, eval_raise_on_error=False, skip_failed_commit=False) self.evaluator = evaluator_module.Evaluator(self.options) self.builder = builder_module.Builder(self.options) self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) self.repo_dir = os.path.join(self.tempdir, builder_module.Builder.DEFAULT_REPO_DIR) def setDefaultCommitInfo(self): """Sets up default commit info.""" self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) def testInit(self): """Tests GitBisector() to expect default data members.""" self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitMissingRequiredArgs(self): """Tests that GitBisector raises error when missing required arguments.""" options = cros_test_lib.EasyAttr() with self.assertRaises(common.MissingRequiredOptionsException) as cm: git_bisector.GitBisector(options, self.builder, self.evaluator) exception_message = str(cm.exception) self.assertIn('Missing command line', exception_message) self.assertIn('GitBisector', exception_message) for arg in git_bisector.GitBisector.REQUIRED_ARGS: self.assertIn(arg, exception_message) def testCheckCommitFormat(self): """Tests CheckCommitFormat().""" sha1 = '900d900d' self.assertEqual(sha1, git_bisector.GitBisector.CheckCommitFormat(sha1)) not_sha1 = 'bad_sha1' self.assertIsNone(git_bisector.GitBisector.CheckCommitFormat(not_sha1)) def testSetUp(self): """Tests that SetUp() calls builder.SetUp().""" with mock.patch.object(builder_module.Builder, 'SetUp') as builder_mock: self.bisector.SetUp() builder_mock.assert_called_with() def testGit(self): """Tests that Git() invokes git.RunGit() as expected.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(['ls']) self.bisector.Git(['ls']) def testUpdateCurrentCommit(self): """Tests that UpdateCurrentCommit() updates self.current_commit.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult( partial_mock.In('show'), output='4fcbdaf6 1493010050 Add "cros bisect" command.\n') self.bisector.UpdateCurrentCommit() current_commit = self.bisector.current_commit self.assertEqual('4fcbdaf6', current_commit.sha1) self.assertEqual(1493010050, current_commit.timestamp) self.assertEqual('Add "cros bisect" command.', current_commit.title) def testUpdateCurrentCommitFail(self): """Tests UpdateCurrentCommit() when 'git show' returns nonzero.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(partial_mock.In('show'), returncode=128) self.bisector.UpdateCurrentCommit() self.assertTrue(not self.bisector.current_commit) def testUpdateCurrentCommitFailUnexpectedOutput(self): """Tests UpdateCurrentCommit() when 'git show' gives unexpected output.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(partial_mock.In('show'), output='Not expected') self.bisector.UpdateCurrentCommit() self.assertTrue(not self.bisector.current_commit) def MockOutBuildDeployEvaluateForSanityCheck(self): """Mocks out BuildDeployEvaluate behavior for SaintyCheck test. It mocks UpdateCurrentCommit() to emit good and bad commits. And mocks CheckLastEvaluate() to return good and bad commit score. """ commit_info_list = [ common.CommitInfo(sha1=self.GOOD_COMMIT_SHA1, title='good', timestamp=self.GOOD_COMMIT_TIMESTAMP), common.CommitInfo(sha1=self.BAD_COMMIT_SHA1, title='bad', timestamp=self.BAD_COMMIT_TIMESTAMP) ] # This mock is problematic. The side effect should modify "self" when # the member method is called. def UpdateCurrentCommitSideEffect(): self.bisector.current_commit = commit_info_list.pop(0) self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit', side_effect=UpdateCurrentCommitSideEffect) self.PatchObject( evaluator_module.Evaluator, 'CheckLastEvaluate', side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE]) def testGetCommitTimestamp(self): """Tests GetCommitTimeStamp by mocking git.RunGit.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) self.assertEqual( self.GOOD_COMMIT_TIMESTAMP, self.bisector.GetCommitTimestamp(self.GOOD_COMMIT_SHA1)) def testGetCommitTimestampNotFound(self): """Tests GetCommitTimeStamp when the commit is not found.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) nonexist_sha1 = '12345' # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder(['show', nonexist_sha1]), output='commit does not exist') self.assertIsNone(self.bisector.GetCommitTimestamp(nonexist_sha1)) def testSanityCheck(self): """Tests SanityCheck(). It tests by mocking out git commands called by SanityCheck(). """ git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for DoesCommitExistInRepo. git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) self.assertTrue(self.bisector.SanityCheck()) def testSanityCheckGoodCommitNotExist(self): """Tests SanityCheck() that good commit does not exist.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) sync_to_head_mock = self.PatchObject(builder_module.Builder, 'SyncToHead') # Mock git result for DoesCommitExistInRepo to return False. git_mock.AddRunGitResult(partial_mock.InOrder( ['rev-list', self.GOOD_COMMIT_SHA1]), returncode=128) # Mock invalid result for GetCommitTimestamp to return None. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output='invalid commit') git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output='invalid commit') self.assertFalse(self.bisector.SanityCheck()) # SyncToHead() called because DoesCommitExistInRepo() returns False for # good commit. sync_to_head_mock.assert_called() def testSanityCheckSyncToHeadWorks(self): """Tests SanityCheck() that good and bad commit do not exist. As good and bad commit do not exist, it calls SyncToHead(). """ git_mock = self.StartPatcher(GitMock(self.repo_dir)) sync_to_head_mock = self.PatchObject(builder_module.Builder, 'SyncToHead') # Mock git result for DoesCommitExistInRepo to return False. git_mock.AddRunGitResult(partial_mock.InOrder( ['rev-list', self.GOOD_COMMIT_SHA1]), returncode=128) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) self.assertTrue(self.bisector.SanityCheck()) # After SyncToHead, found both bad and good commit. sync_to_head_mock.assert_called() def testSanityCheckWrongTimeOrder(self): """Tests SanityCheck() that good and bad commit have wrong time order.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for DoesCommitExistInRepo. git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) # Mock git result for GetCommitTimestamp, but swap timestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) self.assertFalse(self.bisector.SanityCheck()) def testObtainBisectBoundaryScoreImpl(self): """Tests ObtainBisectBoundaryScoreImpl().""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( git_bisector.GitBisector, 'BuildDeployEval', side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE]) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) self.assertEqual([ mock.call(self.repo_dir, ['checkout', self.GOOD_COMMIT_SHA1], error_code_ok=True), mock.call(self.repo_dir, ['checkout', self.BAD_COMMIT_SHA1], error_code_ok=True) ], git_mock.call_args_list) build_deploy_eval_mock.assert_called() def testObtainBisectBoundaryScore(self): """Tests ObtainBisectBoundaryScore(). Normal case.""" def MockedObtainBisectBoundaryScoreImpl(good_side): if good_side: self.bisector.current_commit = copy.deepcopy( self.GOOD_COMMIT_INFO) else: self.bisector.current_commit = copy.deepcopy( self.BAD_COMMIT_INFO) return self.bisector.current_commit.score obtain_score_mock = self.PatchObject( git_bisector.GitBisector, 'ObtainBisectBoundaryScoreImpl', side_effect=MockedObtainBisectBoundaryScoreImpl) self.assertTrue(self.bisector.ObtainBisectBoundaryScore()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.good_commit_info.score) self.assertEqual('last-known-good ', self.bisector.good_commit_info.label) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.bad_commit_info.score) self.assertEqual('last-known-bad ', self.bisector.bad_commit_info.label) obtain_score_mock.assert_called() def testObtainBisectBoundaryScoreBadScoreUnavailable(self): """Tests ObtainBisectBoundaryScore(). Bad score unavailable.""" def UpdateCurrentCommitSideEffect(good_side): if good_side: self.bisector.current_commit = copy.deepcopy( self.GOOD_COMMIT_INFO) else: self.bisector.current_commit = copy.deepcopy( self.BAD_COMMIT_INFO) self.bisector.current_commit.score = None return self.bisector.current_commit.score obtain_score_mock = self.PatchObject( git_bisector.GitBisector, 'ObtainBisectBoundaryScoreImpl', side_effect=UpdateCurrentCommitSideEffect) self.assertFalse(self.bisector.ObtainBisectBoundaryScore()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.good_commit_info.score) self.assertEqual('last-known-good ', self.bisector.good_commit_info.label) self.assertIsNone(self.bisector.bad_commit_info.score) self.assertEqual('last-known-bad ', self.bisector.bad_commit_info.label) obtain_score_mock.assert_called() def testGetThresholdFromUser(self): """Tests GetThresholdFromUser().""" logging.notice('testGetThresholdFromUser') self.setDefaultCommitInfo() input_mock = self.PatchObject(cros_build_lib, 'GetInput', return_value=self.THRESHOLD_SPLITTER) self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(self.THRESHOLD, self.bisector.threshold) input_mock.assert_called() def testGetThresholdFromUserAutoPick(self): """Tests GetThresholdFromUser().""" self.setDefaultCommitInfo() self.bisector.auto_threshold = True self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(10, self.bisector.threshold) def testGetThresholdFromUserOutOfBoundFail(self): """Tests GetThresholdFromUser() with out-of-bound input.""" self.setDefaultCommitInfo() input_mock = self.PatchObject(cros_build_lib, 'GetInput', side_effect=['0', '1000', '-10']) self.assertFalse(self.bisector.GetThresholdFromUser()) self.assertIsNone(self.bisector.threshold) self.assertEqual(3, input_mock.call_count) def testGetThresholdFromUserRetrySuccess(self): """Tests GetThresholdFromUser() with retry.""" self.setDefaultCommitInfo() input_mock = self.PatchObject( cros_build_lib, 'GetInput', side_effect=['not_a_number', '1000', self.THRESHOLD_SPLITTER]) self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(self.THRESHOLD, self.bisector.threshold) self.assertEqual(3, input_mock.call_count) def testBuildDeploy(self): """Tests BuildDeploy().""" # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) build_to_deploy = '/build/to/deploy' build_mock = self.PatchObject(builder_module.Builder, 'Build', return_value=build_to_deploy) deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy') self.assertTrue(self.bisector.BuildDeploy()) build_label = self.GOOD_COMMIT_INFO.sha1 build_mock.assert_called_with(build_label) deploy_mock.assert_called_with(self.DUT, build_to_deploy, build_label) def testBuildDeployBuildFail(self): """Tests BuildDeploy() with Build() failure.""" # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) # Build() failed. build_mock = self.PatchObject(builder_module.Builder, 'Build', return_value=None) deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy') self.assertFalse(self.bisector.BuildDeploy()) build_mock.assert_called() deploy_mock.assert_not_called() def PatchObjectForBuildDeployEval(self): """Returns a dict of patch objects. The patch objects are to mock: git_bisector.UpdateCurrentCommit() evaluator.CheckLastEvaluate() git_bisector.BuildDeploy() evaluator.Evaluate() """ return { 'UpdateCurrentCommit': self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit'), 'CheckLastEvaluate': self.PatchObject(evaluator_module.Evaluator, 'CheckLastEvaluate'), 'BuildDeploy': self.PatchObject(git_bisector.GitBisector, 'BuildDeploy', return_value=True), 'Evaluate': self.PatchObject(evaluator_module.Evaluator, 'Evaluate') } def testBuildDeployEvalShortcutCheckLastEvaluate(self): """Tests BuildDeployEval() with CheckLastEvaluate() found last score.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = self.GOOD_COMMIT_SCORE score = self.bisector.BuildDeployEval() self.assertEqual(self.GOOD_COMMIT_SCORE, score) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in ['UpdateCurrentCommit', 'CheckLastEvaluate']: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) for method_called in ['BuildDeploy', 'Evaluate']: mocks[method_called].assert_not_called() def AssertBuildDeployEvalMocksAllCalled(self, mocks): for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy', 'Evaluate' ]: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, self.GOOD_COMMIT_SHA1, self.REPEAT) def testBuildDeployEvalNoCheckLastEvaluate(self): """Tests BuildDeployEval() without last score.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testBuildDeployEvalBuildFail(self): """Tests BuildDeployEval() with BuildDeploy failure.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['BuildDeploy'].return_value = False score = self.bisector.BuildDeployEval() self.assertFalse(score) self.assertFalse(self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy' ]: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) mocks['Evaluate'].assert_not_called() def testBuildDeployEvalNoCheckLastEvaluateSpecifyEvalLabel(self): """Tests BuildDeployEval() with eval_label specified.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE eval_label = 'customized_label' self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval(eval_label=eval_label)) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy', 'Evaluate' ]: mocks[method_called].assert_called() # Use given label instead of SHA1 as eval label. mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT) @staticmethod def _DummyMethod(): """A dummy method for test to call and mock.""" def testBuildDeployEvalNoCheckLastEvaluateSpecifyBuildDeploy(self): """Tests BuildDeployEval() with customize_build_deploy specified.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE dummy_method = self.PatchObject(TestGitBisector, '_DummyMethod', return_value=True) eval_label = 'customized_label' self.assertEqual( self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval( eval_label=eval_label, customize_build_deploy=TestGitBisector._DummyMethod)) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'Evaluate' ]: mocks[method_called].assert_called() mocks['BuildDeploy'].assert_not_called() dummy_method.assert_called() # Use given label instead of SHA1 as eval label. mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT) def testBuildDeployEvalRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error set.""" self.options.eval_raise_on_error = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertRaises(Exception, self.bisector.BuildDeployEval) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testBuildDeployEvalSuppressRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error unset.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertFalse(self.bisector.BuildDeployEval()) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testLabelBuild(self): """Tests LabelBuild().""" # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([70]))) # Better than bad score, but not good enough. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([85]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([95]))) # Better than the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([98]))) # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([110]))) # No score, default bad. self.assertEqual(bad, self.bisector.LabelBuild(None)) self.assertEqual(bad, self.bisector.LabelBuild(common.Score())) def testLabelBuildSkipNoScore(self): """Tests LabelBuild().""" self.options.skip_failed_commit = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # No score, skip. self.assertEqual('skip', self.bisector.LabelBuild(None)) self.assertEqual('skip', self.bisector.LabelBuild(common.Score())) def testLabelBuildLowerIsBetter(self): """Tests LabelBuild() in lower-is-better condition.""" # Reverse good(80) and bad(100) score (lower is better), same threshold. self.bisector.good_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([70]))) # Worse than good score, but still better than margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([80]))) self.assertEqual(good, self.bisector.LabelBuild(common.Score([82]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([85]))) # Worse than the margin. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([88]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([95]))) # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([110]))) def testGitBisect(self): """Tests GitBisect().""" git_mock = self.PatchObject(git_bisector.GitBisector, 'Git', return_value=cros_build_lib.CommandResult( cmd=['git', 'bisect', 'reset'], output='We are not bisecting.', returncode=0)) result, done = self.bisector.GitBisect(['reset']) git_mock.assert_called_with(['bisect', 'reset']) self.assertFalse(done) self.assertEqual('We are not bisecting.', result.output) self.assertListEqual(['git', 'bisect', 'reset'], result.cmd) def testGitBisectDone(self): """Tests GitBisect() when culprit is found.""" git_mock = self.PatchObject( git_bisector.GitBisector, 'Git', return_value=cros_build_lib.CommandResult( cmd=['git', 'bisect', 'bad'], output='abcedf is the first bad commit\ncommit abcdef', returncode=0)) result, done = self.bisector.GitBisect(['bad']) git_mock.assert_called_with(['bisect', 'bad']) self.assertTrue(done) self.assertListEqual(['git', 'bisect', 'bad'], result.cmd) def testRun(self): """Tests Run().""" bisector_mock = self.StartPatcher(GitBisectorMock()) bisector_mock.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) bisector_mock.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) bisector_mock.threshold = self.THRESHOLD bisector_mock.git_bisect_args_result = [ (['reset'], (None, False)), (['start'], (None, False)), (['bad', self.BAD_COMMIT_SHA1], (None, False)), (['good', self.GOOD_COMMIT_SHA1], (None, False)), (['bad'], (cros_build_lib.CommandResult( cmd=['git', 'bisect', 'bad'], output='%s is the first bad commit\ncommit %s' % (self.CULPRIT_COMMIT_SHA1, self.CULPRIT_COMMIT_SHA1), returncode=0), True)), # bisect bad (assume it found the first bad commit). (['log'], (None, False)), (['reset'], (None, False)) ] bisector_mock.build_deploy_eval_current_commit = [ self.CULPRIT_COMMIT_INFO ] bisector_mock.build_deploy_eval_result = [self.CULPRIT_COMMIT_SCORE] bisector_mock.label_build_result = ['bad'] run_result = self.bisector.Run() self.assertTrue(bisector_mock.patched['PrepareBisect'].called) self.assertEqual(7, bisector_mock.patched['GitBisect'].call_count) self.assertTrue(bisector_mock.patched['BuildDeployEval'].called) self.assertTrue(bisector_mock.patched['LabelBuild'].called) self.assertEqual(self.CULPRIT_COMMIT_SHA1, run_result)
def testEmpty(self): """Tests that empty CommitInfo's data members are initialized correctly.""" info = common.CommitInfo() self.assertEqual( "CommitInfo(sha1='', title='', score=Score(values=[]), label='', " "timestamp=0)", repr(info))
class TestChromeOnCrosBisector(cros_test_lib.MockTempDirTestCase): """Tests ChromeOnCrosBisector class.""" BOARD = 'samus' TEST_NAME = 'graphics_WebGLAquarium' METRIC = 'avg_fps_1000_fishes/summary/value' REPORT_FILE = 'reports.json' DUT_ADDR = '192.168.1.1' DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR) # Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via # copy.deepcopy() as their users are likely to change the content. GOOD_COMMIT_SHA1 = '44af5c9a5505' GOOD_COMMIT_TIMESTAMP = 1486526594 GOOD_COMMIT_SCORE = common.Score([100]) GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1, timestamp=GOOD_COMMIT_TIMESTAMP, title='good', label='last-known-good ', score=GOOD_COMMIT_SCORE) BAD_COMMIT_SHA1 = '6a163bb66c3e' BAD_COMMIT_TIMESTAMP = 1486530021 BAD_COMMIT_SCORE = common.Score([80]) BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1, timestamp=BAD_COMMIT_TIMESTAMP, title='bad', label='last-known-bad ', score=BAD_COMMIT_SCORE) GOOD_CROS_VERSION = 'R60-9592.50.0' BAD_CROS_VERSION = 'R60-9592.51.0' CULPRIT_COMMIT_SHA1 = '12345abcde' CULPRIT_COMMIT_TIMESTAMP = 1486530000 CULPRIT_COMMIT_SCORE = common.Score([81]) CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1, timestamp=CULPRIT_COMMIT_TIMESTAMP, title='bad', score=CULPRIT_COMMIT_SCORE) THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side. THRESHOLD = 5 # Distance between good score and splitter. REPEAT = 3 GOOD_METADATA_CONTENT = '\n'.join([ '{', ' "metadata-version": "2",', ' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",', ' "suite_scheduling": true,', ' "build_id": 1644146,', ' "version": {', ' "full": "R60-9592.50.0",', ' "android-branch": "git_mnc-dr-arc-m60",', ' "chrome": "60.0.3112.53",', ' "platform": "9592.50.0",', ' "milestone": "60",', ' "android": "4150402"', ' }', '}' ]) def setUp(self): """Sets up test case.""" self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir, board=self.BOARD, reuse_repo=True, good=self.GOOD_COMMIT_SHA1, bad=self.BAD_COMMIT_SHA1, remote=self.DUT, eval_repeat=self.REPEAT, auto_threshold=False, reuse_eval=False, cros_flash_sleep=0.01, cros_flash_retry=3, cros_flash_backoff=1, eval_raise_on_error=False, skip_failed_commit=False) self.repo_dir = os.path.join(self.tempdir, builder_module.Builder.DEFAULT_REPO_DIR) self.SetUpBisector() def SetUpBisector(self): """Instantiates self.bisector using self.options.""" self.evaluator = DummyEvaluator(self.options) self.builder = builder_module.Builder(self.options) self.bisector = chrome_on_cros_bisector.ChromeOnCrosBisector( self.options, self.builder, self.evaluator) def SetUpBisectorWithCrosVersion(self): """Instantiates self.bisector using CrOS version as good and bad options.""" self.options.good = self.GOOD_CROS_VERSION self.options.bad = self.BAD_CROS_VERSION self.SetUpBisector() def SetDefaultCommitInfo(self): """Sets up default commit info.""" self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) def testInit(self): """Tests __init__() with SHA1 as good and bad options.""" self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertIsNone(self.bisector.good_cros_version) self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) self.assertIsNone(self.bisector.bad_cros_version) self.assertFalse(self.bisector.bisect_between_cros_version) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitCrosVersion(self): """Tests __init__() with CrOS version as good and bad options.""" self.SetUpBisectorWithCrosVersion() self.assertEqual(self.GOOD_CROS_VERSION, self.bisector.good_cros_version) self.assertIsNone(self.bisector.good_commit) self.assertEqual(self.BAD_CROS_VERSION, self.bisector.bad_cros_version) self.assertIsNone(self.bisector.bad_commit) self.assertTrue(self.bisector.bisect_between_cros_version) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitMissingRequiredArgs(self): """Tests that ChromeOnCrosBisector raises for missing required argument.""" options = cros_test_lib.EasyAttr() with self.assertRaises(common.MissingRequiredOptionsException) as cm: chrome_on_cros_bisector.ChromeOnCrosBisector( options, self.builder, self.evaluator) exception_message = str(cm.exception) self.assertIn('Missing command line', exception_message) self.assertIn('ChromeOnCrosBisector', exception_message) for arg in chrome_on_cros_bisector.ChromeOnCrosBisector.REQUIRED_ARGS: self.assertIn(arg, exception_message) def testCheckCommitFormat(self): """Tests CheckCommitFormat().""" CheckCommitFormat = ( chrome_on_cros_bisector.ChromeOnCrosBisector.CheckCommitFormat) self.assertEqual(self.GOOD_COMMIT_SHA1, CheckCommitFormat(self.GOOD_COMMIT_SHA1)) self.assertEqual(self.GOOD_CROS_VERSION, CheckCommitFormat(self.GOOD_CROS_VERSION)) self.assertEqual('R60-9592.50.0', CheckCommitFormat('60.9592.50.0')) invalid = 'bad_sha1' self.assertIsNone(CheckCommitFormat(invalid)) def testObtainBisectBoundaryScoreImpl(self): """Tests ObtainBisectBoundaryScoreImpl().""" git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval') build_deploy_eval_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) self.assertEqual([ mock.call(customize_build_deploy=None, eval_label=None), mock.call(customize_build_deploy=None, eval_label=None) ], build_deploy_eval_mock.call_args_list) def testObtainBisectBoundaryScoreImplCrosVersion(self): """Tests ObtainBisectBoundaryScoreImpl() with CrOS version.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() being run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') evaluate_mock.return_value = self.GOOD_COMMIT_SCORE self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) evaluate_mock.assert_called_with(self.DUT, 'cros_%s' % self.GOOD_CROS_VERSION, self.REPEAT) evaluate_mock.return_value = self.BAD_COMMIT_SCORE self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) evaluate_mock.assert_called_with(self.DUT, 'cros_%s' % self.BAD_CROS_VERSION, self.REPEAT) def testObtainBisectBoundaryScoreImplCrosVersionFlashError(self): """Tests ObtainBisectBoundaryScoreImpl() with CrOS version.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() being run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') flash_cros_image_mock.side_effect = flash.FlashError('Flash failed.') with self.assertRaises(flash.FlashError): self.bisector.ObtainBisectBoundaryScoreImpl(True) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) evaluate_mock.assert_not_called() with self.assertRaises(flash.FlashError): self.bisector.ObtainBisectBoundaryScoreImpl(False) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) evaluate_mock.assert_not_called() def testGetCrosXbuddyPath(self): """Tests GetCrosXbuddyPath().""" self.assertEqual( 'xbuddy://remote/%s/%s/test' % (self.BOARD, self.GOOD_CROS_VERSION), self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) def testExchangeChromeSanityCheck(self): """Tests the flow of exchanging Chrome between good and bad CrOS.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() has been run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 # Inject commit_info and threshold as if # bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser() # has been run. self.SetDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # Try bad Chrome first. git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') expected_evaluate_calls = [ mock.call(self.DUT, x, self.REPEAT) for x in [ 'cros_%s_cr_%s' % (self.GOOD_CROS_VERSION, self.BAD_COMMIT_SHA1), 'cros_%s_cr_%s' % (self.BAD_CROS_VERSION, self.GOOD_COMMIT_SHA1) ] ] # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') expected_flash_cros_calls = [ mock.call(self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)), mock.call(self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) ] # Make sure bisector.BuildDeploy() is also called. build_deploy_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy') # Assume culprit commit is in Chrome side, i.e. first score is bad. evaluate_mock.side_effect = [ self.BAD_COMMIT_SCORE, self.GOOD_COMMIT_SCORE ] self.assertTrue(self.bisector.ExchangeChromeSanityCheck()) flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls) evaluate_mock.assert_has_calls(expected_evaluate_calls) self.assertEqual(2, build_deploy_mock.call_count) flash_cros_image_mock.reset_mock() evaluate_mock.reset_mock() build_deploy_mock.reset_mock() # Assume culprit commit is not in Chrome side, i.e. first score is good. evaluate_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] self.assertFalse(self.bisector.ExchangeChromeSanityCheck()) flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls) evaluate_mock.assert_has_calls(expected_evaluate_calls) self.assertEqual(2, build_deploy_mock.call_count) def testExchangeChromeSanityCheckFlashError(self): """Tests the flow of exchanging Chrome between good and bad CrOS.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() has been run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 # Inject commit_info and threshold as if # bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser() # has been run. self.SetDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # Try bad Chrome first. git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage', side_effect=flash.FlashError('Flash failed.')) build_deploy_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy') with self.assertRaises(flash.FlashError): self.bisector.ExchangeChromeSanityCheck() evaluate_mock.assert_not_called() flash_cros_image_mock.assert_called() build_deploy_mock.assert_not_called() def testFlashImage(self): """Tests FlashImage().""" flash_mock = self.PatchObject(flash, 'Flash') xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testFlashImageRetry(self): """Tests FlashImage() with retry success.""" flash_mock_call_counter = itertools.count() def flash_mock_return(*unused_args, **unused_kwargs): nth_call = next(flash_mock_call_counter) if nth_call < 3: raise flash.FlashError('Flash failed.') flash_mock = self.PatchObject(flash, 'Flash') flash_mock.side_effect = flash_mock_return xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testFlashImageRetryFailed(self): """Tests FlashImage() with retry failed.""" flash_mock = self.PatchObject(flash, 'Flash') flash_mock.side_effect = flash.FlashError('Flash failed.') xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) with self.assertRaises(flash.FlashError): self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testCrosVersionToChromeCommit(self): """Tests CrosVersionToChromeCommit().""" metadata_url = ( 'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' % (self.BOARD, self.GOOD_CROS_VERSION)) gs_mock = self.StartPatcher(gs_unittest.GSContextMock()) gs_mock.AddCmdResult(['cat', metadata_url], output=self.GOOD_METADATA_CONTENT) git_log_content = '\n'.join([ '8967dd66ad72 (tag: 60.0.3112.53) Publish DEPS for Chromium ' '60.0.3112.53', '27ed0cc0c2f4 Incrementing VERSION to 60.0.3112.53' ]) git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( ['log', '--oneline', '-n', '2', '60.0.3112.53'], output=git_log_content) self.bisector.gs_ctx = gs.GSContext() self.assertEqual( '27ed0cc0c2f4', self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) def testCrosVersionToChromeCommitFail(self): """Tests failure case of CrosVersionToChromeCommit().""" metadata_url = ( 'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' % (self.BOARD, self.GOOD_CROS_VERSION)) gs_mock = self.StartPatcher(gs_unittest.GSContextMock()) gs_mock.AddCmdResult(['cat', metadata_url], returncode=1) self.bisector.gs_ctx = gs.GSContext() self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) metadata_content = 'not_a_json' gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) metadata_content = '\n'.join([ '{', ' "metadata-version": "2",', ' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",', ' "suite_scheduling": true,', ' "build_id": 1644146,', ' "version": {}', '}' ]) gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) gs_mock.AddCmdResult(['cat', metadata_url], output=self.GOOD_METADATA_CONTENT) git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( ['log', '--oneline', '-n', '2', '60.0.3112.53'], returncode=128) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) def testResolveChromeBisectRangeFromCrosVersion(self): """Tests ResolveChromeBisectRangeFromCrosVersion().""" self.SetUpBisectorWithCrosVersion() cros_to_chrome_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'CrosVersionToChromeCommit') cros_to_chrome_mock.side_effect = [ self.GOOD_COMMIT_SHA1, self.BAD_COMMIT_SHA1 ] self.assertTrue( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) self.assertTrue(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertTrue(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) cros_to_chrome_mock.assert_has_calls([ mock.call(self.GOOD_CROS_VERSION), mock.call(self.BAD_CROS_VERSION) ]) cros_to_chrome_mock.reset_mock() cros_to_chrome_mock.side_effect = [None] self.assertFalse( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) cros_to_chrome_mock.assert_called_with(self.GOOD_CROS_VERSION) cros_to_chrome_mock.reset_mock() cros_to_chrome_mock.side_effect = [self.GOOD_COMMIT_SHA1, None] self.assertFalse( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) cros_to_chrome_mock.assert_has_calls([ mock.call(self.GOOD_CROS_VERSION), mock.call(self.BAD_CROS_VERSION) ]) def testPrepareBisect(self): """Tests PrepareBisect().""" # Pass SanityCheck(). git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) # Inject score for both side. git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval') build_deploy_eval_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] # Set auto_threshold. self.bisector.auto_threshold = True self.assertTrue(self.bisector.PrepareBisect()) def testPrepareBisectCrosVersion(self): """Tests PrepareBisect() with CrOS version.""" self.SetUpBisectorWithCrosVersion() self.StartPatcher(gs_unittest.GSContextMock()) self.PatchObject(builder_module.Builder, 'SyncToHead') self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'ResolveChromeBisectRangeFromCrosVersion').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'SanityCheck').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'ObtainBisectBoundaryScore').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'GetThresholdFromUser').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'ExchangeChromeSanityCheck').return_value = True self.assertTrue(self.bisector.PrepareBisect())