def testCheckLastEvaluate(self): """Tests CheckLastEvaluate().""" # Report does not exist. self.assertFalse(self.evaluator.CheckLastEvaluate(self.BUILD_LABEL)) # Generate a report for BUILD_LABEL m = self.PatchObject(cros_build_lib, 'GetInput') m.return_value = 'yes' self.evaluator.Evaluate(None, self.BUILD_LABEL) # Found latest evaluation result. self.assertEqual(common.Score([1.0]), self.evaluator.CheckLastEvaluate(self.BUILD_LABEL)) # Yet another unseen build. self.assertFalse(self.evaluator.CheckLastEvaluate(self.BUILD_LABEL2)) # Generate a report for BUILD_LABEL2 m.return_value = 'no' self.evaluator.Evaluate(None, self.BUILD_LABEL2) # Found latest evaluation result. self.assertEqual(common.Score([1.0]), self.evaluator.CheckLastEvaluate(self.BUILD_LABEL)) self.assertEqual(common.Score([0.0]), self.evaluator.CheckLastEvaluate(self.BUILD_LABEL2))
def CheckLastEvaluate(self, build_label, repeat=1): """Checks if previous evaluate report is available. Args: build_label: Build label used for part of report filename and log message. repeat: Run test for N times. Default 1. Returns: Score object stores a list of autotest running results if report available and reuse_eval is set. Score() otherwise. """ if not self.reuse_eval: return common.Score() score_list = [] for nth in range(repeat): report_file = self.GetReportPath(build_label, nth + 1, repeat) if not os.path.isfile(report_file): return common.Score() score = self.GetAutotestMetricValue(report_file) if score is None: return common.Score() score_list.append(score) scores = common.Score(score_list) logging.info( 'Used archived autotest result. Arithmetic mean(%s:%s) = ' '%.3f (build:%s)', self.test_name, self.metric, scores.mean, build_label) return scores
def testBool(self): """Tests Score's boolean conversion. Only Score without value is treated as False. """ score1 = common.Score() self.assertTrue(not score1) self.assertFalse(bool(score1)) score2 = common.Score([0]) self.assertTrue(bool(score2)) self.assertFalse(not score2)
def testBuildDeployEvalSuppressRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error unset.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertFalse(self.bisector.BuildDeployEval()) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks)
def testNotEqual(self): """Tests inequality of two CommitInfo objects.""" info1 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) info2 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2]), label='GOOD', timestamp=100) self.assertNotEqual(info1, info2) self.assertFalse(info1 == info2)
def testLabelBuildLowerIsBetter(self): """Tests LabelBuild() in lower-is-better condition.""" # Reverse good(80) and bad(100) score (lower is better), same threshold. self.bisector.good_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([70]))) # Worse than good score, but still better than margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([80]))) self.assertEqual(good, self.bisector.LabelBuild(common.Score([82]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([85]))) # Worse than the margin. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([88]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([95]))) # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([110])))
def testEvaluate(self): """Tests Evaluate().""" report_path = self.evaluator.GetReportPath(self.BUILD_LABEL) m = self.PatchObject(cros_build_lib, 'GetInput') m.return_value = 'yes' self.assertEqual(common.Score([1.0]), self.evaluator.Evaluate(None, self.BUILD_LABEL)) self.assertEqual('1', osutils.ReadFile(report_path)) m.return_value = 'no' self.assertEqual(common.Score([0.0]), self.evaluator.Evaluate(None, self.BUILD_LABEL)) self.assertEqual('0', osutils.ReadFile(report_path))
def testBuildDeployEvalNoCheckLastEvaluateSpecifyBuildDeploy(self): """Tests BuildDeployEval() with customize_build_deploy specified.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE dummy_method = self.PatchObject(TestGitBisector, '_DummyMethod', return_value=True) eval_label = 'customized_label' self.assertEqual( self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval( eval_label=eval_label, customize_build_deploy=TestGitBisector._DummyMethod)) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'Evaluate' ]: mocks[method_called].assert_called() mocks['BuildDeploy'].assert_not_called() dummy_method.assert_called() # Use given label instead of SHA1 as eval label. mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT)
def testEqual(self): """Tests equality of two CommitInfo objects.""" info1 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) info2 = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1, 2, 3]), label='GOOD', timestamp=100) self.assertEqual(info1, info2) # In Python 2.7, __ne__() doesn't delegates to "not __eq__()" so that the # sanity check is necessary. self.assertFalse(info1 != info2)
def testScoreInit(self): """Tests that Score() sets up data member correctly.""" score = common.Score([1, 2, 3]) self.assertEqual('Score(values=[1.0, 2.0, 3.0])', repr(score)) self.assertEqual( 'Score(values=[1.0, 2.0, 3.0], mean=2.000, var=1.000, std=1.000)', str(score)) self.assertEqual(3, len(score))
def testBuildDeployEvalRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error set.""" self.options.eval_raise_on_error = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertRaises(Exception, self.bisector.BuildDeployEval) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks)
def testLabelBuild(self): """Tests LabelBuild().""" # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([70]))) # Better than bad score, but not good enough. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([85]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([95]))) # Better than the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([98]))) # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([110]))) # No score, default bad. self.assertEqual(bad, self.bisector.LabelBuild(None)) self.assertEqual(bad, self.bisector.LabelBuild(common.Score()))
def testScoreUpdate(self): """Tests that Update() sets up data member correctly.""" score = common.Score([1, 2, 3]) score.Update([2, 4, 6, 8]) self.assertEqual('Score(values=[2.0, 4.0, 6.0, 8.0])', repr(score)) self.assertEqual( 'Score(values=[2.0, 4.0, 6.0, 8.0], mean=5.000, var=6.667, std=2.582)', str(score)) self.assertEqual(4, len(score))
def testNotEqual(self): """Tests inequality of two Score objects.""" score1 = common.Score([1, 2]) score2 = common.Score([1, 2, 3]) self.assertNotEqual(score1, score2) self.assertTrue(score1 != score2) self.assertFalse(score1 == score2) score3 = common.Score([1, 3]) self.assertNotEqual(score1, score3) self.assertTrue(score1 != score3) self.assertFalse(score1 == score3) score4 = common.Score() score5 = common.Score([0]) self.assertNotEqual(score4, score5) self.assertTrue(score4 != score5) self.assertFalse(score4 == score5)
def testAssigned(self): """Tests that CommitInfo constrcutor sets up data members correctly.""" info = common.CommitInfo(sha1='abcdef', title='test', score=common.Score(values=[1]), label='GOOD', timestamp=100) self.assertEqual( "CommitInfo(sha1='abcdef', title='test', score=Score(values=[1.0]), " "label='GOOD', timestamp=100)", repr(info))
def CheckLastEvaluate(self, build_label, unused_repeat=1): """Checks if previous evaluate report is available. Args: build_label: Build label used for part of report filename and log message. unused_repeat: Unused. Returns: Score([1.0]) if previous result for the build_label is 'Yes'. Score([0.0]) if previous result for the build_label is 'No'. Score() if previous result does not exist or reuse_eval is unset. """ if self.reuse_eval: report_path = self.GetReportPath(build_label) if os.path.isfile(report_path): content = osutils.ReadFile(report_path) if content == '1': return common.Score([1.0]) elif content == '0': return common.Score([0.0]) return common.Score()
def Evaluate(self, remote, build_label, repeat=1): """Runs autotest N-times on DUT and extracts the designated metric values. Args: remote: DUT to evaluate (refer lib.commandline.Device). build_label: Build label used for part of report filename and log message. repeat: Run test for N times. Default 1. Returns: Score object stores a list of autotest running results. """ if repeat == 1: times_str = 'once' elif repeat == 2: times_str = 'twice' else: times_str = '%d times' % repeat logging.info( 'Evaluating build %s performance on DUT %s by running autotest %s %s ' 'to get metric %s', build_label, remote.raw, self.test_name, times_str, self.metric) score_list = [] for nth in range(repeat): report_file = self.GetReportPath(build_label, nth + 1, repeat) score = self._EvaluateOnce(remote, report_file) if score is None: return common.Score() logging.info( 'Run autotest %d/%d. Got result: %s:%s = %.3f (build:%s DUT:%s).', nth + 1, repeat, self.test_name, self.metric, score, build_label, remote.raw) score_list.append(score) scores = common.Score(score_list) logging.info( 'Successfully ran autotest %d times. Arithmetic mean(%s:%s) = %.3f', repeat, self.test_name, self.metric, scores.mean) return scores
def testLabelBuildSkipNoScore(self): """Tests LabelBuild().""" self.options.skip_failed_commit = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # No score, skip. self.assertEqual('skip', self.bisector.LabelBuild(None)) self.assertEqual('skip', self.bisector.LabelBuild(common.Score()))
def testBuildDeployEvalNoCheckLastEvaluate(self): """Tests BuildDeployEval() without last score.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks)
def Evaluate(self, unused_remote, build_label, unused_repeat=1): """Prompts user if the build is good or bad. Args: unused_remote: Unused args. build_label: Build label used for part of report filename and log message. unused_repeat: Unused args. Returns: Score([1.0]) if it is a good build. Otherwise, Score([0.0]). """ report_path = self.GetReportPath(build_label) prompt = 'Is %s a good build on the DUT?' % build_label is_good = cros_build_lib.BooleanPrompt(prompt=prompt) score = 1.0 if is_good else 0.0 osutils.WriteFile(report_path, '%d' % score) return common.Score([score])
def BuildDeployEval(self, eval_label=None, customize_build_deploy=None): """Builds the image, deploys to DUT and evaluates performance. Args: build_deploy: If set, builds current commit and deploys it to DUT. eval_label: Label for the evaluation. Default: current commit SHA1. customize_build_deploy: Method object if specified, call it instead of default self.BuildDeploy(). Returns: Evaluation result. Raises: GitBisectorException if self.eval_raise_on_error and score is empty. """ self.UpdateCurrentCommit() if not eval_label: eval_label = self.current_commit.sha1 score = self.evaluator.CheckLastEvaluate(eval_label, self.eval_repeat) if len(score) > 0: logging.info('Found last evaluated result for %s: %s', eval_label, score) self.current_commit.score = score else: if customize_build_deploy: build_deploy_result = customize_build_deploy() else: build_deploy_result = self.BuildDeploy() if build_deploy_result: self.current_commit.score = self.evaluator.Evaluate( self.remote, eval_label, self.eval_repeat) else: logging.error('Builder fails to build/deploy for %s', eval_label) self.current_commit.score = common.Score() self.bisect_log.append(self.current_commit) if not self.current_commit.score and self.eval_raise_on_error: raise GitBisectorException('Unable to obtain evaluation score') return self.current_commit.score
def testBuildDeployEvalBuildFail(self): """Tests BuildDeployEval() with BuildDeploy failure.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['BuildDeploy'].return_value = False score = self.bisector.BuildDeployEval() self.assertFalse(score) self.assertFalse(self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy' ]: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) mocks['Evaluate'].assert_not_called()
class TestChromeOnCrosBisector(cros_test_lib.MockTempDirTestCase): """Tests ChromeOnCrosBisector class.""" BOARD = 'samus' TEST_NAME = 'graphics_WebGLAquarium' METRIC = 'avg_fps_1000_fishes/summary/value' REPORT_FILE = 'reports.json' DUT_ADDR = '192.168.1.1' DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR) # Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via # copy.deepcopy() as their users are likely to change the content. GOOD_COMMIT_SHA1 = '44af5c9a5505' GOOD_COMMIT_TIMESTAMP = 1486526594 GOOD_COMMIT_SCORE = common.Score([100]) GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1, timestamp=GOOD_COMMIT_TIMESTAMP, title='good', label='last-known-good ', score=GOOD_COMMIT_SCORE) BAD_COMMIT_SHA1 = '6a163bb66c3e' BAD_COMMIT_TIMESTAMP = 1486530021 BAD_COMMIT_SCORE = common.Score([80]) BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1, timestamp=BAD_COMMIT_TIMESTAMP, title='bad', label='last-known-bad ', score=BAD_COMMIT_SCORE) GOOD_CROS_VERSION = 'R60-9592.50.0' BAD_CROS_VERSION = 'R60-9592.51.0' CULPRIT_COMMIT_SHA1 = '12345abcde' CULPRIT_COMMIT_TIMESTAMP = 1486530000 CULPRIT_COMMIT_SCORE = common.Score([81]) CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1, timestamp=CULPRIT_COMMIT_TIMESTAMP, title='bad', score=CULPRIT_COMMIT_SCORE) THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side. THRESHOLD = 5 # Distance between good score and splitter. REPEAT = 3 GOOD_METADATA_CONTENT = '\n'.join([ '{', ' "metadata-version": "2",', ' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",', ' "suite_scheduling": true,', ' "build_id": 1644146,', ' "version": {', ' "full": "R60-9592.50.0",', ' "android-branch": "git_mnc-dr-arc-m60",', ' "chrome": "60.0.3112.53",', ' "platform": "9592.50.0",', ' "milestone": "60",', ' "android": "4150402"', ' }', '}' ]) def setUp(self): """Sets up test case.""" self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir, board=self.BOARD, reuse_repo=True, good=self.GOOD_COMMIT_SHA1, bad=self.BAD_COMMIT_SHA1, remote=self.DUT, eval_repeat=self.REPEAT, auto_threshold=False, reuse_eval=False, cros_flash_sleep=0.01, cros_flash_retry=3, cros_flash_backoff=1, eval_raise_on_error=False, skip_failed_commit=False) self.repo_dir = os.path.join(self.tempdir, builder_module.Builder.DEFAULT_REPO_DIR) self.SetUpBisector() def SetUpBisector(self): """Instantiates self.bisector using self.options.""" self.evaluator = DummyEvaluator(self.options) self.builder = builder_module.Builder(self.options) self.bisector = chrome_on_cros_bisector.ChromeOnCrosBisector( self.options, self.builder, self.evaluator) def SetUpBisectorWithCrosVersion(self): """Instantiates self.bisector using CrOS version as good and bad options.""" self.options.good = self.GOOD_CROS_VERSION self.options.bad = self.BAD_CROS_VERSION self.SetUpBisector() def SetDefaultCommitInfo(self): """Sets up default commit info.""" self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) def testInit(self): """Tests __init__() with SHA1 as good and bad options.""" self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertIsNone(self.bisector.good_cros_version) self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) self.assertIsNone(self.bisector.bad_cros_version) self.assertFalse(self.bisector.bisect_between_cros_version) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitCrosVersion(self): """Tests __init__() with CrOS version as good and bad options.""" self.SetUpBisectorWithCrosVersion() self.assertEqual(self.GOOD_CROS_VERSION, self.bisector.good_cros_version) self.assertIsNone(self.bisector.good_commit) self.assertEqual(self.BAD_CROS_VERSION, self.bisector.bad_cros_version) self.assertIsNone(self.bisector.bad_commit) self.assertTrue(self.bisector.bisect_between_cros_version) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitMissingRequiredArgs(self): """Tests that ChromeOnCrosBisector raises for missing required argument.""" options = cros_test_lib.EasyAttr() with self.assertRaises(common.MissingRequiredOptionsException) as cm: chrome_on_cros_bisector.ChromeOnCrosBisector( options, self.builder, self.evaluator) exception_message = str(cm.exception) self.assertIn('Missing command line', exception_message) self.assertIn('ChromeOnCrosBisector', exception_message) for arg in chrome_on_cros_bisector.ChromeOnCrosBisector.REQUIRED_ARGS: self.assertIn(arg, exception_message) def testCheckCommitFormat(self): """Tests CheckCommitFormat().""" CheckCommitFormat = ( chrome_on_cros_bisector.ChromeOnCrosBisector.CheckCommitFormat) self.assertEqual(self.GOOD_COMMIT_SHA1, CheckCommitFormat(self.GOOD_COMMIT_SHA1)) self.assertEqual(self.GOOD_CROS_VERSION, CheckCommitFormat(self.GOOD_CROS_VERSION)) self.assertEqual('R60-9592.50.0', CheckCommitFormat('60.9592.50.0')) invalid = 'bad_sha1' self.assertIsNone(CheckCommitFormat(invalid)) def testObtainBisectBoundaryScoreImpl(self): """Tests ObtainBisectBoundaryScoreImpl().""" git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval') build_deploy_eval_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) self.assertEqual([ mock.call(customize_build_deploy=None, eval_label=None), mock.call(customize_build_deploy=None, eval_label=None) ], build_deploy_eval_mock.call_args_list) def testObtainBisectBoundaryScoreImplCrosVersion(self): """Tests ObtainBisectBoundaryScoreImpl() with CrOS version.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() being run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') evaluate_mock.return_value = self.GOOD_COMMIT_SCORE self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) evaluate_mock.assert_called_with(self.DUT, 'cros_%s' % self.GOOD_CROS_VERSION, self.REPEAT) evaluate_mock.return_value = self.BAD_COMMIT_SCORE self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) evaluate_mock.assert_called_with(self.DUT, 'cros_%s' % self.BAD_CROS_VERSION, self.REPEAT) def testObtainBisectBoundaryScoreImplCrosVersionFlashError(self): """Tests ObtainBisectBoundaryScoreImpl() with CrOS version.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() being run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') flash_cros_image_mock.side_effect = flash.FlashError('Flash failed.') with self.assertRaises(flash.FlashError): self.bisector.ObtainBisectBoundaryScoreImpl(True) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) evaluate_mock.assert_not_called() with self.assertRaises(flash.FlashError): self.bisector.ObtainBisectBoundaryScoreImpl(False) flash_cros_image_mock.assert_called_with( self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) evaluate_mock.assert_not_called() def testGetCrosXbuddyPath(self): """Tests GetCrosXbuddyPath().""" self.assertEqual( 'xbuddy://remote/%s/%s/test' % (self.BOARD, self.GOOD_CROS_VERSION), self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)) def testExchangeChromeSanityCheck(self): """Tests the flow of exchanging Chrome between good and bad CrOS.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() has been run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 # Inject commit_info and threshold as if # bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser() # has been run. self.SetDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # Try bad Chrome first. git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') expected_evaluate_calls = [ mock.call(self.DUT, x, self.REPEAT) for x in [ 'cros_%s_cr_%s' % (self.GOOD_CROS_VERSION, self.BAD_COMMIT_SHA1), 'cros_%s_cr_%s' % (self.BAD_CROS_VERSION, self.GOOD_COMMIT_SHA1) ] ] # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage') expected_flash_cros_calls = [ mock.call(self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)), mock.call(self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION)) ] # Make sure bisector.BuildDeploy() is also called. build_deploy_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy') # Assume culprit commit is in Chrome side, i.e. first score is bad. evaluate_mock.side_effect = [ self.BAD_COMMIT_SCORE, self.GOOD_COMMIT_SCORE ] self.assertTrue(self.bisector.ExchangeChromeSanityCheck()) flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls) evaluate_mock.assert_has_calls(expected_evaluate_calls) self.assertEqual(2, build_deploy_mock.call_count) flash_cros_image_mock.reset_mock() evaluate_mock.reset_mock() build_deploy_mock.reset_mock() # Assume culprit commit is not in Chrome side, i.e. first score is good. evaluate_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] self.assertFalse(self.bisector.ExchangeChromeSanityCheck()) flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls) evaluate_mock.assert_has_calls(expected_evaluate_calls) self.assertEqual(2, build_deploy_mock.call_count) def testExchangeChromeSanityCheckFlashError(self): """Tests the flow of exchanging Chrome between good and bad CrOS.""" self.SetUpBisectorWithCrosVersion() # Inject good_commit and bad_commit as if # bisector.ResolveChromeBisectRangeFromCrosVersion() has been run. self.bisector.good_commit = self.GOOD_COMMIT_SHA1 self.bisector.bad_commit = self.BAD_COMMIT_SHA1 # Inject commit_info and threshold as if # bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser() # has been run. self.SetDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # Try bad Chrome first. git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'UpdateCurrentCommit') evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate') # Mock FlashCrosImage() to verify that customize_build_deploy is assigned # as expected. flash_cros_image_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage', side_effect=flash.FlashError('Flash failed.')) build_deploy_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy') with self.assertRaises(flash.FlashError): self.bisector.ExchangeChromeSanityCheck() evaluate_mock.assert_not_called() flash_cros_image_mock.assert_called() build_deploy_mock.assert_not_called() def testFlashImage(self): """Tests FlashImage().""" flash_mock = self.PatchObject(flash, 'Flash') xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testFlashImageRetry(self): """Tests FlashImage() with retry success.""" flash_mock_call_counter = itertools.count() def flash_mock_return(*unused_args, **unused_kwargs): nth_call = next(flash_mock_call_counter) if nth_call < 3: raise flash.FlashError('Flash failed.') flash_mock = self.PatchObject(flash, 'Flash') flash_mock.side_effect = flash_mock_return xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testFlashImageRetryFailed(self): """Tests FlashImage() with retry failed.""" flash_mock = self.PatchObject(flash, 'Flash') flash_mock.side_effect = flash.FlashError('Flash failed.') xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION) with self.assertRaises(flash.FlashError): self.bisector.FlashCrosImage(xbuddy_path) flash_mock.assert_called_with(self.DUT, xbuddy_path, board=self.BOARD, clobber_stateful=True, disable_rootfs_verification=True) def testCrosVersionToChromeCommit(self): """Tests CrosVersionToChromeCommit().""" metadata_url = ( 'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' % (self.BOARD, self.GOOD_CROS_VERSION)) gs_mock = self.StartPatcher(gs_unittest.GSContextMock()) gs_mock.AddCmdResult(['cat', metadata_url], output=self.GOOD_METADATA_CONTENT) git_log_content = '\n'.join([ '8967dd66ad72 (tag: 60.0.3112.53) Publish DEPS for Chromium ' '60.0.3112.53', '27ed0cc0c2f4 Incrementing VERSION to 60.0.3112.53' ]) git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( ['log', '--oneline', '-n', '2', '60.0.3112.53'], output=git_log_content) self.bisector.gs_ctx = gs.GSContext() self.assertEqual( '27ed0cc0c2f4', self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) def testCrosVersionToChromeCommitFail(self): """Tests failure case of CrosVersionToChromeCommit().""" metadata_url = ( 'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' % (self.BOARD, self.GOOD_CROS_VERSION)) gs_mock = self.StartPatcher(gs_unittest.GSContextMock()) gs_mock.AddCmdResult(['cat', metadata_url], returncode=1) self.bisector.gs_ctx = gs.GSContext() self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) metadata_content = 'not_a_json' gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) metadata_content = '\n'.join([ '{', ' "metadata-version": "2",', ' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",', ' "suite_scheduling": true,', ' "build_id": 1644146,', ' "version": {}', '}' ]) gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) gs_mock.AddCmdResult(['cat', metadata_url], output=self.GOOD_METADATA_CONTENT) git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( ['log', '--oneline', '-n', '2', '60.0.3112.53'], returncode=128) self.assertIsNone( self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION)) def testResolveChromeBisectRangeFromCrosVersion(self): """Tests ResolveChromeBisectRangeFromCrosVersion().""" self.SetUpBisectorWithCrosVersion() cros_to_chrome_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'CrosVersionToChromeCommit') cros_to_chrome_mock.side_effect = [ self.GOOD_COMMIT_SHA1, self.BAD_COMMIT_SHA1 ] self.assertTrue( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) self.assertTrue(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertTrue(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) cros_to_chrome_mock.assert_has_calls([ mock.call(self.GOOD_CROS_VERSION), mock.call(self.BAD_CROS_VERSION) ]) cros_to_chrome_mock.reset_mock() cros_to_chrome_mock.side_effect = [None] self.assertFalse( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) cros_to_chrome_mock.assert_called_with(self.GOOD_CROS_VERSION) cros_to_chrome_mock.reset_mock() cros_to_chrome_mock.side_effect = [self.GOOD_COMMIT_SHA1, None] self.assertFalse( self.bisector.ResolveChromeBisectRangeFromCrosVersion()) cros_to_chrome_mock.assert_has_calls([ mock.call(self.GOOD_CROS_VERSION), mock.call(self.BAD_CROS_VERSION) ]) def testPrepareBisect(self): """Tests PrepareBisect().""" # Pass SanityCheck(). git_mock = self.StartPatcher( git_bisector_unittest.GitMock(self.repo_dir)) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) # Inject score for both side. git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval') build_deploy_eval_mock.side_effect = [ self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE ] # Set auto_threshold. self.bisector.auto_threshold = True self.assertTrue(self.bisector.PrepareBisect()) def testPrepareBisectCrosVersion(self): """Tests PrepareBisect() with CrOS version.""" self.SetUpBisectorWithCrosVersion() self.StartPatcher(gs_unittest.GSContextMock()) self.PatchObject(builder_module.Builder, 'SyncToHead') self.PatchObject( chrome_on_cros_bisector.ChromeOnCrosBisector, 'ResolveChromeBisectRangeFromCrosVersion').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'SanityCheck').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'ObtainBisectBoundaryScore').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'GetThresholdFromUser').return_value = True self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector, 'ExchangeChromeSanityCheck').return_value = True self.assertTrue(self.bisector.PrepareBisect())
def testScoreUpdateWrongType(self): """Tests that Update() can handles wrong input type by resetting itself.""" score = common.Score([1, 2, 3]) score.Update(['a', 'b']) self.assertTrue(self.IsEmpty(score))
def testScoreUpdateEmpty(self): """Tests that Update() can handle empty input.""" score = common.Score([1, 2, 3]) score.Update([]) self.assertTrue(self.IsEmpty(score))
def testScoreUpdateNotAList(self): """Tests that Update() can handle wrong input type by resetting itself.""" score = common.Score([1, 2, 3]) score.Update(5) self.assertTrue(self.IsEmpty(score))
class TestGitBisector(cros_test_lib.MockTempDirTestCase): """Tests GitBisector class.""" BOARD = 'samus' TEST_NAME = 'graphics_WebGLAquarium' METRIC = 'avg_fps_1000_fishes/summary/value' REPORT_FILE = 'reports.json' DUT_ADDR = '192.168.1.1' DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR) # Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via # copy.deepcopy() as their users are likely to change the content. GOOD_COMMIT_SHA1 = '44af5c9a5505' GOOD_COMMIT_TIMESTAMP = 1486526594 GOOD_COMMIT_SCORE = common.Score([100]) GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1, timestamp=GOOD_COMMIT_TIMESTAMP, title='good', label='last-known-good ', score=GOOD_COMMIT_SCORE) BAD_COMMIT_SHA1 = '6a163bb66c3e' BAD_COMMIT_TIMESTAMP = 1486530021 BAD_COMMIT_SCORE = common.Score([80]) BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1, timestamp=BAD_COMMIT_TIMESTAMP, title='bad', label='last-known-bad ', score=BAD_COMMIT_SCORE) CULPRIT_COMMIT_SHA1 = '12345abcde' CULPRIT_COMMIT_TIMESTAMP = 1486530000 CULPRIT_COMMIT_SCORE = common.Score([81]) CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1, timestamp=CULPRIT_COMMIT_TIMESTAMP, title='bad', score=CULPRIT_COMMIT_SCORE) THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side. THRESHOLD = 5 # Distance between good score and splitter. REPEAT = 3 def setUp(self): """Sets up test case.""" self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir, board=self.BOARD, reuse_repo=True, good=self.GOOD_COMMIT_SHA1, bad=self.BAD_COMMIT_SHA1, remote=self.DUT, eval_repeat=self.REPEAT, auto_threshold=False, reuse_eval=False, eval_raise_on_error=False, skip_failed_commit=False) self.evaluator = evaluator_module.Evaluator(self.options) self.builder = builder_module.Builder(self.options) self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) self.repo_dir = os.path.join(self.tempdir, builder_module.Builder.DEFAULT_REPO_DIR) def setDefaultCommitInfo(self): """Sets up default commit info.""" self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) def testInit(self): """Tests GitBisector() to expect default data members.""" self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit) self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit) self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw) self.assertEqual(self.REPEAT, self.bisector.eval_repeat) self.assertEqual(self.builder, self.bisector.builder) self.assertEqual(self.repo_dir, self.bisector.repo_dir) self.assertIsNone(self.bisector.good_commit_info) self.assertIsNone(self.bisector.bad_commit_info) self.assertEqual(0, len(self.bisector.bisect_log)) self.assertIsNone(self.bisector.threshold) self.assertTrue(not self.bisector.current_commit) def testInitMissingRequiredArgs(self): """Tests that GitBisector raises error when missing required arguments.""" options = cros_test_lib.EasyAttr() with self.assertRaises(common.MissingRequiredOptionsException) as cm: git_bisector.GitBisector(options, self.builder, self.evaluator) exception_message = str(cm.exception) self.assertIn('Missing command line', exception_message) self.assertIn('GitBisector', exception_message) for arg in git_bisector.GitBisector.REQUIRED_ARGS: self.assertIn(arg, exception_message) def testCheckCommitFormat(self): """Tests CheckCommitFormat().""" sha1 = '900d900d' self.assertEqual(sha1, git_bisector.GitBisector.CheckCommitFormat(sha1)) not_sha1 = 'bad_sha1' self.assertIsNone(git_bisector.GitBisector.CheckCommitFormat(not_sha1)) def testSetUp(self): """Tests that SetUp() calls builder.SetUp().""" with mock.patch.object(builder_module.Builder, 'SetUp') as builder_mock: self.bisector.SetUp() builder_mock.assert_called_with() def testGit(self): """Tests that Git() invokes git.RunGit() as expected.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(['ls']) self.bisector.Git(['ls']) def testUpdateCurrentCommit(self): """Tests that UpdateCurrentCommit() updates self.current_commit.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult( partial_mock.In('show'), output='4fcbdaf6 1493010050 Add "cros bisect" command.\n') self.bisector.UpdateCurrentCommit() current_commit = self.bisector.current_commit self.assertEqual('4fcbdaf6', current_commit.sha1) self.assertEqual(1493010050, current_commit.timestamp) self.assertEqual('Add "cros bisect" command.', current_commit.title) def testUpdateCurrentCommitFail(self): """Tests UpdateCurrentCommit() when 'git show' returns nonzero.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(partial_mock.In('show'), returncode=128) self.bisector.UpdateCurrentCommit() self.assertTrue(not self.bisector.current_commit) def testUpdateCurrentCommitFailUnexpectedOutput(self): """Tests UpdateCurrentCommit() when 'git show' gives unexpected output.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(partial_mock.In('show'), output='Not expected') self.bisector.UpdateCurrentCommit() self.assertTrue(not self.bisector.current_commit) def MockOutBuildDeployEvaluateForSanityCheck(self): """Mocks out BuildDeployEvaluate behavior for SaintyCheck test. It mocks UpdateCurrentCommit() to emit good and bad commits. And mocks CheckLastEvaluate() to return good and bad commit score. """ commit_info_list = [ common.CommitInfo(sha1=self.GOOD_COMMIT_SHA1, title='good', timestamp=self.GOOD_COMMIT_TIMESTAMP), common.CommitInfo(sha1=self.BAD_COMMIT_SHA1, title='bad', timestamp=self.BAD_COMMIT_TIMESTAMP) ] # This mock is problematic. The side effect should modify "self" when # the member method is called. def UpdateCurrentCommitSideEffect(): self.bisector.current_commit = commit_info_list.pop(0) self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit', side_effect=UpdateCurrentCommitSideEffect) self.PatchObject( evaluator_module.Evaluator, 'CheckLastEvaluate', side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE]) def testGetCommitTimestamp(self): """Tests GetCommitTimeStamp by mocking git.RunGit.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) self.assertEqual( self.GOOD_COMMIT_TIMESTAMP, self.bisector.GetCommitTimestamp(self.GOOD_COMMIT_SHA1)) def testGetCommitTimestampNotFound(self): """Tests GetCommitTimeStamp when the commit is not found.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) nonexist_sha1 = '12345' # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder(['show', nonexist_sha1]), output='commit does not exist') self.assertIsNone(self.bisector.GetCommitTimestamp(nonexist_sha1)) def testSanityCheck(self): """Tests SanityCheck(). It tests by mocking out git commands called by SanityCheck(). """ git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for DoesCommitExistInRepo. git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) self.assertTrue(self.bisector.SanityCheck()) def testSanityCheckGoodCommitNotExist(self): """Tests SanityCheck() that good commit does not exist.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) sync_to_head_mock = self.PatchObject(builder_module.Builder, 'SyncToHead') # Mock git result for DoesCommitExistInRepo to return False. git_mock.AddRunGitResult(partial_mock.InOrder( ['rev-list', self.GOOD_COMMIT_SHA1]), returncode=128) # Mock invalid result for GetCommitTimestamp to return None. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output='invalid commit') git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output='invalid commit') self.assertFalse(self.bisector.SanityCheck()) # SyncToHead() called because DoesCommitExistInRepo() returns False for # good commit. sync_to_head_mock.assert_called() def testSanityCheckSyncToHeadWorks(self): """Tests SanityCheck() that good and bad commit do not exist. As good and bad commit do not exist, it calls SyncToHead(). """ git_mock = self.StartPatcher(GitMock(self.repo_dir)) sync_to_head_mock = self.PatchObject(builder_module.Builder, 'SyncToHead') # Mock git result for DoesCommitExistInRepo to return False. git_mock.AddRunGitResult(partial_mock.InOrder( ['rev-list', self.GOOD_COMMIT_SHA1]), returncode=128) # Mock git result for GetCommitTimestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) self.assertTrue(self.bisector.SanityCheck()) # After SyncToHead, found both bad and good commit. sync_to_head_mock.assert_called() def testSanityCheckWrongTimeOrder(self): """Tests SanityCheck() that good and bad commit have wrong time order.""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) # Mock git result for DoesCommitExistInRepo. git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1])) git_mock.AddRunGitResult( partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1])) # Mock git result for GetCommitTimestamp, but swap timestamp. git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.GOOD_COMMIT_SHA1]), output=str(self.BAD_COMMIT_TIMESTAMP)) git_mock.AddRunGitResult(partial_mock.InOrder( ['show', self.BAD_COMMIT_SHA1]), output=str(self.GOOD_COMMIT_TIMESTAMP)) self.assertFalse(self.bisector.SanityCheck()) def testObtainBisectBoundaryScoreImpl(self): """Tests ObtainBisectBoundaryScoreImpl().""" git_mock = self.StartPatcher(GitMock(self.repo_dir)) git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1]) git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1]) build_deploy_eval_mock = self.PatchObject( git_bisector.GitBisector, 'BuildDeployEval', side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE]) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(True)) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.ObtainBisectBoundaryScoreImpl(False)) self.assertEqual([ mock.call(self.repo_dir, ['checkout', self.GOOD_COMMIT_SHA1], error_code_ok=True), mock.call(self.repo_dir, ['checkout', self.BAD_COMMIT_SHA1], error_code_ok=True) ], git_mock.call_args_list) build_deploy_eval_mock.assert_called() def testObtainBisectBoundaryScore(self): """Tests ObtainBisectBoundaryScore(). Normal case.""" def MockedObtainBisectBoundaryScoreImpl(good_side): if good_side: self.bisector.current_commit = copy.deepcopy( self.GOOD_COMMIT_INFO) else: self.bisector.current_commit = copy.deepcopy( self.BAD_COMMIT_INFO) return self.bisector.current_commit.score obtain_score_mock = self.PatchObject( git_bisector.GitBisector, 'ObtainBisectBoundaryScoreImpl', side_effect=MockedObtainBisectBoundaryScoreImpl) self.assertTrue(self.bisector.ObtainBisectBoundaryScore()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.good_commit_info.score) self.assertEqual('last-known-good ', self.bisector.good_commit_info.label) self.assertEqual(self.BAD_COMMIT_SCORE, self.bisector.bad_commit_info.score) self.assertEqual('last-known-bad ', self.bisector.bad_commit_info.label) obtain_score_mock.assert_called() def testObtainBisectBoundaryScoreBadScoreUnavailable(self): """Tests ObtainBisectBoundaryScore(). Bad score unavailable.""" def UpdateCurrentCommitSideEffect(good_side): if good_side: self.bisector.current_commit = copy.deepcopy( self.GOOD_COMMIT_INFO) else: self.bisector.current_commit = copy.deepcopy( self.BAD_COMMIT_INFO) self.bisector.current_commit.score = None return self.bisector.current_commit.score obtain_score_mock = self.PatchObject( git_bisector.GitBisector, 'ObtainBisectBoundaryScoreImpl', side_effect=UpdateCurrentCommitSideEffect) self.assertFalse(self.bisector.ObtainBisectBoundaryScore()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.good_commit_info.score) self.assertEqual('last-known-good ', self.bisector.good_commit_info.label) self.assertIsNone(self.bisector.bad_commit_info.score) self.assertEqual('last-known-bad ', self.bisector.bad_commit_info.label) obtain_score_mock.assert_called() def testGetThresholdFromUser(self): """Tests GetThresholdFromUser().""" logging.notice('testGetThresholdFromUser') self.setDefaultCommitInfo() input_mock = self.PatchObject(cros_build_lib, 'GetInput', return_value=self.THRESHOLD_SPLITTER) self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(self.THRESHOLD, self.bisector.threshold) input_mock.assert_called() def testGetThresholdFromUserAutoPick(self): """Tests GetThresholdFromUser().""" self.setDefaultCommitInfo() self.bisector.auto_threshold = True self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(10, self.bisector.threshold) def testGetThresholdFromUserOutOfBoundFail(self): """Tests GetThresholdFromUser() with out-of-bound input.""" self.setDefaultCommitInfo() input_mock = self.PatchObject(cros_build_lib, 'GetInput', side_effect=['0', '1000', '-10']) self.assertFalse(self.bisector.GetThresholdFromUser()) self.assertIsNone(self.bisector.threshold) self.assertEqual(3, input_mock.call_count) def testGetThresholdFromUserRetrySuccess(self): """Tests GetThresholdFromUser() with retry.""" self.setDefaultCommitInfo() input_mock = self.PatchObject( cros_build_lib, 'GetInput', side_effect=['not_a_number', '1000', self.THRESHOLD_SPLITTER]) self.assertTrue(self.bisector.GetThresholdFromUser()) self.assertEqual(self.THRESHOLD, self.bisector.threshold) self.assertEqual(3, input_mock.call_count) def testBuildDeploy(self): """Tests BuildDeploy().""" # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) build_to_deploy = '/build/to/deploy' build_mock = self.PatchObject(builder_module.Builder, 'Build', return_value=build_to_deploy) deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy') self.assertTrue(self.bisector.BuildDeploy()) build_label = self.GOOD_COMMIT_INFO.sha1 build_mock.assert_called_with(build_label) deploy_mock.assert_called_with(self.DUT, build_to_deploy, build_label) def testBuildDeployBuildFail(self): """Tests BuildDeploy() with Build() failure.""" # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) # Build() failed. build_mock = self.PatchObject(builder_module.Builder, 'Build', return_value=None) deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy') self.assertFalse(self.bisector.BuildDeploy()) build_mock.assert_called() deploy_mock.assert_not_called() def PatchObjectForBuildDeployEval(self): """Returns a dict of patch objects. The patch objects are to mock: git_bisector.UpdateCurrentCommit() evaluator.CheckLastEvaluate() git_bisector.BuildDeploy() evaluator.Evaluate() """ return { 'UpdateCurrentCommit': self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit'), 'CheckLastEvaluate': self.PatchObject(evaluator_module.Evaluator, 'CheckLastEvaluate'), 'BuildDeploy': self.PatchObject(git_bisector.GitBisector, 'BuildDeploy', return_value=True), 'Evaluate': self.PatchObject(evaluator_module.Evaluator, 'Evaluate') } def testBuildDeployEvalShortcutCheckLastEvaluate(self): """Tests BuildDeployEval() with CheckLastEvaluate() found last score.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = self.GOOD_COMMIT_SCORE score = self.bisector.BuildDeployEval() self.assertEqual(self.GOOD_COMMIT_SCORE, score) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in ['UpdateCurrentCommit', 'CheckLastEvaluate']: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) for method_called in ['BuildDeploy', 'Evaluate']: mocks[method_called].assert_not_called() def AssertBuildDeployEvalMocksAllCalled(self, mocks): for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy', 'Evaluate' ]: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, self.GOOD_COMMIT_SHA1, self.REPEAT) def testBuildDeployEvalNoCheckLastEvaluate(self): """Tests BuildDeployEval() without last score.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval()) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testBuildDeployEvalBuildFail(self): """Tests BuildDeployEval() with BuildDeploy failure.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['BuildDeploy'].return_value = False score = self.bisector.BuildDeployEval() self.assertFalse(score) self.assertFalse(self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy' ]: mocks[method_called].assert_called() mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1, self.REPEAT) mocks['Evaluate'].assert_not_called() def testBuildDeployEvalNoCheckLastEvaluateSpecifyEvalLabel(self): """Tests BuildDeployEval() with eval_label specified.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE eval_label = 'customized_label' self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval(eval_label=eval_label)) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy', 'Evaluate' ]: mocks[method_called].assert_called() # Use given label instead of SHA1 as eval label. mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT) @staticmethod def _DummyMethod(): """A dummy method for test to call and mock.""" def testBuildDeployEvalNoCheckLastEvaluateSpecifyBuildDeploy(self): """Tests BuildDeployEval() with customize_build_deploy specified.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE dummy_method = self.PatchObject(TestGitBisector, '_DummyMethod', return_value=True) eval_label = 'customized_label' self.assertEqual( self.GOOD_COMMIT_SCORE, self.bisector.BuildDeployEval( eval_label=eval_label, customize_build_deploy=TestGitBisector._DummyMethod)) self.assertEqual(self.GOOD_COMMIT_SCORE, self.bisector.current_commit.score) for method_called in [ 'UpdateCurrentCommit', 'CheckLastEvaluate', 'Evaluate' ]: mocks[method_called].assert_called() mocks['BuildDeploy'].assert_not_called() dummy_method.assert_called() # Use given label instead of SHA1 as eval label. mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT) mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT) def testBuildDeployEvalRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error set.""" self.options.eval_raise_on_error = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertRaises(Exception, self.bisector.BuildDeployEval) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testBuildDeployEvalSuppressRaiseNoScore(self): """Tests BuildDeployEval() without score with eval_raise_on_error unset.""" mocks = self.PatchObjectForBuildDeployEval() # Inject this as UpdateCurrentCommit's side effect. self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO) mocks['CheckLastEvaluate'].return_value = common.Score() mocks['Evaluate'].return_value = common.Score() self.assertFalse(self.bisector.BuildDeployEval()) self.assertFalse(self.bisector.current_commit.score) self.AssertBuildDeployEvalMocksAllCalled(mocks) def testLabelBuild(self): """Tests LabelBuild().""" # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([70]))) # Better than bad score, but not good enough. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([85]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([95]))) # Better than the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([98]))) # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([110]))) # No score, default bad. self.assertEqual(bad, self.bisector.LabelBuild(None)) self.assertEqual(bad, self.bisector.LabelBuild(common.Score())) def testLabelBuildSkipNoScore(self): """Tests LabelBuild().""" self.options.skip_failed_commit = True self.bisector = git_bisector.GitBisector(self.options, self.builder, self.evaluator) # Inject good(100), bad(80) score and threshold. self.setDefaultCommitInfo() self.bisector.threshold = self.THRESHOLD # No score, skip. self.assertEqual('skip', self.bisector.LabelBuild(None)) self.assertEqual('skip', self.bisector.LabelBuild(common.Score())) def testLabelBuildLowerIsBetter(self): """Tests LabelBuild() in lower-is-better condition.""" # Reverse good(80) and bad(100) score (lower is better), same threshold. self.bisector.good_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) self.bisector.bad_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) self.bisector.threshold = self.THRESHOLD good = 'good' bad = 'bad' # Better than given good score. self.assertEqual(good, self.bisector.LabelBuild(common.Score([70]))) # Worse than good score, but still better than margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([80]))) self.assertEqual(good, self.bisector.LabelBuild(common.Score([82]))) # On the margin. self.assertEqual(good, self.bisector.LabelBuild(common.Score([85]))) # Worse than the margin. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([88]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90]))) self.assertEqual(bad, self.bisector.LabelBuild(common.Score([95]))) # Worse than given bad score. self.assertEqual(bad, self.bisector.LabelBuild(common.Score([110]))) def testGitBisect(self): """Tests GitBisect().""" git_mock = self.PatchObject(git_bisector.GitBisector, 'Git', return_value=cros_build_lib.CommandResult( cmd=['git', 'bisect', 'reset'], output='We are not bisecting.', returncode=0)) result, done = self.bisector.GitBisect(['reset']) git_mock.assert_called_with(['bisect', 'reset']) self.assertFalse(done) self.assertEqual('We are not bisecting.', result.output) self.assertListEqual(['git', 'bisect', 'reset'], result.cmd) def testGitBisectDone(self): """Tests GitBisect() when culprit is found.""" git_mock = self.PatchObject( git_bisector.GitBisector, 'Git', return_value=cros_build_lib.CommandResult( cmd=['git', 'bisect', 'bad'], output='abcedf is the first bad commit\ncommit abcdef', returncode=0)) result, done = self.bisector.GitBisect(['bad']) git_mock.assert_called_with(['bisect', 'bad']) self.assertTrue(done) self.assertListEqual(['git', 'bisect', 'bad'], result.cmd) def testRun(self): """Tests Run().""" bisector_mock = self.StartPatcher(GitBisectorMock()) bisector_mock.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO) bisector_mock.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO) bisector_mock.threshold = self.THRESHOLD bisector_mock.git_bisect_args_result = [ (['reset'], (None, False)), (['start'], (None, False)), (['bad', self.BAD_COMMIT_SHA1], (None, False)), (['good', self.GOOD_COMMIT_SHA1], (None, False)), (['bad'], (cros_build_lib.CommandResult( cmd=['git', 'bisect', 'bad'], output='%s is the first bad commit\ncommit %s' % (self.CULPRIT_COMMIT_SHA1, self.CULPRIT_COMMIT_SHA1), returncode=0), True)), # bisect bad (assume it found the first bad commit). (['log'], (None, False)), (['reset'], (None, False)) ] bisector_mock.build_deploy_eval_current_commit = [ self.CULPRIT_COMMIT_INFO ] bisector_mock.build_deploy_eval_result = [self.CULPRIT_COMMIT_SCORE] bisector_mock.label_build_result = ['bad'] run_result = self.bisector.Run() self.assertTrue(bisector_mock.patched['PrepareBisect'].called) self.assertEqual(7, bisector_mock.patched['GitBisect'].call_count) self.assertTrue(bisector_mock.patched['BuildDeployEval'].called) self.assertTrue(bisector_mock.patched['LabelBuild'].called) self.assertEqual(self.CULPRIT_COMMIT_SHA1, run_result)
def testScoreInitWrongType(self): """Tests that Init() can handles wrong input type by resetting itself.""" self.assertTrue(self.IsEmpty(common.Score(['a', 'b']))) self.assertTrue(self.IsEmpty(common.Score([]))) self.assertTrue(self.IsEmpty(common.Score(1)))
def testEmpty(self): """Tests that default Score object is empty.""" score = common.Score() self.assertTrue(self.IsEmpty(score))
def CheckLastEvaluate(self, build_label, repeat=1): return common.Score()