def MockOutBuildDeployEvaluateForSanityCheck(self):
"""Mocks out BuildDeployEvaluate behavior for SaintyCheck test.
It mocks UpdateCurrentCommit() to emit good and bad commits. And mocks
CheckLastEvaluate() to return good and bad commit score.
"""
commit_info_list = [
common.CommitInfo(sha1=self.GOOD_COMMIT_SHA1,
title='good',
timestamp=self.GOOD_COMMIT_TIMESTAMP),
common.CommitInfo(sha1=self.BAD_COMMIT_SHA1,
title='bad',
timestamp=self.BAD_COMMIT_TIMESTAMP)
]
# This mock is problematic. The side effect should modify "self" when
# the member method is called.
def UpdateCurrentCommitSideEffect():
self.bisector.current_commit = commit_info_list.pop(0)
self.PatchObject(git_bisector.GitBisector,
'UpdateCurrentCommit',
side_effect=UpdateCurrentCommitSideEffect)
self.PatchObject(
evaluator_module.Evaluator,
'CheckLastEvaluate',
side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE])
def testBool(self):
"""Tests CommitInfo's boolean value conversion.
Only default(empty) CommitInfo's boolean value is False.
"""
info1 = common.CommitInfo()
self.assertTrue(not info1)
self.assertFalse(bool(info1))
info2 = common.CommitInfo(title='something')
self.assertTrue(bool(info2))
self.assertFalse(not info2)
def testNotEqual(self):
"""Tests inequality of two CommitInfo objects."""
info1 = common.CommitInfo(sha1='abcdef',
title='test',
score=common.Score(values=[1, 2, 3]),
label='GOOD',
timestamp=100)
info2 = common.CommitInfo(sha1='abcdef',
title='test',
score=common.Score(values=[1, 2]),
label='GOOD',
timestamp=100)
self.assertNotEqual(info1, info2)
self.assertFalse(info1 == info2)
def testEqual(self):
"""Tests equality of two CommitInfo objects."""
info1 = common.CommitInfo(sha1='abcdef',
title='test',
score=common.Score(values=[1, 2, 3]),
label='GOOD',
timestamp=100)
info2 = common.CommitInfo(sha1='abcdef',
title='test',
score=common.Score(values=[1, 2, 3]),
label='GOOD',
timestamp=100)
self.assertEqual(info1, info2)
# In Python 2.7, __ne__() doesn't delegates to "not __eq__()" so that the
# sanity check is necessary.
self.assertFalse(info1 != info2)
def testAssigned(self):
"""Tests that CommitInfo constrcutor sets up data members correctly."""
info = common.CommitInfo(sha1='abcdef',
title='test',
score=common.Score(values=[1]),
label='GOOD',
timestamp=100)
self.assertEqual(
"CommitInfo(sha1='abcdef', title='test', score=Score(values=[1.0]), "
"label='GOOD', timestamp=100)", repr(info))
def __init__(self, options, builder, evaluator):
"""Constructor.
Args:
options: An argparse.Namespace to hold command line arguments. Should
contain:
* good: Last known good commit.
* bad: Last known bad commit.
* remote: DUT (refer lib.commandline.Device).
* eval_repeat: Run test for N times. None means 1.
* auto_threshold: True to set threshold automatically without prompt.
* board: board name of the DUT.
* eval_raise_on_error: If set, raises if evaluation failed.
* skip_failed_commit: If set, mark the failed commit (build failed /
no result) as 'skip' instead of 'bad'.
builder: Builder to build/deploy image. Should contain repo_dir.
evaluator: Evaluator to get score
"""
super(GitBisector, self).__init__(options)
self.good_commit = options.good
self.bad_commit = options.bad
self.remote = options.remote
self.eval_repeat = options.eval_repeat
self.builder = builder
self.evaluator = evaluator
self.repo_dir = self.builder.repo_dir
self.auto_threshold = options.auto_threshold
self.board = options.board
self.eval_raise_on_error = options.eval_raise_on_error
self.skip_failed_commit = options.skip_failed_commit
# Initialized in ObtainBisectBoundaryScore().
self.good_commit_info = None
self.bad_commit_info = None
# Record bisect log (list of CommitInfo).
self.bisect_log = []
# If the distance between current commit's score and given good commit is
# shorter than threshold, treat the commit as good one. Its default value
# comes from evaluator's THRESHOLD constant. If it is None (undefined),
# the bisector will prompt user to input threshold after obtaining both
# sides' score.
self.threshold = evaluator.THRESHOLD
# Set auto_threshold to True if the threshold is preset.
if self.threshold is not None:
self.auto_threshold = True
# Init with empty CommitInfo. Will update in UpdateCurrentCommit().
self.current_commit = common.CommitInfo()
def UpdateCurrentCommit(self):
"""Updates self.current_commit."""
result = self.Git(['show', '--no-patch', '--format=%h %ct %s'])
if result.returncode != 0:
logging.error('Failed to update current commit info.')
return
fields = result.output.strip().split()
if len(fields) < 3:
logging.error('Failed to update current commit info.')
return
sha1 = fields[0]
timestamp = int(fields[1])
title = ' '.join(fields[2:])
self.current_commit = common.CommitInfo(sha1=sha1, title=title,
timestamp=timestamp)
class TestGitBisector(cros_test_lib.MockTempDirTestCase):
"""Tests GitBisector class."""
BOARD = 'samus'
TEST_NAME = 'graphics_WebGLAquarium'
METRIC = 'avg_fps_1000_fishes/summary/value'
REPORT_FILE = 'reports.json'
DUT_ADDR = '192.168.1.1'
DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR)
# Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via
# copy.deepcopy() as their users are likely to change the content.
GOOD_COMMIT_SHA1 = '44af5c9a5505'
GOOD_COMMIT_TIMESTAMP = 1486526594
GOOD_COMMIT_SCORE = common.Score([100])
GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1,
timestamp=GOOD_COMMIT_TIMESTAMP,
title='good',
label='last-known-good ',
score=GOOD_COMMIT_SCORE)
BAD_COMMIT_SHA1 = '6a163bb66c3e'
BAD_COMMIT_TIMESTAMP = 1486530021
BAD_COMMIT_SCORE = common.Score([80])
BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1,
timestamp=BAD_COMMIT_TIMESTAMP,
title='bad',
label='last-known-bad ',
score=BAD_COMMIT_SCORE)
CULPRIT_COMMIT_SHA1 = '12345abcde'
CULPRIT_COMMIT_TIMESTAMP = 1486530000
CULPRIT_COMMIT_SCORE = common.Score([81])
CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1,
timestamp=CULPRIT_COMMIT_TIMESTAMP,
title='bad',
score=CULPRIT_COMMIT_SCORE)
THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side.
THRESHOLD = 5 # Distance between good score and splitter.
REPEAT = 3
def setUp(self):
"""Sets up test case."""
self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir,
board=self.BOARD,
reuse_repo=True,
good=self.GOOD_COMMIT_SHA1,
bad=self.BAD_COMMIT_SHA1,
remote=self.DUT,
eval_repeat=self.REPEAT,
auto_threshold=False,
reuse_eval=False,
eval_raise_on_error=False,
skip_failed_commit=False)
self.evaluator = evaluator_module.Evaluator(self.options)
self.builder = builder_module.Builder(self.options)
self.bisector = git_bisector.GitBisector(self.options, self.builder,
self.evaluator)
self.repo_dir = os.path.join(self.tempdir,
builder_module.Builder.DEFAULT_REPO_DIR)
def setDefaultCommitInfo(self):
"""Sets up default commit info."""
self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO)
self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO)
def testInit(self):
"""Tests GitBisector() to expect default data members."""
self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit)
self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit)
self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw)
self.assertEqual(self.REPEAT, self.bisector.eval_repeat)
self.assertEqual(self.builder, self.bisector.builder)
self.assertEqual(self.repo_dir, self.bisector.repo_dir)
self.assertIsNone(self.bisector.good_commit_info)
self.assertIsNone(self.bisector.bad_commit_info)
self.assertEqual(0, len(self.bisector.bisect_log))
self.assertIsNone(self.bisector.threshold)
self.assertTrue(not self.bisector.current_commit)
def testInitMissingRequiredArgs(self):
"""Tests that GitBisector raises error when missing required arguments."""
options = cros_test_lib.EasyAttr()
with self.assertRaises(common.MissingRequiredOptionsException) as cm:
git_bisector.GitBisector(options, self.builder, self.evaluator)
exception_message = str(cm.exception)
self.assertIn('Missing command line', exception_message)
self.assertIn('GitBisector', exception_message)
for arg in git_bisector.GitBisector.REQUIRED_ARGS:
self.assertIn(arg, exception_message)
def testCheckCommitFormat(self):
"""Tests CheckCommitFormat()."""
sha1 = '900d900d'
self.assertEqual(sha1,
git_bisector.GitBisector.CheckCommitFormat(sha1))
not_sha1 = 'bad_sha1'
self.assertIsNone(git_bisector.GitBisector.CheckCommitFormat(not_sha1))
def testSetUp(self):
"""Tests that SetUp() calls builder.SetUp()."""
with mock.patch.object(builder_module.Builder,
'SetUp') as builder_mock:
self.bisector.SetUp()
builder_mock.assert_called_with()
def testGit(self):
"""Tests that Git() invokes git.RunGit() as expected."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
git_mock.AddRunGitResult(['ls'])
self.bisector.Git(['ls'])
def testUpdateCurrentCommit(self):
"""Tests that UpdateCurrentCommit() updates self.current_commit."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
git_mock.AddRunGitResult(
partial_mock.In('show'),
output='4fcbdaf6 1493010050 Add "cros bisect" command.\n')
self.bisector.UpdateCurrentCommit()
current_commit = self.bisector.current_commit
self.assertEqual('4fcbdaf6', current_commit.sha1)
self.assertEqual(1493010050, current_commit.timestamp)
self.assertEqual('Add "cros bisect" command.', current_commit.title)
def testUpdateCurrentCommitFail(self):
"""Tests UpdateCurrentCommit() when 'git show' returns nonzero."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
git_mock.AddRunGitResult(partial_mock.In('show'), returncode=128)
self.bisector.UpdateCurrentCommit()
self.assertTrue(not self.bisector.current_commit)
def testUpdateCurrentCommitFailUnexpectedOutput(self):
"""Tests UpdateCurrentCommit() when 'git show' gives unexpected output."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
git_mock.AddRunGitResult(partial_mock.In('show'),
output='Not expected')
self.bisector.UpdateCurrentCommit()
self.assertTrue(not self.bisector.current_commit)
def MockOutBuildDeployEvaluateForSanityCheck(self):
"""Mocks out BuildDeployEvaluate behavior for SaintyCheck test.
It mocks UpdateCurrentCommit() to emit good and bad commits. And mocks
CheckLastEvaluate() to return good and bad commit score.
"""
commit_info_list = [
common.CommitInfo(sha1=self.GOOD_COMMIT_SHA1,
title='good',
timestamp=self.GOOD_COMMIT_TIMESTAMP),
common.CommitInfo(sha1=self.BAD_COMMIT_SHA1,
title='bad',
timestamp=self.BAD_COMMIT_TIMESTAMP)
]
# This mock is problematic. The side effect should modify "self" when
# the member method is called.
def UpdateCurrentCommitSideEffect():
self.bisector.current_commit = commit_info_list.pop(0)
self.PatchObject(git_bisector.GitBisector,
'UpdateCurrentCommit',
side_effect=UpdateCurrentCommitSideEffect)
self.PatchObject(
evaluator_module.Evaluator,
'CheckLastEvaluate',
side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE])
def testGetCommitTimestamp(self):
"""Tests GetCommitTimeStamp by mocking git.RunGit."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
# Mock git result for GetCommitTimestamp.
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output=str(self.GOOD_COMMIT_TIMESTAMP))
self.assertEqual(
self.GOOD_COMMIT_TIMESTAMP,
self.bisector.GetCommitTimestamp(self.GOOD_COMMIT_SHA1))
def testGetCommitTimestampNotFound(self):
"""Tests GetCommitTimeStamp when the commit is not found."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
nonexist_sha1 = '12345'
# Mock git result for GetCommitTimestamp.
git_mock.AddRunGitResult(partial_mock.InOrder(['show', nonexist_sha1]),
output='commit does not exist')
self.assertIsNone(self.bisector.GetCommitTimestamp(nonexist_sha1))
def testSanityCheck(self):
"""Tests SanityCheck().
It tests by mocking out git commands called by SanityCheck().
"""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
# Mock git result for DoesCommitExistInRepo.
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1]))
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1]))
# Mock git result for GetCommitTimestamp.
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output=str(self.GOOD_COMMIT_TIMESTAMP))
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.BAD_COMMIT_SHA1]),
output=str(self.BAD_COMMIT_TIMESTAMP))
self.assertTrue(self.bisector.SanityCheck())
def testSanityCheckGoodCommitNotExist(self):
"""Tests SanityCheck() that good commit does not exist."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
sync_to_head_mock = self.PatchObject(builder_module.Builder,
'SyncToHead')
# Mock git result for DoesCommitExistInRepo to return False.
git_mock.AddRunGitResult(partial_mock.InOrder(
['rev-list', self.GOOD_COMMIT_SHA1]),
returncode=128)
# Mock invalid result for GetCommitTimestamp to return None.
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output='invalid commit')
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.BAD_COMMIT_SHA1]),
output='invalid commit')
self.assertFalse(self.bisector.SanityCheck())
# SyncToHead() called because DoesCommitExistInRepo() returns False for
# good commit.
sync_to_head_mock.assert_called()
def testSanityCheckSyncToHeadWorks(self):
"""Tests SanityCheck() that good and bad commit do not exist.
As good and bad commit do not exist, it calls SyncToHead().
"""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
sync_to_head_mock = self.PatchObject(builder_module.Builder,
'SyncToHead')
# Mock git result for DoesCommitExistInRepo to return False.
git_mock.AddRunGitResult(partial_mock.InOrder(
['rev-list', self.GOOD_COMMIT_SHA1]),
returncode=128)
# Mock git result for GetCommitTimestamp.
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output=str(self.GOOD_COMMIT_TIMESTAMP))
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.BAD_COMMIT_SHA1]),
output=str(self.BAD_COMMIT_TIMESTAMP))
self.assertTrue(self.bisector.SanityCheck())
# After SyncToHead, found both bad and good commit.
sync_to_head_mock.assert_called()
def testSanityCheckWrongTimeOrder(self):
"""Tests SanityCheck() that good and bad commit have wrong time order."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
# Mock git result for DoesCommitExistInRepo.
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1]))
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1]))
# Mock git result for GetCommitTimestamp, but swap timestamp.
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output=str(self.BAD_COMMIT_TIMESTAMP))
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.BAD_COMMIT_SHA1]),
output=str(self.GOOD_COMMIT_TIMESTAMP))
self.assertFalse(self.bisector.SanityCheck())
def testObtainBisectBoundaryScoreImpl(self):
"""Tests ObtainBisectBoundaryScoreImpl()."""
git_mock = self.StartPatcher(GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
build_deploy_eval_mock = self.PatchObject(
git_bisector.GitBisector,
'BuildDeployEval',
side_effect=[self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE])
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(True))
self.assertEqual(self.BAD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(False))
self.assertEqual([
mock.call(self.repo_dir, ['checkout', self.GOOD_COMMIT_SHA1],
error_code_ok=True),
mock.call(self.repo_dir, ['checkout', self.BAD_COMMIT_SHA1],
error_code_ok=True)
], git_mock.call_args_list)
build_deploy_eval_mock.assert_called()
def testObtainBisectBoundaryScore(self):
"""Tests ObtainBisectBoundaryScore(). Normal case."""
def MockedObtainBisectBoundaryScoreImpl(good_side):
if good_side:
self.bisector.current_commit = copy.deepcopy(
self.GOOD_COMMIT_INFO)
else:
self.bisector.current_commit = copy.deepcopy(
self.BAD_COMMIT_INFO)
return self.bisector.current_commit.score
obtain_score_mock = self.PatchObject(
git_bisector.GitBisector,
'ObtainBisectBoundaryScoreImpl',
side_effect=MockedObtainBisectBoundaryScoreImpl)
self.assertTrue(self.bisector.ObtainBisectBoundaryScore())
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.good_commit_info.score)
self.assertEqual('last-known-good ',
self.bisector.good_commit_info.label)
self.assertEqual(self.BAD_COMMIT_SCORE,
self.bisector.bad_commit_info.score)
self.assertEqual('last-known-bad ',
self.bisector.bad_commit_info.label)
obtain_score_mock.assert_called()
def testObtainBisectBoundaryScoreBadScoreUnavailable(self):
"""Tests ObtainBisectBoundaryScore(). Bad score unavailable."""
def UpdateCurrentCommitSideEffect(good_side):
if good_side:
self.bisector.current_commit = copy.deepcopy(
self.GOOD_COMMIT_INFO)
else:
self.bisector.current_commit = copy.deepcopy(
self.BAD_COMMIT_INFO)
self.bisector.current_commit.score = None
return self.bisector.current_commit.score
obtain_score_mock = self.PatchObject(
git_bisector.GitBisector,
'ObtainBisectBoundaryScoreImpl',
side_effect=UpdateCurrentCommitSideEffect)
self.assertFalse(self.bisector.ObtainBisectBoundaryScore())
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.good_commit_info.score)
self.assertEqual('last-known-good ',
self.bisector.good_commit_info.label)
self.assertIsNone(self.bisector.bad_commit_info.score)
self.assertEqual('last-known-bad ',
self.bisector.bad_commit_info.label)
obtain_score_mock.assert_called()
def testGetThresholdFromUser(self):
"""Tests GetThresholdFromUser()."""
logging.notice('testGetThresholdFromUser')
self.setDefaultCommitInfo()
input_mock = self.PatchObject(cros_build_lib,
'GetInput',
return_value=self.THRESHOLD_SPLITTER)
self.assertTrue(self.bisector.GetThresholdFromUser())
self.assertEqual(self.THRESHOLD, self.bisector.threshold)
input_mock.assert_called()
def testGetThresholdFromUserAutoPick(self):
"""Tests GetThresholdFromUser()."""
self.setDefaultCommitInfo()
self.bisector.auto_threshold = True
self.assertTrue(self.bisector.GetThresholdFromUser())
self.assertEqual(10, self.bisector.threshold)
def testGetThresholdFromUserOutOfBoundFail(self):
"""Tests GetThresholdFromUser() with out-of-bound input."""
self.setDefaultCommitInfo()
input_mock = self.PatchObject(cros_build_lib,
'GetInput',
side_effect=['0', '1000', '-10'])
self.assertFalse(self.bisector.GetThresholdFromUser())
self.assertIsNone(self.bisector.threshold)
self.assertEqual(3, input_mock.call_count)
def testGetThresholdFromUserRetrySuccess(self):
"""Tests GetThresholdFromUser() with retry."""
self.setDefaultCommitInfo()
input_mock = self.PatchObject(
cros_build_lib,
'GetInput',
side_effect=['not_a_number', '1000', self.THRESHOLD_SPLITTER])
self.assertTrue(self.bisector.GetThresholdFromUser())
self.assertEqual(self.THRESHOLD, self.bisector.threshold)
self.assertEqual(3, input_mock.call_count)
def testBuildDeploy(self):
"""Tests BuildDeploy()."""
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
build_to_deploy = '/build/to/deploy'
build_mock = self.PatchObject(builder_module.Builder,
'Build',
return_value=build_to_deploy)
deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy')
self.assertTrue(self.bisector.BuildDeploy())
build_label = self.GOOD_COMMIT_INFO.sha1
build_mock.assert_called_with(build_label)
deploy_mock.assert_called_with(self.DUT, build_to_deploy, build_label)
def testBuildDeployBuildFail(self):
"""Tests BuildDeploy() with Build() failure."""
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
# Build() failed.
build_mock = self.PatchObject(builder_module.Builder,
'Build',
return_value=None)
deploy_mock = self.PatchObject(builder_module.Builder, 'Deploy')
self.assertFalse(self.bisector.BuildDeploy())
build_mock.assert_called()
deploy_mock.assert_not_called()
def PatchObjectForBuildDeployEval(self):
"""Returns a dict of patch objects.
The patch objects are to mock:
git_bisector.UpdateCurrentCommit()
evaluator.CheckLastEvaluate()
git_bisector.BuildDeploy()
evaluator.Evaluate()
"""
return {
'UpdateCurrentCommit':
self.PatchObject(git_bisector.GitBisector, 'UpdateCurrentCommit'),
'CheckLastEvaluate':
self.PatchObject(evaluator_module.Evaluator, 'CheckLastEvaluate'),
'BuildDeploy':
self.PatchObject(git_bisector.GitBisector,
'BuildDeploy',
return_value=True),
'Evaluate':
self.PatchObject(evaluator_module.Evaluator, 'Evaluate')
}
def testBuildDeployEvalShortcutCheckLastEvaluate(self):
"""Tests BuildDeployEval() with CheckLastEvaluate() found last score."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = self.GOOD_COMMIT_SCORE
score = self.bisector.BuildDeployEval()
self.assertEqual(self.GOOD_COMMIT_SCORE, score)
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.current_commit.score)
for method_called in ['UpdateCurrentCommit', 'CheckLastEvaluate']:
mocks[method_called].assert_called()
mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1,
self.REPEAT)
for method_called in ['BuildDeploy', 'Evaluate']:
mocks[method_called].assert_not_called()
def AssertBuildDeployEvalMocksAllCalled(self, mocks):
for method_called in [
'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy',
'Evaluate'
]:
mocks[method_called].assert_called()
mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1,
self.REPEAT)
mocks['Evaluate'].assert_called_with(self.DUT, self.GOOD_COMMIT_SHA1,
self.REPEAT)
def testBuildDeployEvalNoCheckLastEvaluate(self):
"""Tests BuildDeployEval() without last score."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.BuildDeployEval())
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.current_commit.score)
self.AssertBuildDeployEvalMocksAllCalled(mocks)
def testBuildDeployEvalBuildFail(self):
"""Tests BuildDeployEval() with BuildDeploy failure."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['BuildDeploy'].return_value = False
score = self.bisector.BuildDeployEval()
self.assertFalse(score)
self.assertFalse(self.bisector.current_commit.score)
for method_called in [
'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy'
]:
mocks[method_called].assert_called()
mocks['CheckLastEvaluate'].assert_called_with(self.GOOD_COMMIT_SHA1,
self.REPEAT)
mocks['Evaluate'].assert_not_called()
def testBuildDeployEvalNoCheckLastEvaluateSpecifyEvalLabel(self):
"""Tests BuildDeployEval() with eval_label specified."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE
eval_label = 'customized_label'
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.BuildDeployEval(eval_label=eval_label))
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.current_commit.score)
for method_called in [
'UpdateCurrentCommit', 'CheckLastEvaluate', 'BuildDeploy',
'Evaluate'
]:
mocks[method_called].assert_called()
# Use given label instead of SHA1 as eval label.
mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT)
mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT)
@staticmethod
def _DummyMethod():
"""A dummy method for test to call and mock."""
def testBuildDeployEvalNoCheckLastEvaluateSpecifyBuildDeploy(self):
"""Tests BuildDeployEval() with customize_build_deploy specified."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['Evaluate'].return_value = self.GOOD_COMMIT_SCORE
dummy_method = self.PatchObject(TestGitBisector,
'_DummyMethod',
return_value=True)
eval_label = 'customized_label'
self.assertEqual(
self.GOOD_COMMIT_SCORE,
self.bisector.BuildDeployEval(
eval_label=eval_label,
customize_build_deploy=TestGitBisector._DummyMethod))
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.current_commit.score)
for method_called in [
'UpdateCurrentCommit', 'CheckLastEvaluate', 'Evaluate'
]:
mocks[method_called].assert_called()
mocks['BuildDeploy'].assert_not_called()
dummy_method.assert_called()
# Use given label instead of SHA1 as eval label.
mocks['CheckLastEvaluate'].assert_called_with(eval_label, self.REPEAT)
mocks['Evaluate'].assert_called_with(self.DUT, eval_label, self.REPEAT)
def testBuildDeployEvalRaiseNoScore(self):
"""Tests BuildDeployEval() without score with eval_raise_on_error set."""
self.options.eval_raise_on_error = True
self.bisector = git_bisector.GitBisector(self.options, self.builder,
self.evaluator)
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['Evaluate'].return_value = common.Score()
self.assertRaises(Exception, self.bisector.BuildDeployEval)
self.assertFalse(self.bisector.current_commit.score)
self.AssertBuildDeployEvalMocksAllCalled(mocks)
def testBuildDeployEvalSuppressRaiseNoScore(self):
"""Tests BuildDeployEval() without score with eval_raise_on_error unset."""
mocks = self.PatchObjectForBuildDeployEval()
# Inject this as UpdateCurrentCommit's side effect.
self.bisector.current_commit = copy.deepcopy(self.GOOD_COMMIT_INFO)
mocks['CheckLastEvaluate'].return_value = common.Score()
mocks['Evaluate'].return_value = common.Score()
self.assertFalse(self.bisector.BuildDeployEval())
self.assertFalse(self.bisector.current_commit.score)
self.AssertBuildDeployEvalMocksAllCalled(mocks)
def testLabelBuild(self):
"""Tests LabelBuild()."""
# Inject good(100), bad(80) score and threshold.
self.setDefaultCommitInfo()
self.bisector.threshold = self.THRESHOLD
good = 'good'
bad = 'bad'
# Worse than given bad score.
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([70])))
# Better than bad score, but not good enough.
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([85])))
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90])))
# On the margin.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([95])))
# Better than the margin.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([98])))
# Better than given good score.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([110])))
# No score, default bad.
self.assertEqual(bad, self.bisector.LabelBuild(None))
self.assertEqual(bad, self.bisector.LabelBuild(common.Score()))
def testLabelBuildSkipNoScore(self):
"""Tests LabelBuild()."""
self.options.skip_failed_commit = True
self.bisector = git_bisector.GitBisector(self.options, self.builder,
self.evaluator)
# Inject good(100), bad(80) score and threshold.
self.setDefaultCommitInfo()
self.bisector.threshold = self.THRESHOLD
# No score, skip.
self.assertEqual('skip', self.bisector.LabelBuild(None))
self.assertEqual('skip', self.bisector.LabelBuild(common.Score()))
def testLabelBuildLowerIsBetter(self):
"""Tests LabelBuild() in lower-is-better condition."""
# Reverse good(80) and bad(100) score (lower is better), same threshold.
self.bisector.good_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO)
self.bisector.bad_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO)
self.bisector.threshold = self.THRESHOLD
good = 'good'
bad = 'bad'
# Better than given good score.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([70])))
# Worse than good score, but still better than margin.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([80])))
self.assertEqual(good, self.bisector.LabelBuild(common.Score([82])))
# On the margin.
self.assertEqual(good, self.bisector.LabelBuild(common.Score([85])))
# Worse than the margin.
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([88])))
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([90])))
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([95])))
# Worse than given bad score.
self.assertEqual(bad, self.bisector.LabelBuild(common.Score([110])))
def testGitBisect(self):
"""Tests GitBisect()."""
git_mock = self.PatchObject(git_bisector.GitBisector,
'Git',
return_value=cros_build_lib.CommandResult(
cmd=['git', 'bisect', 'reset'],
output='We are not bisecting.',
returncode=0))
result, done = self.bisector.GitBisect(['reset'])
git_mock.assert_called_with(['bisect', 'reset'])
self.assertFalse(done)
self.assertEqual('We are not bisecting.', result.output)
self.assertListEqual(['git', 'bisect', 'reset'], result.cmd)
def testGitBisectDone(self):
"""Tests GitBisect() when culprit is found."""
git_mock = self.PatchObject(
git_bisector.GitBisector,
'Git',
return_value=cros_build_lib.CommandResult(
cmd=['git', 'bisect', 'bad'],
output='abcedf is the first bad commit\ncommit abcdef',
returncode=0))
result, done = self.bisector.GitBisect(['bad'])
git_mock.assert_called_with(['bisect', 'bad'])
self.assertTrue(done)
self.assertListEqual(['git', 'bisect', 'bad'], result.cmd)
def testRun(self):
"""Tests Run()."""
bisector_mock = self.StartPatcher(GitBisectorMock())
bisector_mock.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO)
bisector_mock.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO)
bisector_mock.threshold = self.THRESHOLD
bisector_mock.git_bisect_args_result = [
(['reset'], (None, False)),
(['start'], (None, False)),
(['bad', self.BAD_COMMIT_SHA1], (None, False)),
(['good', self.GOOD_COMMIT_SHA1], (None, False)),
(['bad'],
(cros_build_lib.CommandResult(
cmd=['git', 'bisect', 'bad'],
output='%s is the first bad commit\ncommit %s' %
(self.CULPRIT_COMMIT_SHA1, self.CULPRIT_COMMIT_SHA1),
returncode=0),
True)), # bisect bad (assume it found the first bad commit).
(['log'], (None, False)),
(['reset'], (None, False))
]
bisector_mock.build_deploy_eval_current_commit = [
self.CULPRIT_COMMIT_INFO
]
bisector_mock.build_deploy_eval_result = [self.CULPRIT_COMMIT_SCORE]
bisector_mock.label_build_result = ['bad']
run_result = self.bisector.Run()
self.assertTrue(bisector_mock.patched['PrepareBisect'].called)
self.assertEqual(7, bisector_mock.patched['GitBisect'].call_count)
self.assertTrue(bisector_mock.patched['BuildDeployEval'].called)
self.assertTrue(bisector_mock.patched['LabelBuild'].called)
self.assertEqual(self.CULPRIT_COMMIT_SHA1, run_result)
def testEmpty(self):
"""Tests that empty CommitInfo's data members are initialized correctly."""
info = common.CommitInfo()
self.assertEqual(
"CommitInfo(sha1='', title='', score=Score(values=[]), label='', "
"timestamp=0)", repr(info))
class TestChromeOnCrosBisector(cros_test_lib.MockTempDirTestCase):
"""Tests ChromeOnCrosBisector class."""
BOARD = 'samus'
TEST_NAME = 'graphics_WebGLAquarium'
METRIC = 'avg_fps_1000_fishes/summary/value'
REPORT_FILE = 'reports.json'
DUT_ADDR = '192.168.1.1'
DUT = commandline.DeviceParser(commandline.DEVICE_SCHEME_SSH)(DUT_ADDR)
# Be aware that GOOD_COMMIT_INFO and BAD_COMMIT_INFO should be assigned via
# copy.deepcopy() as their users are likely to change the content.
GOOD_COMMIT_SHA1 = '44af5c9a5505'
GOOD_COMMIT_TIMESTAMP = 1486526594
GOOD_COMMIT_SCORE = common.Score([100])
GOOD_COMMIT_INFO = common.CommitInfo(sha1=GOOD_COMMIT_SHA1,
timestamp=GOOD_COMMIT_TIMESTAMP,
title='good',
label='last-known-good ',
score=GOOD_COMMIT_SCORE)
BAD_COMMIT_SHA1 = '6a163bb66c3e'
BAD_COMMIT_TIMESTAMP = 1486530021
BAD_COMMIT_SCORE = common.Score([80])
BAD_COMMIT_INFO = common.CommitInfo(sha1=BAD_COMMIT_SHA1,
timestamp=BAD_COMMIT_TIMESTAMP,
title='bad',
label='last-known-bad ',
score=BAD_COMMIT_SCORE)
GOOD_CROS_VERSION = 'R60-9592.50.0'
BAD_CROS_VERSION = 'R60-9592.51.0'
CULPRIT_COMMIT_SHA1 = '12345abcde'
CULPRIT_COMMIT_TIMESTAMP = 1486530000
CULPRIT_COMMIT_SCORE = common.Score([81])
CULPRIT_COMMIT_INFO = common.CommitInfo(sha1=CULPRIT_COMMIT_SHA1,
timestamp=CULPRIT_COMMIT_TIMESTAMP,
title='bad',
score=CULPRIT_COMMIT_SCORE)
THRESHOLD_SPLITTER = 95 # Score between good and bad, closer to good side.
THRESHOLD = 5 # Distance between good score and splitter.
REPEAT = 3
GOOD_METADATA_CONTENT = '\n'.join([
'{', ' "metadata-version": "2",',
' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",',
' "suite_scheduling": true,', ' "build_id": 1644146,',
' "version": {', ' "full": "R60-9592.50.0",',
' "android-branch": "git_mnc-dr-arc-m60",',
' "chrome": "60.0.3112.53",', ' "platform": "9592.50.0",',
' "milestone": "60",', ' "android": "4150402"', ' }', '}'
])
def setUp(self):
"""Sets up test case."""
self.options = cros_test_lib.EasyAttr(base_dir=self.tempdir,
board=self.BOARD,
reuse_repo=True,
good=self.GOOD_COMMIT_SHA1,
bad=self.BAD_COMMIT_SHA1,
remote=self.DUT,
eval_repeat=self.REPEAT,
auto_threshold=False,
reuse_eval=False,
cros_flash_sleep=0.01,
cros_flash_retry=3,
cros_flash_backoff=1,
eval_raise_on_error=False,
skip_failed_commit=False)
self.repo_dir = os.path.join(self.tempdir,
builder_module.Builder.DEFAULT_REPO_DIR)
self.SetUpBisector()
def SetUpBisector(self):
"""Instantiates self.bisector using self.options."""
self.evaluator = DummyEvaluator(self.options)
self.builder = builder_module.Builder(self.options)
self.bisector = chrome_on_cros_bisector.ChromeOnCrosBisector(
self.options, self.builder, self.evaluator)
def SetUpBisectorWithCrosVersion(self):
"""Instantiates self.bisector using CrOS version as good and bad options."""
self.options.good = self.GOOD_CROS_VERSION
self.options.bad = self.BAD_CROS_VERSION
self.SetUpBisector()
def SetDefaultCommitInfo(self):
"""Sets up default commit info."""
self.bisector.good_commit_info = copy.deepcopy(self.GOOD_COMMIT_INFO)
self.bisector.bad_commit_info = copy.deepcopy(self.BAD_COMMIT_INFO)
def testInit(self):
"""Tests __init__() with SHA1 as good and bad options."""
self.assertEqual(self.GOOD_COMMIT_SHA1, self.bisector.good_commit)
self.assertIsNone(self.bisector.good_cros_version)
self.assertEqual(self.BAD_COMMIT_SHA1, self.bisector.bad_commit)
self.assertIsNone(self.bisector.bad_cros_version)
self.assertFalse(self.bisector.bisect_between_cros_version)
self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw)
self.assertEqual(self.REPEAT, self.bisector.eval_repeat)
self.assertEqual(self.builder, self.bisector.builder)
self.assertEqual(self.repo_dir, self.bisector.repo_dir)
self.assertIsNone(self.bisector.good_commit_info)
self.assertIsNone(self.bisector.bad_commit_info)
self.assertEqual(0, len(self.bisector.bisect_log))
self.assertIsNone(self.bisector.threshold)
self.assertTrue(not self.bisector.current_commit)
def testInitCrosVersion(self):
"""Tests __init__() with CrOS version as good and bad options."""
self.SetUpBisectorWithCrosVersion()
self.assertEqual(self.GOOD_CROS_VERSION,
self.bisector.good_cros_version)
self.assertIsNone(self.bisector.good_commit)
self.assertEqual(self.BAD_CROS_VERSION, self.bisector.bad_cros_version)
self.assertIsNone(self.bisector.bad_commit)
self.assertTrue(self.bisector.bisect_between_cros_version)
self.assertEqual(self.DUT_ADDR, self.bisector.remote.raw)
self.assertEqual(self.REPEAT, self.bisector.eval_repeat)
self.assertEqual(self.builder, self.bisector.builder)
self.assertEqual(self.repo_dir, self.bisector.repo_dir)
self.assertIsNone(self.bisector.good_commit_info)
self.assertIsNone(self.bisector.bad_commit_info)
self.assertEqual(0, len(self.bisector.bisect_log))
self.assertIsNone(self.bisector.threshold)
self.assertTrue(not self.bisector.current_commit)
def testInitMissingRequiredArgs(self):
"""Tests that ChromeOnCrosBisector raises for missing required argument."""
options = cros_test_lib.EasyAttr()
with self.assertRaises(common.MissingRequiredOptionsException) as cm:
chrome_on_cros_bisector.ChromeOnCrosBisector(
options, self.builder, self.evaluator)
exception_message = str(cm.exception)
self.assertIn('Missing command line', exception_message)
self.assertIn('ChromeOnCrosBisector', exception_message)
for arg in chrome_on_cros_bisector.ChromeOnCrosBisector.REQUIRED_ARGS:
self.assertIn(arg, exception_message)
def testCheckCommitFormat(self):
"""Tests CheckCommitFormat()."""
CheckCommitFormat = (
chrome_on_cros_bisector.ChromeOnCrosBisector.CheckCommitFormat)
self.assertEqual(self.GOOD_COMMIT_SHA1,
CheckCommitFormat(self.GOOD_COMMIT_SHA1))
self.assertEqual(self.GOOD_CROS_VERSION,
CheckCommitFormat(self.GOOD_CROS_VERSION))
self.assertEqual('R60-9592.50.0', CheckCommitFormat('60.9592.50.0'))
invalid = 'bad_sha1'
self.assertIsNone(CheckCommitFormat(invalid))
def testObtainBisectBoundaryScoreImpl(self):
"""Tests ObtainBisectBoundaryScoreImpl()."""
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
build_deploy_eval_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval')
build_deploy_eval_mock.side_effect = [
self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE
]
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(True))
self.assertEqual(self.BAD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(False))
self.assertEqual([
mock.call(customize_build_deploy=None, eval_label=None),
mock.call(customize_build_deploy=None, eval_label=None)
], build_deploy_eval_mock.call_args_list)
def testObtainBisectBoundaryScoreImplCrosVersion(self):
"""Tests ObtainBisectBoundaryScoreImpl() with CrOS version."""
self.SetUpBisectorWithCrosVersion()
# Inject good_commit and bad_commit as if
# bisector.ResolveChromeBisectRangeFromCrosVersion() being run.
self.bisector.good_commit = self.GOOD_COMMIT_SHA1
self.bisector.bad_commit = self.BAD_COMMIT_SHA1
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'UpdateCurrentCommit')
evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate')
# Mock FlashCrosImage() to verify that customize_build_deploy is assigned
# as expected.
flash_cros_image_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage')
evaluate_mock.return_value = self.GOOD_COMMIT_SCORE
self.assertEqual(self.GOOD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(True))
flash_cros_image_mock.assert_called_with(
self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION))
evaluate_mock.assert_called_with(self.DUT,
'cros_%s' % self.GOOD_CROS_VERSION,
self.REPEAT)
evaluate_mock.return_value = self.BAD_COMMIT_SCORE
self.assertEqual(self.BAD_COMMIT_SCORE,
self.bisector.ObtainBisectBoundaryScoreImpl(False))
flash_cros_image_mock.assert_called_with(
self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION))
evaluate_mock.assert_called_with(self.DUT,
'cros_%s' % self.BAD_CROS_VERSION,
self.REPEAT)
def testObtainBisectBoundaryScoreImplCrosVersionFlashError(self):
"""Tests ObtainBisectBoundaryScoreImpl() with CrOS version."""
self.SetUpBisectorWithCrosVersion()
# Inject good_commit and bad_commit as if
# bisector.ResolveChromeBisectRangeFromCrosVersion() being run.
self.bisector.good_commit = self.GOOD_COMMIT_SHA1
self.bisector.bad_commit = self.BAD_COMMIT_SHA1
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'UpdateCurrentCommit')
evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate')
# Mock FlashCrosImage() to verify that customize_build_deploy is assigned
# as expected.
flash_cros_image_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage')
flash_cros_image_mock.side_effect = flash.FlashError('Flash failed.')
with self.assertRaises(flash.FlashError):
self.bisector.ObtainBisectBoundaryScoreImpl(True)
flash_cros_image_mock.assert_called_with(
self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION))
evaluate_mock.assert_not_called()
with self.assertRaises(flash.FlashError):
self.bisector.ObtainBisectBoundaryScoreImpl(False)
flash_cros_image_mock.assert_called_with(
self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION))
evaluate_mock.assert_not_called()
def testGetCrosXbuddyPath(self):
"""Tests GetCrosXbuddyPath()."""
self.assertEqual(
'xbuddy://remote/%s/%s/test' %
(self.BOARD, self.GOOD_CROS_VERSION),
self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION))
def testExchangeChromeSanityCheck(self):
"""Tests the flow of exchanging Chrome between good and bad CrOS."""
self.SetUpBisectorWithCrosVersion()
# Inject good_commit and bad_commit as if
# bisector.ResolveChromeBisectRangeFromCrosVersion() has been run.
self.bisector.good_commit = self.GOOD_COMMIT_SHA1
self.bisector.bad_commit = self.BAD_COMMIT_SHA1
# Inject commit_info and threshold as if
# bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser()
# has been run.
self.SetDefaultCommitInfo()
self.bisector.threshold = self.THRESHOLD
# Try bad Chrome first.
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'UpdateCurrentCommit')
evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate')
expected_evaluate_calls = [
mock.call(self.DUT, x, self.REPEAT) for x in [
'cros_%s_cr_%s' %
(self.GOOD_CROS_VERSION, self.BAD_COMMIT_SHA1),
'cros_%s_cr_%s' %
(self.BAD_CROS_VERSION, self.GOOD_COMMIT_SHA1)
]
]
# Mock FlashCrosImage() to verify that customize_build_deploy is assigned
# as expected.
flash_cros_image_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'FlashCrosImage')
expected_flash_cros_calls = [
mock.call(self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)),
mock.call(self.bisector.GetCrosXbuddyPath(self.BAD_CROS_VERSION))
]
# Make sure bisector.BuildDeploy() is also called.
build_deploy_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy')
# Assume culprit commit is in Chrome side, i.e. first score is bad.
evaluate_mock.side_effect = [
self.BAD_COMMIT_SCORE, self.GOOD_COMMIT_SCORE
]
self.assertTrue(self.bisector.ExchangeChromeSanityCheck())
flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls)
evaluate_mock.assert_has_calls(expected_evaluate_calls)
self.assertEqual(2, build_deploy_mock.call_count)
flash_cros_image_mock.reset_mock()
evaluate_mock.reset_mock()
build_deploy_mock.reset_mock()
# Assume culprit commit is not in Chrome side, i.e. first score is good.
evaluate_mock.side_effect = [
self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE
]
self.assertFalse(self.bisector.ExchangeChromeSanityCheck())
flash_cros_image_mock.assert_has_calls(expected_flash_cros_calls)
evaluate_mock.assert_has_calls(expected_evaluate_calls)
self.assertEqual(2, build_deploy_mock.call_count)
def testExchangeChromeSanityCheckFlashError(self):
"""Tests the flow of exchanging Chrome between good and bad CrOS."""
self.SetUpBisectorWithCrosVersion()
# Inject good_commit and bad_commit as if
# bisector.ResolveChromeBisectRangeFromCrosVersion() has been run.
self.bisector.good_commit = self.GOOD_COMMIT_SHA1
self.bisector.bad_commit = self.BAD_COMMIT_SHA1
# Inject commit_info and threshold as if
# bisector.ObtainBisectBoundaryScore() and bisector.GetThresholdFromUser()
# has been run.
self.SetDefaultCommitInfo()
self.bisector.threshold = self.THRESHOLD
# Try bad Chrome first.
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'UpdateCurrentCommit')
evaluate_mock = self.PatchObject(DummyEvaluator, 'Evaluate')
# Mock FlashCrosImage() to verify that customize_build_deploy is assigned
# as expected.
flash_cros_image_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector,
'FlashCrosImage',
side_effect=flash.FlashError('Flash failed.'))
build_deploy_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeploy')
with self.assertRaises(flash.FlashError):
self.bisector.ExchangeChromeSanityCheck()
evaluate_mock.assert_not_called()
flash_cros_image_mock.assert_called()
build_deploy_mock.assert_not_called()
def testFlashImage(self):
"""Tests FlashImage()."""
flash_mock = self.PatchObject(flash, 'Flash')
xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)
self.bisector.FlashCrosImage(xbuddy_path)
flash_mock.assert_called_with(self.DUT,
xbuddy_path,
board=self.BOARD,
clobber_stateful=True,
disable_rootfs_verification=True)
def testFlashImageRetry(self):
"""Tests FlashImage() with retry success."""
flash_mock_call_counter = itertools.count()
def flash_mock_return(*unused_args, **unused_kwargs):
nth_call = next(flash_mock_call_counter)
if nth_call < 3:
raise flash.FlashError('Flash failed.')
flash_mock = self.PatchObject(flash, 'Flash')
flash_mock.side_effect = flash_mock_return
xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)
self.bisector.FlashCrosImage(xbuddy_path)
flash_mock.assert_called_with(self.DUT,
xbuddy_path,
board=self.BOARD,
clobber_stateful=True,
disable_rootfs_verification=True)
def testFlashImageRetryFailed(self):
"""Tests FlashImage() with retry failed."""
flash_mock = self.PatchObject(flash, 'Flash')
flash_mock.side_effect = flash.FlashError('Flash failed.')
xbuddy_path = self.bisector.GetCrosXbuddyPath(self.GOOD_CROS_VERSION)
with self.assertRaises(flash.FlashError):
self.bisector.FlashCrosImage(xbuddy_path)
flash_mock.assert_called_with(self.DUT,
xbuddy_path,
board=self.BOARD,
clobber_stateful=True,
disable_rootfs_verification=True)
def testCrosVersionToChromeCommit(self):
"""Tests CrosVersionToChromeCommit()."""
metadata_url = (
'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' %
(self.BOARD, self.GOOD_CROS_VERSION))
gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
gs_mock.AddCmdResult(['cat', metadata_url],
output=self.GOOD_METADATA_CONTENT)
git_log_content = '\n'.join([
'8967dd66ad72 (tag: 60.0.3112.53) Publish DEPS for Chromium '
'60.0.3112.53', '27ed0cc0c2f4 Incrementing VERSION to 60.0.3112.53'
])
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(
['log', '--oneline', '-n', '2', '60.0.3112.53'],
output=git_log_content)
self.bisector.gs_ctx = gs.GSContext()
self.assertEqual(
'27ed0cc0c2f4',
self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION))
def testCrosVersionToChromeCommitFail(self):
"""Tests failure case of CrosVersionToChromeCommit()."""
metadata_url = (
'gs://chromeos-image-archive/%s-release/%s/partial-metadata.json' %
(self.BOARD, self.GOOD_CROS_VERSION))
gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
gs_mock.AddCmdResult(['cat', metadata_url], returncode=1)
self.bisector.gs_ctx = gs.GSContext()
self.assertIsNone(
self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION))
metadata_content = 'not_a_json'
gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content)
self.assertIsNone(
self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION))
metadata_content = '\n'.join([
'{', ' "metadata-version": "2",',
' "toolchain-url": "2017/05/%(target)s-2017.05.25.101355.tar.xz",',
' "suite_scheduling": true,', ' "build_id": 1644146,',
' "version": {}', '}'
])
gs_mock.AddCmdResult(['cat', metadata_url], output=metadata_content)
self.assertIsNone(
self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION))
gs_mock.AddCmdResult(['cat', metadata_url],
output=self.GOOD_METADATA_CONTENT)
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(
['log', '--oneline', '-n', '2', '60.0.3112.53'], returncode=128)
self.assertIsNone(
self.bisector.CrosVersionToChromeCommit(self.GOOD_CROS_VERSION))
def testResolveChromeBisectRangeFromCrosVersion(self):
"""Tests ResolveChromeBisectRangeFromCrosVersion()."""
self.SetUpBisectorWithCrosVersion()
cros_to_chrome_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector,
'CrosVersionToChromeCommit')
cros_to_chrome_mock.side_effect = [
self.GOOD_COMMIT_SHA1, self.BAD_COMMIT_SHA1
]
self.assertTrue(
self.bisector.ResolveChromeBisectRangeFromCrosVersion())
self.assertTrue(self.GOOD_COMMIT_SHA1, self.bisector.good_commit)
self.assertTrue(self.BAD_COMMIT_SHA1, self.bisector.bad_commit)
cros_to_chrome_mock.assert_has_calls([
mock.call(self.GOOD_CROS_VERSION),
mock.call(self.BAD_CROS_VERSION)
])
cros_to_chrome_mock.reset_mock()
cros_to_chrome_mock.side_effect = [None]
self.assertFalse(
self.bisector.ResolveChromeBisectRangeFromCrosVersion())
cros_to_chrome_mock.assert_called_with(self.GOOD_CROS_VERSION)
cros_to_chrome_mock.reset_mock()
cros_to_chrome_mock.side_effect = [self.GOOD_COMMIT_SHA1, None]
self.assertFalse(
self.bisector.ResolveChromeBisectRangeFromCrosVersion())
cros_to_chrome_mock.assert_has_calls([
mock.call(self.GOOD_CROS_VERSION),
mock.call(self.BAD_CROS_VERSION)
])
def testPrepareBisect(self):
"""Tests PrepareBisect()."""
# Pass SanityCheck().
git_mock = self.StartPatcher(
git_bisector_unittest.GitMock(self.repo_dir))
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.GOOD_COMMIT_SHA1]))
git_mock.AddRunGitResult(
partial_mock.InOrder(['rev-list', self.BAD_COMMIT_SHA1]))
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.GOOD_COMMIT_SHA1]),
output=str(self.GOOD_COMMIT_TIMESTAMP))
git_mock.AddRunGitResult(partial_mock.InOrder(
['show', self.BAD_COMMIT_SHA1]),
output=str(self.BAD_COMMIT_TIMESTAMP))
# Inject score for both side.
git_mock.AddRunGitResult(['checkout', self.GOOD_COMMIT_SHA1])
git_mock.AddRunGitResult(['checkout', self.BAD_COMMIT_SHA1])
build_deploy_eval_mock = self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector, 'BuildDeployEval')
build_deploy_eval_mock.side_effect = [
self.GOOD_COMMIT_SCORE, self.BAD_COMMIT_SCORE
]
# Set auto_threshold.
self.bisector.auto_threshold = True
self.assertTrue(self.bisector.PrepareBisect())
def testPrepareBisectCrosVersion(self):
"""Tests PrepareBisect() with CrOS version."""
self.SetUpBisectorWithCrosVersion()
self.StartPatcher(gs_unittest.GSContextMock())
self.PatchObject(builder_module.Builder, 'SyncToHead')
self.PatchObject(
chrome_on_cros_bisector.ChromeOnCrosBisector,
'ResolveChromeBisectRangeFromCrosVersion').return_value = True
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'SanityCheck').return_value = True
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'ObtainBisectBoundaryScore').return_value = True
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'GetThresholdFromUser').return_value = True
self.PatchObject(chrome_on_cros_bisector.ChromeOnCrosBisector,
'ExchangeChromeSanityCheck').return_value = True
self.assertTrue(self.bisector.PrepareBisect())