def testRootOwnedCache(self): """Test CleanBuildRoot with no history.""" seed_distfiles_ts = time.time() - 60 old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=2, branch='branchA', distfiles_ts=seed_distfiles_ts) self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchA' osutils.Chown(self.cache, 'root', 'root') build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchA') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'branchA') # Same cache creation timestamp is rewritten to state. self.assertEqual(new_summary.distfiles_ts, seed_distfiles_ts) self.assertEqual(new_summary, build_state) self.assertExists(self.repo) self.assertExists(self.chroot) self.assertExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state)
def testBuildrootFormatMismatch(self): """Test CleanBuildRoot with buildroot layout mismatch.""" old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=1, branch='master') self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'master' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='master') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'master') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) self.assertNotExists(self.repo) self.assertNotExists(self.chroot) self.assertNotExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state)
def testBuildrootDistfilesRecentCache(self): """Test CleanBuildRoot does not delete distfiles when cache is recent.""" seed_distfiles_ts = time.time() - 60 old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=2, branch='branchA', distfiles_ts=seed_distfiles_ts) self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchA' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchA') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.metrics, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'branchA') # Same cache creation timestamp is rewritten to state. self.assertEqual(new_summary.distfiles_ts, seed_distfiles_ts) self.assertEqual(new_summary, build_state) self.assertExists(self.repo) self.assertExists(self.chroot) self.assertExists(self.general) self.assertExists(self.distfiles) self.assertExists(self.previous_build_state)
def testBuildrootRepoCleanFailure(self): """Test CleanBuildRoot with repo checkout failure.""" old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=1, branch='branchA') self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchA' self.mock_repo.BuildRootGitCleanup.side_effect = Exception build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchA') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'branchA') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) self.assertNotExists(self.repo) self.assertNotExists(self.chroot) self.assertNotExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state)
def GetLastBuildState(root): """Fetch the state of the last build run from |root|. If the saved state file can't be read or doesn't contain valid JSON, a default state will be returned. Args: root: Root of the working directory tree as a string. Returns: A BuildSummary object representing the previous build. """ state_file = os.path.join(root, BUILDER_STATE_FILENAME) state = build_summary.BuildSummary() try: state_raw = osutils.ReadFile(state_file) state.from_json(state_raw) except IOError as e: logging.warning('Unable to read %s: %s', state_file, e) return state except ValueError as e: logging.warning('Saved state file %s is not valid JSON: %s', state_file, e) return state if not state.is_valid(): logging.warning('Previous build state is not valid. Ignoring.') state = build_summary.BuildSummary() return state
def testBuildrootDistfilesCacheExpired(self): """Test CleanBuildRoot when the distfiles cache is too old.""" old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=2, branch='branchA', distfiles_ts=100.0) self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchA' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchA') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'branchA') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) self.assertExists(self.repo) self.assertExists(self.chroot) self.assertExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state)
def testBuildrootBranchChange(self): """Test CleanBuildRoot with a change in branches.""" old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_PASSED, buildroot_layout=2, branch='branchA') self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchB' m = self.PatchObject(cros_sdk_lib, 'CleanupChrootMount') build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchB') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'branchB') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) # self.assertExists(self.repo) self.assertExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state) m.assert_called_with(self.chroot, delete=True)
def testPartialJson(self): """Test that fields not present in JSON aren't changed.""" summary = build_summary.BuildSummary() raw_json = '{"build_number": 10}' summary.from_json(raw_json) # Only build_number changes. default_summary = build_summary.BuildSummary() self.assertEqual(summary.build_number, 10) self.assertEqual(summary.master_build_id, default_summary.master_build_id) self.assertEqual(summary.status, default_summary.status)
def testRoundTrip(self): """Test that a BuildSummary can round-trip through JSON.""" summary1 = build_summary.BuildSummary() summary1.build_number = 314 summary1.buildbucket_id = 3140000 summary1.status = constants.BUILDER_STATUS_PASSED summary1.master_build_id = 2718 encoded = summary1.to_json() summary2 = build_summary.BuildSummary() summary2.from_json(encoded) self.assertEqual(summary1, summary2)
def testSetLastBuildState(self): """Verifies that SetLastBuildState writes to the expected file.""" osutils.SafeMakedirs(self.root) old_state = build_summary.BuildSummary( build_number=314, master_build_id=2178, status=constants.BUILDER_STATUS_PASSED) cbuildbot_launch.SetLastBuildState(self.root, old_state) saved_state = osutils.ReadFile(self.previous_build_state) new_state = build_summary.BuildSummary() new_state.from_json(saved_state) self.assertEqual(old_state, new_state)
def testChrootReusePreviousFailed(self): self.PatchObject(build_stages.CleanUpStage, '_GetPreviousBuildStatus', return_value=build_summary.BuildSummary( build_number=314, status=constants.BUILDER_STATUS_FAILED)) chroot_path = os.path.join(self.build_root, 'chroot') stage = self.ConstructStage() self.assertFalse(stage.CanReuseChroot(chroot_path))
def testGetLastBuildStateGoodFile(self): """Tests GetLastBuildState on a good file.""" osutils.SafeMakedirs(self.root) osutils.WriteFile( self.previous_build_state, '{"build_number": 1, "master_build_id": 3, "status": "pass"}') state = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual( state, build_summary.BuildSummary( build_number=1, master_build_id=3, status='pass'))
def testGetCurrentBuildStateNoArgs(self): """Tests GetCurrentBuildState without arguments.""" options = cbuildbot_launch.PreParseArguments( ['--buildroot', self.root, 'config']) state = cbuildbot_launch.GetCurrentBuildState(options, 'master') expected_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=2, branch='master') self.assertEqual(state, expected_state)
def testBuildrootGitLocksPrevFail(self): """Verify not CleanStaleLocks, if previous build was in failed.""" old_build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_FAILED, buildroot_layout=2, branch='branchA') self.populateBuildroot(previous_build_state=old_build_state.to_json()) self.mock_repo.branch = 'branchA' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branchA') cbuildbot_launch.CleanBuildRoot(self.root, self.mock_repo, self.cache, build_state) self.assertEqual(self.mock_repo.mock_calls, [ mock.call.PreLoad(), mock.call.BuildRootGitCleanup(prune_all=True), ])
def testGetCurrentBuildStateLayout(self): """Test that GetCurrentBuildState uses the current buildroot layout.""" # Change to a future version. self.PatchObject(cbuildbot_launch, 'BUILDROOT_BUILDROOT_LAYOUT', 22) options = cbuildbot_launch.PreParseArguments( ['--buildroot', self.root, 'config']) state = cbuildbot_launch.GetCurrentBuildState(options, 'branchA') expected_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=22, branch='branchA') self.assertEqual(state, expected_state)
def testChrootReuseChrootReplace(self): self._Prepare(extra_config={ 'chroot_use_image': False, 'chroot_replace': True }) self.PatchObject(build_stages.CleanUpStage, '_GetPreviousBuildStatus', return_value=build_summary.BuildSummary( build_number=314, status=constants.BUILDER_STATUS_PASSED)) chroot_path = os.path.join(self.build_root, 'chroot') stage = self.ConstructStage() self.assertFalse(stage.CanReuseChroot(chroot_path))
def testGetCurrentBuildStateHasArgs(self): """Tests GetCurrentBuildState with arguments.""" options = cbuildbot_launch.PreParseArguments([ '--buildroot', self.root, '--buildnumber', '20', '--master-build-id', '50', 'config' ]) state = cbuildbot_launch.GetCurrentBuildState(options, 'branchA') expected_state = build_summary.BuildSummary( build_number=20, master_build_id=50, status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=2, branch='branchA') self.assertEqual(state, expected_state)
def testBuildSummary(self): summary = build_summary.BuildSummary() description = summary.build_description() self.assertEqual(description, 'local build') # buildbucket_id string summary = build_summary.BuildSummary(buildbucket_id='bb_id') description = summary.build_description() self.assertEqual(description, 'buildbucket_id=bb_id') # buildbucket_id int (can this really happen) summary = build_summary.BuildSummary(buildbucket_id=1234) description = summary.build_description() self.assertEqual(description, 'buildbucket_id=1234') # build_number int. summary = build_summary.BuildSummary(build_number=12) description = summary.build_description() self.assertEqual(description, 'build_number=12') # build_number str. summary = build_summary.BuildSummary(build_number='12') description = summary.build_description() self.assertEqual(description, 'build_number=12')
def _GetPreviousBuildStatus(self): """Extract the status of the previous build from command-line arguments. Returns: A BuildSummary object representing the previous build. """ previous_state = build_summary.BuildSummary() if self._run.options.previous_build_state: try: state_json = base64.b64decode( self._run.options.previous_build_state) previous_state.from_json(state_json) logging.info('Previous local build %s finished in state %s.', previous_state.build_description(), previous_state.status) except ValueError as e: logging.error('Failed to decode previous build state: %s', e) return previous_state
def testNoBuildroot(self): """Test CleanBuildRoot with no history.""" self.mock_repo.branch = 'master' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='master') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.cache, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'master') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) self.assertExists(self.previous_build_state)
def testChrootReuseAllPassed(self): master_id = self.fake_db.InsertBuild( 'test_builder', waterfall.WATERFALL_TRYBOT, 123, 'test_config', 'test_hostname', status=constants.BUILDER_STATUS_PASSED, buildbucket_id='2178') self.PatchObject(build_stages.CleanUpStage, '_GetPreviousBuildStatus', return_value=build_summary.BuildSummary( build_number=314, master_build_id=master_id, status=constants.BUILDER_STATUS_PASSED)) chroot_path = os.path.join(self.build_root, 'chroot') stage = self.ConstructStage() self.assertTrue(stage.CanReuseChroot(chroot_path))
def GetCurrentBuildState(options, branch): """Extract information about the current build state from command-line args. Args: options: A parsed options object from a cbuildbot parser. branch: The name of the branch this builder was called with. Returns: A BuildSummary object describing the current build. """ build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=BUILDROOT_BUILDROOT_LAYOUT, branch=branch) if options.buildnumber: build_state.build_number = options.buildnumber if options.buildbucket_id: build_state.buildbucket_id = options.buildbucket_id if options.master_build_id: build_state.master_build_id = options.master_build_id return build_state
def testBuildrootNoState(self): """Test CleanBuildRoot with no state information.""" self.populateBuildroot() self.mock_repo.branch = 'master' build_state = build_summary.BuildSummary( status=constants.BUILDER_STATUS_INFLIGHT, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='master') cbuildbot_launch.CleanBuildRoot( self.root, self.mock_repo, self.metrics, build_state) new_summary = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(new_summary.buildroot_layout, 2) self.assertEqual(new_summary.branch, 'master') self.assertIsNotNone(new_summary.distfiles_ts) self.assertEqual(new_summary, build_state) self.assertNotExists(self.repo) self.assertNotExists(self.chroot) self.assertNotExists(self.general) self.assertNotExists(self.distfiles) self.assertExists(self.previous_build_state)
def testJsonNoUnecessaryKeys(self): """Test that the JSON output doesn't contain keys for zero values.""" summary = build_summary.BuildSummary(build_number=5) out_fields = json.loads(summary.to_json()) self.assertSetEqual(set(out_fields.keys()), set(['build_number', 'status']))
def testAttributesListMatchObject(self): """Test that all attributes are listed in _PERSIST_ATTRIBUTES.""" summary = build_summary.BuildSummary() # pylint: disable=protected-access self.assertSetEqual(set(vars(summary).keys()), set(summary._PERSIST_ATTRIBUTES))
def testGetLastBuildStateMissingBuildStatus(self): """Tests GetLastBuildState if the file doesn't have a valid status.""" osutils.SafeMakedirs(self.root) osutils.WriteFile(self.previous_build_state, '{"build_number": "3"}') state = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(state, build_summary.BuildSummary())
def testGetLastBuildStateBadFile(self): """Tests GetLastBuildState if the file contains invalid JSON.""" osutils.SafeMakedirs(self.root) osutils.WriteFile(self.previous_build_state, '}}') state = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(state, build_summary.BuildSummary())
def testGetLastBuildStateNoFile(self): """Tests GetLastBuildState if the file is missing.""" osutils.SafeMakedirs(self.root) state = cbuildbot_launch.GetLastBuildState(self.root) self.assertEqual(state, build_summary.BuildSummary())
def testMainMin(self): """Test a minimal set of command line options.""" self.PatchObject(osutils, 'SafeMakedirs', autospec=True) self.PatchObject(commands, 'GetTargetChromiteApiVersion', autospec=True, return_value=(constants.REEXEC_API_MAJOR, constants.REEXEC_API_MINOR)) mock_repo = mock.MagicMock() mock_repo.branch = 'master' mock_repo.directory = '/root/repository' mock_repo_create = self.PatchObject(repository, 'RepoRepository', autospec=True, return_value=mock_repo) mock_clean = self.PatchObject(cbuildbot_launch, 'CleanBuildRoot', autospec=True) mock_checkout = self.PatchObject(cbuildbot_launch, 'InitialCheckout', autospec=True) mock_cleanup_chroot = self.PatchObject(cbuildbot_launch, 'CleanupChroot', autospec=True) mock_set_last_build_state = self.PatchObject( cbuildbot_launch, 'SetLastBuildState', autospec=True) expected_build_state = build_summary.BuildSummary( build_number=0, master_build_id=0, status=mock.ANY, buildroot_layout=2, branch='master') argv = ['-r', '/root', 'config'] options = cbuildbot_launch.PreParseArguments(argv) cbuildbot_launch._main(options, argv) # Did we create the repo instance correctly? self.assertEqual(mock_repo_create.mock_calls, [mock.call(EXPECTED_MANIFEST_URL, '/root/repository', git_cache_dir=None, branch='master')]) # Ensure we clean, as expected. self.assertEqual(mock_clean.mock_calls, [ mock.call('/root', mock_repo, '/root/repository/.cache', expected_build_state)]) # Ensure we checkout, as expected. self.assertEqual(mock_checkout.mock_calls, [mock.call(mock_repo)]) # Ensure we invoke cbuildbot, as expected. self.assertCommandCalled( [ '/root/repository/chromite/bin/cbuildbot', 'config', '-r', '/root/repository', '--workspace', '/root/workspace', '--cache-dir', '/root/repository/.cache', # The duplication is a bug, but not harmful. '--cache-dir', '/root/repository/.cache', ], extra_env={'PATH': mock.ANY}, cwd='/root/repository', check=False) # Ensure we saved the final state, as expected. self.assertEqual(expected_build_state.status, constants.BUILDER_STATUS_PASSED) self.assertEqual(mock_set_last_build_state.mock_calls, [ mock.call('/root', expected_build_state)]) # Ensure we clean the chroot, as expected. mock_cleanup_chroot.assert_called_once_with('/root/repository')
def testMainMax(self): """Test a larger set of command line options.""" self.PatchObject(osutils, 'SafeMakedirs', autospec=True) self.PatchObject(commands, 'GetTargetChromiteApiVersion', autospec=True, return_value=(constants.REEXEC_API_MAJOR, constants.REEXEC_API_MINOR)) mock_repo = mock.MagicMock() mock_repo.branch = 'branch' mock_repo.directory = '/root/repository' mock_summary = build_summary.BuildSummary( build_number=313, master_build_id=123123123, status=constants.BUILDER_STATUS_FAILED, buildroot_layout=cbuildbot_launch.BUILDROOT_BUILDROOT_LAYOUT, branch='branch') mock_get_last_build_state = self.PatchObject( cbuildbot_launch, 'GetLastBuildState', autospec=True, return_value=mock_summary) mock_repo_create = self.PatchObject(repository, 'RepoRepository', autospec=True, return_value=mock_repo) mock_clean = self.PatchObject(cbuildbot_launch, 'CleanBuildRoot', autospec=True) mock_checkout = self.PatchObject(cbuildbot_launch, 'InitialCheckout', autospec=True) mock_cleanup_chroot = self.PatchObject(cbuildbot_launch, 'CleanupChroot', autospec=True) mock_set_last_build_state = self.PatchObject( cbuildbot_launch, 'SetLastBuildState', autospec=True) argv = ['--buildroot', '/root', '--branch', 'branch', '--git-cache-dir', '/git-cache', '--cache-dir', '/cache', '--remote-trybot', '--master-build-id', '123456789', '--buildnumber', '314', 'config'] options = cbuildbot_launch.PreParseArguments(argv) cbuildbot_launch._main(options, argv) # Did we create the repo instance correctly? self.assertEqual(mock_repo_create.mock_calls, [mock.call(EXPECTED_MANIFEST_URL, '/root/repository', git_cache_dir='/git-cache', branch='branch')]) # Ensure we look up the previous status. self.assertEqual(mock_get_last_build_state.mock_calls, [ mock.call('/root')]) # Ensure we clean, as expected. self.assertEqual(mock_clean.mock_calls, [ mock.call('/root', mock_repo, '/cache', build_summary.BuildSummary( build_number=314, master_build_id=123456789, status=mock.ANY, branch='branch', buildroot_layout=2 ))]) # Ensure we checkout, as expected. self.assertEqual(mock_checkout.mock_calls, [mock.call(mock_repo)]) # Ensure we invoke cbuildbot, as expected. self.assertCommandCalled( [ '/root/repository/chromite/bin/cbuildbot', 'config', '--buildroot', '/root/repository', '--branch', 'branch', '--git-cache-dir', '/git-cache', '--cache-dir', '/cache', '--remote-trybot', '--master-build-id', '123456789', '--buildnumber', '314', '--previous-build-state', 'eyJicmFuY2giOiJicmFuY2giLCJidWlsZF9udW1iZXIiOjMxMywiYnVpbGRyb290X' '2xheW91dCI6MiwibWFzdGVyX2J1aWxkX2lkIjoxMjMxMjMxMjMsInN0YXR1cyI6Im' 'ZhaWwifQ==', '--workspace', '/root/workspace', '--cache-dir', '/cache', ], extra_env={'PATH': mock.ANY}, cwd='/root/repository', check=False) # Ensure we write the final build state, as expected. final_state = build_summary.BuildSummary( build_number=314, master_build_id=123456789, status=constants.BUILDER_STATUS_PASSED, buildroot_layout=2, branch='branch') self.assertEqual(mock_set_last_build_state.mock_calls, [ mock.call('/root', final_state)]) # Ensure we clean the chroot, as expected. mock_cleanup_chroot.assert_called_once_with('/root/repository')