def testCompareLess(self):
"""Verify comparisons of less versions."""
lhs = manifest_version.VersionInfo(version_string='1.0.3')
rhs = manifest_version.VersionInfo(version_string='1.2.3')
self.assertTrue(lhs < rhs)
self.assertTrue(lhs <= rhs)
self.assertFalse(lhs == rhs)
self.assertTrue(lhs != rhs)
self.assertFalse(lhs > rhs)
self.assertFalse(lhs >= rhs)
def _IncrementVersionOnDiskForNewBranch(self, push_remote):
"""Bumps the version found in chromeos_version.sh on the new branch
When a new branch is created, the branch component of the new branch's
version needs to bumped.
For example, say 'stabilize-link' is created from a the 4230.0.0 manifest.
The new branch's version needs to be bumped to 4230.1.0.
Args:
push_remote: a git remote name where the new branch lives.
"""
# This needs to happen before the source branch version bumping above
# because we rely on the fact that since our current overlay checkout
# is what we just pushed to the new branch, we don't need to do another
# sync. This also makes it easier to implement skip_remote_push
# functionality (the new branch doesn't actually get created in
# skip_remote_push mode).
# Use local branch ref.
branch_ref = git.NormalizeRef(self.branch_name)
push_to = git.RemoteRef(push_remote, branch_ref)
version_info = manifest_version.VersionInfo(
version_string=self._run.options.force_version)
incr_type, incr_target = self.DetermineBranchIncrParams(version_info)
message = self.COMMIT_MESSAGE % {
'target': incr_target,
'branch': branch_ref,
}
self._IncrementVersionOnDisk(incr_type, push_to, message)
def __init__(self,
overlays,
manifest,
build_targets=None,
chroot=None,
output_dir=None):
"""Init function.
Args:
overlays (list[str]): The overlays to search for ebuilds.
manifest (git.ManifestCheckout): The manifest object.
build_targets (list[build_target_lib.BuildTarget]|None): The build
targets to clean in |chroot|, if desired. No effect unless |chroot| is
provided.
chroot (chroot_lib.Chroot|None): The chroot to clean, if desired.
output_dir (str|None): The path to optionally dump result files.
"""
self.overlays = overlays
self.manifest = manifest
self.build_targets = build_targets or []
self.chroot = chroot
self.output_dir = output_dir
self._revved_packages = None
self._new_package_atoms = None
self._new_ebuild_files = None
self._removed_ebuild_files = None
self._overlay_ebuilds = None
# We cleaned up self referential ebuilds by this version, but don't enforce
# the check on older ones to avoid breaking factory/firmware branches.
root_version = manifest_version.VersionInfo.from_repo(
constants.SOURCE_ROOT)
no_self_repos_version = manifest_version.VersionInfo('13099.0.0')
self._reject_self_repo = root_version >= no_self_repos_version
def setUp(self):
self.version_info = manifest_version.VersionInfo('1.2.3', '11')
self.manifest_versions_int = os.path.join(self.tempdir,
'manifest_versions_int')
self.manifest_versions_ext = os.path.join(self.tempdir,
'manifest_versions_ext')
def testInitializeManifestVariablesWithPassedBuild(self):
"""Test InitializeManifestVariables with passed build."""
self.manager = self.BuildManager()
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
for_build = os.path.join(self.manager.manifest_dir, 'build-name',
self.build_names[0])
m1, m2, m3, m4 = self._buildManifest()
# Fail 1, pass 2, pass 3, pass 4
manifest_version.CreateSymlink(
m1,
os.path.join(for_build, 'fail', CHROME_BRANCH,
os.path.basename(m1)))
for m in [m2, m3, m4]:
manifest_version.CreateSymlink(
m,
os.path.join(for_build, 'pass', CHROME_BRANCH,
os.path.basename(m)))
latest_builds = [{
'build_config': self.build_names[0],
'status': 'pass',
'platform_version': '1.2.5'
}]
self.manager.buildstore.fake_cidb.GetBuildHistory.return_value = (
latest_builds)
self.manager.InitializeManifestVariables(info)
self.assertIsNone(self.manager.latest_unprocessed)
self.assertEqual(self.manager._latest_build, latest_builds[0])
def _GetOverlayToEbuildsMap(options, overlays, package_list):
"""Get ebuilds for overlays.
Args:
options: The options object returned by the argument parser.
overlays: A list of overlays to work on.
package_list: A list of packages passed from commandline to work on.
Returns:
A dict mapping each overlay to a list of ebuilds belonging to it.
"""
root_version = manifest_version.VersionInfo.from_repo(options.buildroot)
subdir_removal = manifest_version.VersionInfo('10363.0.0')
require_subdir_support = root_version < subdir_removal
overlay_ebuilds = {}
inputs = [[
overlay, options.all, package_list, options.force,
require_subdir_support
] for overlay in overlays]
result = parallel.RunTasksInProcessPool(portage_util.GetOverlayEBuilds,
inputs)
for idx, ebuilds in enumerate(result):
overlay_ebuilds[overlays[idx]] = ebuilds
return overlay_ebuilds
def _GetOverlayToEbuildsMap(overlays, package_list, force):
"""Get ebuilds for overlays.
Args:
overlays: A list of overlays to work on.
package_list: A list of packages passed from commandline to work on.
force (bool): Whether to use packages even if in blacklist.
Returns:
A dict mapping each overlay to a list of ebuilds belonging to it.
"""
root_version = manifest_version.VersionInfo.from_repo(
constants.SOURCE_ROOT)
subdir_removal = manifest_version.VersionInfo('10363.0.0')
require_subdir_support = root_version < subdir_removal
use_all = not package_list
overlay_ebuilds = {}
inputs = [[overlay, use_all, package_list, force, require_subdir_support]
for overlay in overlays]
result = parallel.RunTasksInProcessPool(portage_util.GetOverlayEBuilds,
inputs)
for idx, ebuilds in enumerate(result):
overlay_ebuilds[overlays[idx]] = ebuilds
return overlay_ebuilds
def CommonTestIncrementVersion(self,
incr_type,
version,
chrome_branch=None):
"""Common test increment. Returns path to new incremented file."""
message = 'Incrementing cuz I sed so'
create_mock = self.PatchObject(git, 'CreateBranch')
push_mock = self.PatchObject(manifest_version, '_PushGitChanges')
clean_mock = self.PatchObject(git, 'CleanAndCheckoutUpstream')
version_file = self.CreateFakeVersionFile(self.tempdir,
version=version,
chrome_branch=chrome_branch)
info = manifest_version.VersionInfo(version_file=version_file,
incr_type=incr_type)
info.IncrementVersion()
info.UpdateVersionFile(message, dry_run=False)
create_mock.assert_called_once_with(self.tempdir,
manifest_version.PUSH_BRANCH)
push_mock.assert_called_once_with(self.tempdir,
message,
dry_run=False,
push_to=None)
clean_mock.assert_called_once_with(self.tempdir)
return version_file
def testIncrementVersionChrome(self):
"""Tests whether we can increment the chrome version."""
version_file = self.CommonTestIncrementVersion('chrome_branch',
version='1.0.0',
chrome_branch='29')
new_info = manifest_version.VersionInfo(version_file=version_file)
self.assertEqual(new_info.VersionString(), '2.0.0')
self.assertEqual(new_info.chrome_branch, '30')
def GetVersionInfo(self):
"""Returns a manifest_version.VersionInfo object for this build.
Chrome OS Subclasses must override this method. Site specific builds which
don't use Chrome OS versioning should leave this alone.
"""
# Placeholder version for non-Chrome OS builds.
return manifest_version.VersionInfo('1.0.0')
def setUp(self):
verinfo = manifest_version.VersionInfo(
version_string=self.VERSION, chrome_branch=self.CHROME_BRANCH)
self.StartPatcher(BuilderRunMock(verinfo))
self.PatchObject(simple_builders.SimpleBuilder, 'GetVersionInfo',
return_value=verinfo)
def ConstructStage(self):
self._run.attrs.version_info = manifest_version.VersionInfo(
'10.0.0', chrome_branch='10')
self._run.attrs.release_tag = 'infra-tag'
return branch_archive_stages.FirmwareArchiveStage(
self._run,
self.buildstore,
build_root=self.workspace,
board='board')
def ConstructStage(self):
# Version for the infra branch.
self._run.attrs.version_info = manifest_version.VersionInfo(
'10.0.0', chrome_branch='10')
self._run.attrs.release_tag = 'infra-tag'
return workspace_stages.WorkspaceDebugSymbolsStage(
self._run,
self.buildstore,
build_root=self.workspace,
board='board')
def __init__(self, user_email, workdir, lkgm, dryrun=False):
self._committer = chrome_committer.ChromeCommitter(
user_email, workdir, dryrun)
# Strip any chrome branch from the lkgm version.
self._lkgm = manifest_version.VersionInfo(lkgm).VersionString()
self._old_lkgm = None
if not self._lkgm:
raise LKGMNotValid('LKGM not provided.')
logging.info('lkgm=%s', lkgm)
def __init__(self, args):
self._committer = chrome_committer.ChromeCommitter(args)
# Strip any chrome branch from the lkgm version.
self._lkgm = manifest_version.VersionInfo(args.lkgm).VersionString()
self._old_lkgm = None
if not self._lkgm:
raise LKGMNotValid('LKGM not provided.')
logging.info('lkgm=%s', self._lkgm)
def AfterLimit(self, limit):
"""Is worksapce version newer than cutoff limit?
Args:
limit: String version of format '123.0.0'
Returns:
bool: True if workspace has newer version than limit.
"""
version_info = self.GetWorkspaceVersionInfo()
return version_info > manifest_version.VersionInfo(limit)
def testGetNextVersionNoIncrement(self):
"""Tests whether we can get the next version to be built correctly.
Tests without pre-existing version in manifest dir.
"""
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
self.manager.latest = None
version = self.manager.GetNextVersion(info)
self.assertEqual(FAKE_VERSION_STRING, version)
def WriteFakeVersionFile(cls,
version_file,
version=None,
chrome_branch=None):
"""Helper method to write a version file from specified version number."""
if version is None:
version = FAKE_VERSION_STRING
if chrome_branch is None:
chrome_branch = CHROME_BRANCH
osutils.SafeMakedirs(os.path.split(version_file)[0])
info = manifest_version.VersionInfo(version, chrome_branch)
osutils.WriteFile(version_file, FAKE_VERSION % info.__dict__)
def testGetNextVersionIncrement(self):
"""Tests that we create a new version if a previous one exists."""
m = self.PatchObject(manifest_version.VersionInfo, 'UpdateVersionFile')
version_file = VersionInfoTest.CreateFakeVersionFile(self.tempdir)
info = manifest_version.VersionInfo(version_file=version_file,
incr_type='branch')
self.manager.latest = FAKE_VERSION_STRING
version = self.manager.GetNextVersion(info)
self.assertEqual(FAKE_VERSION_STRING_NEXT, version)
m.assert_called_once_with(
'Automatic: %s - Updating to a new version number from %s' %
(self.build_names[0], FAKE_VERSION_STRING),
dry_run=True)
def testGetNextBuildSpec(self):
"""End-to-end test of updating the manifest."""
my_info = manifest_version.VersionInfo('1.2.3', chrome_branch='4')
self.PatchObject(manifest_version.BuildSpecsManager,
'GetCurrentVersionInfo',
return_value=my_info)
self.PatchObject(repository.RepoRepository, 'Sync')
self.PatchObject(repository.RepoRepository,
'ExportManifest',
return_value='<manifest />')
rc = self.StartPatcher(cros_build_lib_unittest.RunCommandMock())
rc.SetDefaultCmdResult()
self.manager.GetNextBuildSpec(retries=0)
self.manager.UpdateStatus({self.build_names[0]: True})
def test_versions_based_on_mock(self):
# Create a test version_info object, and than mock VersionInfo.from_repo
# return it.
test_platform_version = '12575.0.0'
test_chrome_branch = '75'
version_info_mock = manifest_version.VersionInfo(test_platform_version)
version_info_mock.chrome_branch = test_chrome_branch
self.PatchObject(manifest_version.VersionInfo, 'from_repo',
return_value=version_info_mock)
test_full_version = 'R' + test_chrome_branch + '-' + test_platform_version
platform_version = packages.determine_platform_version()
milestone_version = packages.determine_milestone_version()
full_version = packages.determine_full_version()
self.assertEqual(platform_version, test_platform_version)
self.assertEqual(milestone_version, test_chrome_branch)
self.assertEqual(full_version, test_full_version)
def testPublishManifestCommitMessageWithNoneBuildId(self):
"""Tests that PublishManifest doesn't write a non-existant build_id"""
expected_message = 'Automatic: Start x86-generic-paladin master 1'
push_mock = self.PatchObject(self.manager, 'PushSpecChanges')
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
# Create a fake manifest file.
m = os.path.join(self.tmpmandir, '1.xml')
osutils.Touch(m)
self.manager.InitializeManifestVariables(info)
self.manager.PublishManifest(m, '1')
push_mock.assert_called_once_with(expected_message)
def setUp(self):
self.fake_db = fake_cidb.FakeCIDBConnection()
cidb.CIDBConnectionFactory.SetupMockCidb(self.fake_db)
build_id = self.fake_db.InsertBuild('builder name', 'waterfall', 1,
'build config', 'bot hostname')
self._Prepare(build_id=build_id)
release_tag = '4815.0.0-rc1'
self._run.attrs.release_tag = '4815.0.0-rc1'
fake_versioninfo = manifest_version.VersionInfo(release_tag, '39')
self.gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
self.gs_mock.SetDefaultCmdResult()
self.PatchObject(cbuildbot_run._BuilderRunBase,
'GetVersionInfo',
return_value=fake_versioninfo)
self.PatchObject(toolchain, 'GetToolchainsForBoard')
def testPublishManifestCommitMessageWithBuildId(self):
"""Tests that PublishManifest writes a build id."""
self.manager = self.BuildManager()
expected_message = ('Automatic: Start x86-generic-paladin master 1\n'
'CrOS-Build-Id: %s' % MOCK_BUILD_ID)
push_mock = self.PatchObject(self.manager, 'PushSpecChanges')
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
# Create a fake manifest file.
m = os.path.join(self.tmpmandir, '1.xml')
osutils.Touch(m)
self.manager.InitializeManifestVariables(info)
self.manager.PublishManifest(m, '1', build_id=MOCK_BUILD_ID)
push_mock.assert_called_once_with(expected_message)
def _WorkOnCommit(options, overlays, overlay_tracking_branch,
git_project_overlays, manifest, package_list):
"""Commit uprevs of overlays belonging to different git projects in parallel.
Args:
options: The options object returned by the argument parser.
overlays: A list of overlays to work on.
overlay_tracking_branch: A dict mapping from each overlay to its tracking
branch.
git_project_overlays: A dict mapping from each git repository to a list of
its overlays.
manifest: The manifest of the given source root.
package_list: A list of packages passed from commandline to work on.
"""
# We cleaned up self referential ebuilds by this version, but don't enforce
# the check on older ones to avoid breaking factory/firmware branches.
root_version = manifest_version.VersionInfo.from_repo(options.buildroot)
no_self_repos_version = manifest_version.VersionInfo('13099.0.0')
reject_self_repo = root_version >= no_self_repos_version
overlay_ebuilds = _GetOverlayToEbuildsMap(options, overlays, package_list)
with parallel.Manager() as manager:
# Contains the array of packages we actually revved.
revved_packages = manager.list()
new_package_atoms = manager.list()
inputs = [[
options, manifest, overlays_per_project, overlay_tracking_branch,
overlay_ebuilds, revved_packages, new_package_atoms,
reject_self_repo
] for overlays_per_project in git_project_overlays.values()]
parallel.RunTasksInProcessPool(_CommitOverlays, inputs)
chroot_path = os.path.join(options.buildroot,
constants.DEFAULT_CHROOT_DIR)
if os.path.exists(chroot_path):
CleanStalePackages(options.buildroot, options.boards.split(':'),
new_package_atoms)
if options.drop_file:
osutils.WriteFile(options.drop_file, ' '.join(revved_packages))
def testLatestSpecFromDir(self):
"""Tests whether we can get sorted specs correctly from a directory."""
self.PatchObject(repository, 'CloneGitRepo', side_effect=Exception())
info = manifest_version.VersionInfo('99.1.2',
CHROME_BRANCH,
incr_type='branch')
specs_dir = os.path.join(self.manager.manifest_dir, 'buildspecs',
CHROME_BRANCH)
m1, m2, m3, m4 = [
os.path.join(specs_dir, x) for x in
['100.0.0.xml', '99.3.3.xml', '99.1.10.xml', '99.1.5.xml']
]
# Create fake buildspecs.
osutils.SafeMakedirs(specs_dir)
for m in [m1, m2, m3, m4]:
osutils.Touch(m)
spec = self.manager._LatestSpecFromDir(info, specs_dir)
# Should be the latest on the 99.1 branch.
self.assertEqual(spec, '99.1.10')
def testLoadSpecs(self):
"""Tests whether we can load specs correctly."""
self.manager = self.BuildManager()
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
mpath = os.path.join(self.manager.manifest_dir, 'buildspecs',
CHROME_BRANCH)
m1, m2, m3, m4 = [
os.path.join(mpath, '1.2.%d.xml' % x) for x in [2, 3, 4, 5]
]
for_build = os.path.join(self.manager.manifest_dir, 'build-name',
self.build_names[0])
# Create fake buildspecs.
osutils.SafeMakedirs(os.path.join(mpath))
for m in [m1, m2, m3, m4]:
osutils.Touch(m)
# Fake BuilderStatus with status MISSING.
missing = manifest_version.BuilderStatus(
constants.BUILDER_STATUS_MISSING, None)
# Fail 1, pass 2, leave 3,4 unprocessed.
manifest_version.CreateSymlink(
m1,
os.path.join(for_build, 'fail', CHROME_BRANCH,
os.path.basename(m1)))
manifest_version.CreateSymlink(
m1,
os.path.join(for_build, 'pass', CHROME_BRANCH,
os.path.basename(m2)))
m = self.PatchObject(self.manager,
'GetBuildStatus',
return_value=missing)
self.manager.InitializeManifestVariables(info)
self.assertEqual(self.manager.latest_unprocessed, '1.2.5')
m.assert_called_once_with(self.build_names[0], '1.2.5')
def _populate_overlay_ebuilds(self,
use_all=True,
package_list=None,
force=False):
"""Populates the overlay to ebuilds mapping.
Populate self._overlay_ebuilds for all overlays in self.overlays unless
otherwise specified by package_list.
Args:
use_all: Whether to include all ebuilds in the specified directories.
If true, then we gather all packages in the directories regardless
of whether they are in our set of packages.
package_list (list[str]): A set of the packages we want to gather. If
use_all is True, this argument is ignored, and should be None.
force: Boolean indicating whether or not to consider blacklisted ebuilds.
"""
# See crrev.com/c/1257944 for origins of this.
root_version = manifest_version.VersionInfo.from_repo(
constants.SOURCE_ROOT)
subdir_removal = manifest_version.VersionInfo('10363.0.0')
require_subdir_support = root_version < subdir_removal
if not package_list:
package_list = []
overlay_ebuilds = {}
inputs = [[
overlay, use_all, package_list, force, require_subdir_support
] for overlay in self.overlays]
result = parallel.RunTasksInProcessPool(portage_util.GetOverlayEBuilds,
inputs)
for idx, ebuilds in enumerate(result):
overlay_ebuilds[self.overlays[idx]] = ebuilds
self._overlay_ebuilds = overlay_ebuilds
def testInitializeManifestVariablesWithUnprocessedBuild(self):
"""Test InitializeManifestVariables with unprocessed build."""
self.manager = self.BuildManager()
info = manifest_version.VersionInfo(FAKE_VERSION_STRING,
CHROME_BRANCH,
incr_type='branch')
for_build = os.path.join(self.manager.manifest_dir, 'build-name',
self.build_names[0])
m1, m2, _, _ = self._buildManifest()
# Fail 1, pass 2, leave 3,4 unprocessed.
manifest_version.CreateSymlink(
m1,
os.path.join(for_build, 'fail', CHROME_BRANCH,
os.path.basename(m1)))
manifest_version.CreateSymlink(
m1,
os.path.join(for_build, 'pass', CHROME_BRANCH,
os.path.basename(m2)))
self.manager.buildstore.fake_cidb.GetBuildHistory.return_value = None
self.manager.InitializeManifestVariables(info)
self.assertEqual(self.manager.latest_unprocessed, '1.2.5')
self.assertIsNone(self.manager._latest_build)
def testLoadFromString(self):
"""Tests whether we can load from a string."""
info = manifest_version.VersionInfo(FAKE_VERSION_STRING, CHROME_BRANCH)
self.assertEqual(info.VersionString(), FAKE_VERSION_STRING)
