def testMarkAndroidEBuildAsStable(self):
"""Test updating of ebuild."""
self.PatchObject(cros_build_lib, 'RunCommand')
self.PatchObject(portage_util.EBuild, 'GetCrosWorkonVars',
return_value=None)
git_mock = self.PatchObject(git, 'RunGit')
commit_mock = self.PatchObject(portage_util.EBuild, 'CommitChange')
stable_candidate = portage_util.EBuild(self.old2)
unstable = portage_util.EBuild(self.unstable)
android_version = self.new_version
package_dir = self.mock_android_dir
version_atom = cros_mark_android_as_stable.MarkAndroidEBuildAsStable(
stable_candidate, unstable, self.android_package, android_version,
package_dir, self.build_branch, self.arc_bucket_url,
self.targets)
git_mock.assert_has_calls([
mock.call(package_dir, ['add', self.new]),
mock.call(package_dir, ['add', 'Manifest']),
])
commit_mock.assert_called_with(partial_mock.HasString('latest'),
package_dir)
self.assertEqual(
version_atom,
'%s-%s-r1' % (
portage_util.GetFullAndroidPortagePackageName(self.android_package),
self.new_version))
def testGetAndroidRevisionListLink(self):
"""Test generation of revision diff list."""
old_ebuild = portage_util.EBuild(self.old)
old2_ebuild = portage_util.EBuild(self.old2)
link = cros_mark_android_as_stable.GetAndroidRevisionListLink(
self.build_branch, old_ebuild, old2_ebuild)
self.assertEqual(link, ('http://android-build-uber.corp.google.com/'
'repo.html?last_bid=25&bid=50&branch=x'))
def FindAndroidCandidates(package_dir):
"""Return a tuple of Android's unstable ebuild and stable ebuilds.
Args:
package_dir: The path to where the package ebuild is stored.
Returns:
Tuple [unstable_ebuild, stable_ebuilds].
Raises:
Exception: if no unstable ebuild exists for Android.
"""
stable_ebuilds = []
unstable_ebuilds = []
for path in glob.glob(os.path.join(package_dir, '*.ebuild')):
ebuild = portage_util.EBuild(path)
if ebuild.version == '9999':
unstable_ebuilds.append(ebuild)
else:
stable_ebuilds.append(ebuild)
# Apply some sanity checks.
if not unstable_ebuilds:
raise Exception('Missing 9999 ebuild for %s' % package_dir)
if not stable_ebuilds:
logging.warning('Missing stable ebuild for %s' % package_dir)
return portage_util.BestEBuild(unstable_ebuilds), stable_ebuilds
def run_case(content, expected):
with osutils.TempDir() as temp:
ebuild = os.path.join(temp, 'overlay', 'app-misc',
'foo-0.0.1-r1.ebuild')
osutils.WriteFile(ebuild, content, makedirs=True)
self.assertEqual(expected,
portage_util.EBuild(ebuild).has_test)
def main(argv: List[str]) -> None:
cros_build_lib.AssertInsideChroot()
parser = get_parser()
opts = parser.parse_args(argv)
opts.Freeze()
only_files = {Path(f).resolve() for f in opts.file}
git_repo_base = opts.git_repo_base
if git_repo_base:
git_repo_base = Path(opts.git_repo_base)
if not (git_repo_base / '.git').exists():
# This script doesn't strictly care if there's a .git dir there; more of
# a smoke check.
parser.error(f'Given git repo base ({git_repo_base}) has no .git dir')
package_ebuilds = [
portage_util.EBuild(x)
for x in resolve_package_ebuilds(opts.board, opts.package)
]
setup_tidy(opts.board, package_ebuilds)
lints = filter_tidy_lints(
only_files,
git_repo_base,
diags=run_tidy(opts.board, package_ebuilds, opts.keep_lint_dirs,
opts.nonfatal_parse_errors))
osutils.WriteFile(
opts.output,
json.dumps({'tidy_diagnostics': [x.to_dict() for x in lints]}),
atomic=True)
def GetPackageInfo(self, packages, use_all=False, use_workon_only=False):
"""Get information about packages.
Args:
packages: list of package name fragments. These will be mapped to
canonical portage atoms via the same process as
StartWorkingOnPackages().
use_all: True iff instead of the provided package list, we should just
stop working on all currently worked on atoms for the system in
question.
use_workon_only: True iff instead of the provided package list, we should
stop working on all currently worked on atoms that define only a
-9999 ebuild.
Returns:
Returns a list of PackageInfo tuples.
"""
if use_all or use_workon_only:
# You can't use info to find the source code from Chrome, since that
# workflow is different.
ebuilds = self._GetWorkonEbuilds(filter_workon=use_workon_only,
include_chrome=False)
else:
atoms = self._GetCanonicalAtoms(packages)
ebuilds = [self._FindEbuildForPackage(atom) for atom in atoms]
build_root = self._src_root
src_root = os.path.join(build_root, 'src')
manifest = git.ManifestCheckout.Cached(build_root)
ebuild_to_repos = {}
ebuild_to_src_paths = collections.defaultdict(list)
for ebuild in ebuilds:
workon_vars = portage_util.EBuild.GetCrosWorkonVars(
ebuild, portage_util.EbuildToCP(ebuild))
projects = workon_vars.project if workon_vars else []
ebuild_to_repos[ebuild] = projects
ebuild_obj = portage_util.EBuild(ebuild)
if ebuild_obj.is_blacklisted:
# blacklisted ebuilds may have source infos incorrectly defined since
# they are not validated by bots
continue
src_paths = ebuild_obj.GetSourceInfo(src_root, manifest).srcdirs
src_paths = [
os.path.relpath(path, build_root) for path in src_paths
]
ebuild_to_src_paths[ebuild] = src_paths
result = []
for ebuild in ebuilds:
package = portage_util.EbuildToCP(ebuild)
repos = ebuild_to_repos.get(ebuild, [])
src_paths = ebuild_to_src_paths.get(ebuild, [])
result.append(PackageInfo(package, repos, src_paths))
result.sort()
return result
def testMarkAndroidEBuildAsStable(self):
"""Test updating of ebuild."""
self.PatchObject(cros_build_lib, 'RunCommand')
git_mock = self.PatchObject(git, 'RunGit')
commit_mock = self.PatchObject(portage_util.EBuild, 'CommitChange')
stable_candidate = portage_util.EBuild(self.old2)
unstable = portage_util.EBuild(self.unstable)
android_version = self.new_version
package_dir = self.mock_android_dir
version_atom = cros_mark_android_as_stable.MarkAndroidEBuildAsStable(
stable_candidate, unstable, constants.ANDROID_PN, android_version,
package_dir, self.build_branch, self.arc_bucket_url)
git_mock.assert_has_calls([
mock.call(package_dir, ['add', self.new]),
mock.call(package_dir, ['add', 'Manifest']),
])
commit_mock.assert_call(mock.call('latest', package_dir))
self.assertEqual(version_atom,
'%s-%s-r1' % (constants.ANDROID_CP, self.new_version))
def _MockParseWorkonVariables(self, fake_projects, fake_srcpaths,
fake_localnames, fake_subdirs,
fake_ebuild_contents):
"""Mock the necessary calls, call GetSourcePath()."""
def _isdir(path):
"""Mock function for os.path.isdir"""
if any(fake_srcpaths):
if path == os.path.join(self.tempdir, 'src'):
return True
for srcpath in fake_srcpaths:
if srcpath:
if path == os.path.join(self.tempdir, 'src', srcpath):
return True
else:
for localname, subdir in zip(fake_localnames,
fake_subdirs):
if path == os.path.join(self.tempdir, localname,
subdir):
return False
elif path == os.path.join(self.tempdir, 'platform',
localname, subdir):
return True
raise Exception('unhandled path: %s' % path)
def _FindCheckoutFromPath(path):
"""Mock function for manifest.FindCheckoutFromPath"""
for project, localname, subdir in zip(fake_projects,
fake_localnames,
fake_subdirs):
if path == os.path.join(self.tempdir, 'platform', localname,
subdir):
return {'name': project}
return {}
self.PatchObject(os.path, 'isdir', side_effect=_isdir)
self.PatchObject(MANIFEST,
'FindCheckoutFromPath',
side_effect=_FindCheckoutFromPath)
if not fake_srcpaths:
fake_srcpaths = [''] * len(fake_projects)
if not fake_projects:
fake_projects = [''] * len(fake_srcpaths)
# We need 'chromeos-base' here because it controls default _SUBDIR values.
ebuild_path = os.path.join(self.tempdir, 'packages', 'chromeos-base',
'package', 'package-9999.ebuild')
osutils.WriteFile(ebuild_path, fake_ebuild_contents, makedirs=True)
ebuild = portage_util.EBuild(ebuild_path)
return ebuild.GetSourcePath(self.tempdir, MANIFEST)
def DetermineToolchainSourcePaths():
"""Returns a list of all source paths relevant to toolchain packages.
A package is a 'toolchain package' if it is listed as a direct dependency
of virtual/toolchain-packages. This function deliberately does not return
deeper transitive dependencies so that only direct changes to toolchain
packages trigger the expensive full re-compilation required to test toolchain
changes. Eclasses & overlays are not returned as relevant paths for the same
reason.
Returned paths are relative to the root of the project checkout.
Returns:
List[str]: A list of paths considered relevant to toolchain packages.
"""
source_paths = set()
toolchain_pkgs = portage_util.GetFlattenedDepsForPackage(
'virtual/toolchain-packages', depth=1)
toolchain_pkg_ebuilds = portage_util.FindEbuildsForPackages(
toolchain_pkgs, sysroot='/', check=True)
# Include the entire directory containing the toolchain ebuild, as the
# package's FILESDIR and patches also live there.
source_paths.update(
os.path.dirname(ebuild_path)
for ebuild_path in toolchain_pkg_ebuilds.values())
# Source paths which are cros workon source paths.
buildroot = os.path.join(constants.CHROOT_SOURCE_ROOT, 'src')
manifest = git.ManifestCheckout.Cached(buildroot)
for ebuild_path in toolchain_pkg_ebuilds.values():
attrs = portage_util.EBuild.Classify(ebuild_path)
if (not attrs.is_workon or
# Blacklisted ebuild is pinned to a specific git sha1, so change in
# that repo matter to the ebuild.
attrs.is_blacklisted):
continue
ebuild = portage_util.EBuild(ebuild_path)
workon_subtrees = ebuild.GetSourceInfo(buildroot, manifest).subtrees
for path in workon_subtrees:
source_paths.add(path)
return NormalizeSourcePaths(list(source_paths))
def main(argv):
parser = get_parser()
options = parser.parse_args(argv)
options.Freeze()
board = options.board
package = options.package
ebuild_path = portage_util.FindEbuildForBoardPackage(package, board)
if not ebuild_path:
cros_build_lib.Die('Could not find package %s for board %s.', package,
board)
logging.info('Found corresponding ebuild at: %s', ebuild_path)
ebuild = portage_util.EBuild(ebuild_path)
start_date = options.start_date
end_date = options.end_date
if start_date and end_date and start_date > end_date:
cros_build_lib.Die('Start date must be before end date.')
ebuild_commits = get_directory_commits(os.path.dirname(ebuild.ebuild_path),
start_date=start_date,
end_date=end_date)
logging.info('Found %d commits for ebuild.', len(ebuild_commits))
ebuild_uprev_commits = get_uprev_commits(ebuild_commits)
ebuild_uprev_commit_count = len(ebuild_uprev_commits)
logging.info('%d of those commits were uprevs.', ebuild_uprev_commit_count)
if ebuild_uprev_commit_count < 2:
cros_build_lib.Die(
'Alas, you need at least 2 uprevs to compute uprev frequency. '
'Try setting a larger time range?', ebuild_uprev_commit_count)
ebuild_uprev_timestamps = get_commit_timestamps(ebuild_uprev_commits)
average_delta_days = get_average_timestamp_delta_days(
ebuild_uprev_timestamps)
logging.info('Package %s for %s was upreved every %.2f days on average.',
ebuild.package, board, average_delta_days)
def MarkAndroidEBuildAsStable(stable_candidate, unstable_ebuild,
android_package, android_version, package_dir,
build_branch, arc_bucket_url, build_targets):
r"""Uprevs the Android ebuild.
This is the main function that uprevs from a stable candidate
to its new version.
Args:
stable_candidate: ebuild that corresponds to the stable ebuild we are
revving from. If None, builds the a new ebuild given the version
with revision set to 1.
unstable_ebuild: ebuild corresponding to the unstable ebuild for Android.
android_package: android package name.
android_version: The \d+ build id of Android.
package_dir: Path to the android-container package dir.
build_branch: branch of Android builds.
arc_bucket_url: URL of the target ARC build gs bucket.
build_targets: build targets for this particular Android branch.
Returns:
Full portage version atom (including rc's, etc) that was revved.
"""
def IsTheNewEBuildRedundant(new_ebuild, stable_ebuild):
"""Returns True if the new ebuild is redundant.
This is True if there if the current stable ebuild is the exact same copy
of the new one.
"""
if not stable_ebuild:
return False
if stable_candidate.version_no_rev == new_ebuild.version_no_rev:
return filecmp.cmp(new_ebuild.ebuild_path,
stable_ebuild.ebuild_path,
shallow=False)
# Case where we have the last stable candidate with same version just rev.
if stable_candidate and stable_candidate.version_no_rev == android_version:
new_ebuild_path = '%s-r%d.ebuild' % (
stable_candidate.ebuild_path_no_revision,
stable_candidate.current_revision + 1)
else:
pf = '%s-%s-r1' % (android_package, android_version)
new_ebuild_path = os.path.join(package_dir, '%s.ebuild' % pf)
variables = {'BASE_URL': arc_bucket_url}
for build, (target, _) in build_targets.iteritems():
variables[build + '_TARGET'] = '%s-%s' % (build_branch, target)
portage_util.EBuild.MarkAsStable(unstable_ebuild.ebuild_path,
new_ebuild_path,
variables,
make_stable=True)
new_ebuild = portage_util.EBuild(new_ebuild_path)
# Determine whether this is ebuild is redundant.
if IsTheNewEBuildRedundant(new_ebuild, stable_candidate):
msg = 'Previous ebuild with same version found and ebuild is redundant.'
logging.info(msg)
os.unlink(new_ebuild_path)
return None
if stable_candidate:
logging.PrintBuildbotLink(
'Android revisions',
GetAndroidRevisionListLink(build_branch, stable_candidate,
new_ebuild))
git.RunGit(package_dir, ['add', new_ebuild_path])
if stable_candidate and not stable_candidate.IsSticky():
git.RunGit(package_dir, ['rm', stable_candidate.ebuild_path])
# Update ebuild manifest and git add it.
gen_manifest_cmd = ['ebuild', new_ebuild_path, 'manifest', '--force']
cros_build_lib.RunCommand(gen_manifest_cmd, extra_env=None, print_cmd=True)
git.RunGit(package_dir, ['add', 'Manifest'])
portage_util.EBuild.CommitChange(
_GIT_COMMIT_MESSAGE % {
'android_package': android_package,
'android_version': android_version
}, package_dir)
return '%s-%s' % (new_ebuild.package, new_ebuild.version)
def _MakeFakeEbuild(self, fake_ebuild_path, fake_ebuild_content=''): osutils.WriteFile(fake_ebuild_path, fake_ebuild_content, makedirs=True) fake_ebuild = portage_util.EBuild(fake_ebuild_path) return fake_ebuild
def uprev_ebuild_from_pin(package_path, version_no_rev, chroot):
"""Changes the package ebuild's version to match the version pin file.
Args:
package_path: The path of the package relative to the src root. This path
should contain a stable and an unstable ebuild with the same name
as the package.
version_no_rev: The version string to uprev to (excluding revision). The
ebuild's version will be directly set to this number.
chroot (chroot_lib.Chroot): specify a chroot to enter.
Returns:
UprevVersionedPackageResult: The result.
"""
package = os.path.basename(package_path)
package_src_path = os.path.join(constants.SOURCE_ROOT, package_path)
ebuild_paths = list(portage_util.EBuild.List(package_src_path))
stable_ebuild = None
unstable_ebuild = None
for path in ebuild_paths:
ebuild = portage_util.EBuild(path)
if ebuild.is_stable:
stable_ebuild = ebuild
else:
unstable_ebuild = ebuild
if stable_ebuild is None:
raise UprevError('No stable ebuild found for %s' % package)
if unstable_ebuild is None:
raise UprevError('No unstable ebuild found for %s' % package)
if len(ebuild_paths) > 2:
raise UprevError('Found too many ebuilds for %s: '
'expected one stable and one unstable' % package)
# If the new version is the same as the old version, bump the revision number,
# otherwise reset it to 1
if version_no_rev == stable_ebuild.version_no_rev:
version = '%s-r%d' % (version_no_rev,
stable_ebuild.current_revision + 1)
else:
version = version_no_rev + '-r1'
new_ebuild_path = os.path.join(package_path,
'%s-%s.ebuild' % (package, version))
new_ebuild_src_path = os.path.join(constants.SOURCE_ROOT, new_ebuild_path)
manifest_src_path = os.path.join(package_src_path, 'Manifest')
portage_util.EBuild.MarkAsStable(unstable_ebuild.ebuild_path,
new_ebuild_src_path, {})
osutils.SafeUnlink(stable_ebuild.ebuild_path)
try:
# UpdateEbuildManifest runs inside the chroot and therefore needs a
# chroot-relative path.
new_ebuild_chroot_path = os.path.join(constants.CHROOT_SOURCE_ROOT,
new_ebuild_path)
portage_util.UpdateEbuildManifest(new_ebuild_chroot_path,
chroot=chroot)
except cros_build_lib.RunCommandError as e:
raise EbuildManifestError('Unable to update manifest for %s: %s' %
(package, e.stderr))
result = UprevVersionedPackageResult()
result.add_result(
version,
[new_ebuild_src_path, stable_ebuild.ebuild_path, manifest_src_path])
return result
def GenerateSourcePathMapping(packages, sysroot_path, board):
"""Returns a map from each package to the source paths it depends on.
A source path is considered dependency of a package if modifying files in that
path might change the content of the resulting package.
Notes:
1) This method errs on the side of returning unneeded dependent paths.
i.e: for a given package X, some of its dependency source paths may
contain files which doesn't affect the content of X.
On the other hands, any missing dependency source paths for package X is
considered a bug.
2) This only outputs the direct dependency source paths for a given package
and does not takes include the dependency source paths of dependency
packages.
e.g: if package A depends on B (DEPEND=B), then results of computing
dependency source paths of A doesn't include dependency source paths
of B.
Args:
packages: The list of packages CPV names (str)
sysroot_path (str): The path to the sysroot. If the packages are board
agnostic, then this should be '/'.
board (str): The name of the board if packages are dependency of board. If
the packages are board agnostic, then this should be None.
Returns:
Map from each package to the source path (relative to the repo checkout
root, i.e: ~/trunk/ in your cros_sdk) it depends on.
For each source path which is a directory, the string is ended with a
trailing '/'.
"""
results = {}
packages_to_ebuild_paths = portage_util.FindEbuildsForPackages(
packages, sysroot=sysroot_path, check=True)
# Source paths which are the directory of ebuild files.
for package, ebuild_path in packages_to_ebuild_paths.items():
# Include the entire directory that contains the ebuild as the package's
# FILESDIR probably lives there too.
results[package] = [os.path.dirname(ebuild_path)]
# Source paths which are cros workon source paths.
buildroot = os.path.join(constants.CHROOT_SOURCE_ROOT, 'src')
manifest = git.ManifestCheckout.Cached(buildroot)
for package, ebuild_path in packages_to_ebuild_paths.items():
attrs = portage_util.EBuild.Classify(ebuild_path)
if (not attrs.is_workon or
# Blacklisted ebuild is pinned to a specific git sha1, so change in
# that repo matter to the ebuild.
attrs.is_blacklisted):
continue
ebuild = portage_util.EBuild(ebuild_path)
workon_subtrees = ebuild.GetSourceInfo(buildroot, manifest).subtrees
for path in workon_subtrees:
results[package].append(path)
if board:
overlay_directories = portage_util.FindOverlays(
overlay_type='both', board=board)
else:
# If a board is not specified we assume the package is intended for the SDK
# and so we use the overlays for the SDK builder.
overlay_directories = portage_util.FindOverlays(
overlay_type='both', board=constants.CHROOT_BUILDER_BOARD)
eclass_path_cache = {}
for package, ebuild_path in packages_to_ebuild_paths.items():
eclass_paths = GetRelevantEclassesForEbuild(ebuild_path, eclass_path_cache,
overlay_directories)
results[package].extend(eclass_paths)
# Source paths which are the overlay directories for the given board
# (packages are board specific).
# The only parts of the overlay that affect every package are the current
# profile (which lives somewhere in the profiles/ subdir) and a top-level
# make.conf (if it exists).
profile_directories = [
os.path.join(x, 'profiles') for x in overlay_directories
]
make_conf_paths = [os.path.join(x, 'make.conf') for x in overlay_directories]
# These directories *might* affect a build, so we include them for now to
# be safe.
metadata_directories = [
os.path.join(x, 'metadata') for x in overlay_directories
]
scripts_directories = [
os.path.join(x, 'scripts') for x in overlay_directories
]
for package in results:
results[package].extend(profile_directories)
results[package].extend(make_conf_paths)
results[package].extend(metadata_directories)
results[package].extend(scripts_directories)
# The 'crosutils' repo potentially affects the build of every package.
results[package].append(constants.CROSUTILS_DIR)
# chromiumos-overlay specifies default settings for every target in
# chromeos/config and so can potentially affect every board.
for package in results:
results[package].append(
os.path.join(constants.CHROOT_SOURCE_ROOT,
constants.CHROMIUMOS_OVERLAY_DIR, 'chromeos', 'config'))
for p in results:
results[p] = NormalizeSourcePaths(results[p])
return results
