def ListInstalledPackages(board, all_packages=False):
"""Return a list of all packages installed for a particular board."""
# If all_packages is set to True, all packages visible in the build
# chroot are used to generate the licensing file. This is not what you want
# for a release license file, but it's a way to run licensing checks against
# all packages.
# If it's set to False, it will only generate a licensing file that contains
# packages used for a release build (as determined by the dependencies for
# virtual/target-os).
if all_packages:
# The following returns all packages that were part of the build tree
# (many get built or used during the build, but do not get shipped).
# Note that it also contains packages that are in the build as
# defined by build_packages but not part of the image we ship.
equery_cmd = cros_build_lib.GetSysrootToolPath(
cros_build_lib.GetSysroot(board), 'equery')
args = [equery_cmd, 'list', '*']
packages = cros_build_lib.RunCommand(
args, print_cmd=debug, redirect_stdout=True).output.splitlines()
else:
# The following returns all packages that were part of the build tree
# (many get built or used during the build, but do not get shipped).
# Note that it also contains packages that are in the build as
# defined by build_packages but not part of the image we ship.
emerge_cmd = cros_build_lib.GetSysrootToolPath(
cros_build_lib.GetSysroot(board), 'emerge')
args = [
emerge_cmd, '--with-bdeps=y', '--usepkgonly', '--emptytree',
'--pretend', '--color=n', 'virtual/target-os'
]
emerge = cros_build_lib.RunCommand(
args, print_cmd=debug, redirect_stdout=True).output.splitlines()
# Another option which we've decided not to use, is bdeps=n. This outputs
# just the packages we ship, but does not packages that were used to build
# them, including a package like flex which generates a .a that is included
# and shipped in ChromeOS.
# We've decided to credit build packages, even if we're not legally required
# to (it's always nice to do), and that way we get corner case packages like
# flex. This is why we use bdep=y and not bdep=n.
packages = []
# [binary R ] x11-libs/libva-1.1.1 to /build/x86-alex/
pkg_rgx = re.compile(r'\[[^]]+R[^]]+\] (.+) to /build/.*')
# If we match something else without the 'R' like
# [binary U ] chromeos-base/pepper-flash-13.0.0.133-r1 [12.0.0.77-r1]
# this is bad and we should die on this.
pkg_rgx2 = re.compile(r'(\[[^]]+\] .+) to /build/.*')
for line in emerge:
match = pkg_rgx.search(line)
match2 = pkg_rgx2.search(line)
if match:
packages.append(match.group(1))
elif match2:
raise AssertionError(
'Package incorrectly installed, try eclean-%s' % board,
'\n%s' % match2.group(1))
return packages
def _FindEbuildPath(self):
"""Discover the path to a package's associated ebuild.
This method is not valid during the emerge hook process.
Returns:
full path file name of the ebuild file for this package.
Raises:
AssertionError if it can't be discovered for some reason.
"""
equery_cmd = cros_build_lib.GetSysrootToolPath(
cros_build_lib.GetSysroot(self.board), 'equery')
args = [equery_cmd, '-q', '-C', 'which', self.fullnamerev]
try:
path = cros_build_lib.RunCommand(
args, print_cmd=True, redirect_stdout=True).output.strip()
except cros_build_lib.RunCommandError:
path = None
# Path can be false because of an exception, or a command result.
if not path:
raise AssertionError('_FindEbuildPath for %s failed.\n'
'Is your tree clean? Try a rebuild?' %
self.fullnamerev)
logging.debug('%s -> %s', ' '.join(args), path)
if not os.access(path, os.F_OK):
raise AssertionError('Can\'t access %s', path)
return path
def _RunEbuildPhases(self, ebuild_path, phases):
"""Run a list of ebuild phases on an ebuild.
Args:
ebuild_path: exact path of the ebuild file.
phases: list of phases like ['clean', 'fetch'] or ['unpack'].
Returns:
ebuild command output
"""
ebuild_cmd = cros_build_lib.GetSysrootToolPath(
cros_build_lib.GetSysroot(self.board), 'ebuild')
return cros_build_lib.RunCommand([ebuild_cmd, ebuild_path] + phases,
print_cmd=debug,
redirect_stdout=True)
def _ExtractLicenses(self, src_dir, need_copyright_attribution):
"""Scrounge for text licenses in the source of package we'll unpack.
This is only called if we couldn't get usable licenses from the ebuild,
or one of them is BSD/MIT like which forces us to look for a file with
copyright attribution in the source code itself.
First, we have a shortcut where we scan COPYRIGHT_ATTRIBUTION_DIR to see if
we find a license for this package. If so, we use that.
Typically it'll be used if the unpacked source does not have the license
that we're required to display for copyright attribution (in some cases it's
plain absent, in other cases, it could be in a filename we don't look for).
Otherwise, we scan the unpacked source code for what looks like license
files as defined in LICENSE_NAMES_REGEX.
Raises:
AssertionError: on runtime errors
PackageLicenseError: couldn't find copyright attribution file.
"""
license_override = self._GetOverrideLicense()
if license_override:
self.license_text_scanned = [license_override]
return
if not src_dir:
ebuild_path = self._FindEbuildPath()
self._RunEbuildPhases(ebuild_path, ['clean', 'fetch'])
raw_output = self._RunEbuildPhases(ebuild_path, ['unpack'])
output = raw_output.output.splitlines()
# Output is spammy, it looks like this:
# * gc-7.2d.tar.gz RMD160 SHA1 SHA256 size ;-) ... [ ok ]
# * checking gc-7.2d.tar.gz ;-) ... [ ok ]
# * Running stacked hooks for pre_pkg_setup
# * sysroot_build_bin_dir ...
# [ ok ]
# * Running stacked hooks for pre_src_unpack
# * python_multilib_setup ...
# [ ok ]
# >>> Unpacking source...
# >>> Unpacking gc-7.2d.tar.gz to /build/x86-alex/tmp/po/[...]ps-7.2d/work
# >>> Source unpacked in /build/x86-alex/tmp/portage/[...]ops-7.2d/work
# So we only keep the last 2 lines, the others we don't care about.
output = [
line for line in output
if line[0:3] == '>>>' and line != '>>> Unpacking source...'
]
for line in output:
logging.info(line)
portageq_cmd = cros_build_lib.GetSysrootToolPath(
cros_build_lib.GetSysroot(self.board), 'portageq')
args = [portageq_cmd, 'envvar', 'PORTAGE_TMPDIR']
result = cros_build_lib.RunCommand(args,
print_cmd=debug,
redirect_stdout=True)
tmpdir = result.output.splitlines()[0]
# tmpdir gets something like /build/daisy/tmp/
src_dir = os.path.join(tmpdir, 'portage', self.fullnamerev, 'work')
if not os.path.exists(src_dir):
raise AssertionError(
'Unpack of %s didn\'t create %s. Version mismatch' %
(self.fullnamerev, src_dir))
# You may wonder how deep should we go?
# In case of packages with sub-packages, it could be deep.
# Let's just be safe and get everything we can find.
# In the case of libatomic_ops, it's actually required to look deep
# to find the MIT license:
# dev-libs/libatomic_ops-7.2d/work/gc-7.2/libatomic_ops/doc/LICENSING.txt
args = ['find', src_dir, '-type', 'f']
result = cros_build_lib.RunCommand(
args, print_cmd=debug, redirect_stdout=True).output.splitlines()
# Truncate results to look like this: swig-2.0.4/COPYRIGHT
files = [x[len(src_dir):].lstrip('/') for x in result]
license_files = []
for name in files:
# When we scan a source tree managed by git, this can contain license
# files that are not part of the source. Exclude those.
# (e.g. .git/refs/heads/licensing)
if '.git/' in name:
continue
basename = os.path.basename(name)
# Looking for license.* brings up things like license.gpl, and we
# never want a GPL license when looking for copyright attribution,
# so we skip them here. We also skip regexes that can return
# license.py (seen in some code).
if re.search(r'.*GPL.*', basename) or re.search(
r'\.py$', basename):
continue
for regex in LICENSE_NAMES_REGEX:
if re.search(regex, basename, re.IGNORECASE):
license_files.append(name)
break
if not license_files:
if need_copyright_attribution:
logging.error(
"""
%s: unable to find usable license.
Typically this will happen because the ebuild says it's MIT or BSD, but there
was no license file that this script could find to include along with a
copyright attribution (required for BSD/MIT).
If this is Google source, please change
LICENSE="BSD"
to
LICENSE="BSD-Google"
If not, go investigate the unpacked source in %s,
and find which license to assign. Once you found it, you should copy that
license to a file under %s
(or you can modify LICENSE_NAMES_REGEX to pickup a license file that isn't
being scraped currently).""", self.fullnamerev, src_dir,
COPYRIGHT_ATTRIBUTION_DIR)
raise PackageLicenseError()
else:
# We can get called for a license like as-is where it's preferable
# to find a better one in the source, but not fatal if we didn't.
logging.info(
'Was not able to find a better license for %s '
'in %s to replace the more generic one from ebuild',
self.fullnamerev, src_dir)
# Examples of multiple license matches:
# dev-lang/swig-2.0.4-r1: swig-2.0.4/COPYRIGHT swig-2.0.4/LICENSE
# dev-libs/glib-2.32.4-r1: glib-2.32.4/COPYING pkg-config-0.26/COPYING
# dev-libs/libnl-3.2.14: libnl-doc-3.2.14/COPYING libnl-3.2.14/COPYING
# dev-libs/libpcre-8.30-r2: pcre-8.30/LICENCE pcre-8.30/COPYING
# dev-libs/libusb-0.1.12-r6: libusb-0.1.12/COPYING libusb-0.1.12/LICENSE
# dev-libs/pyzy-0.1.0-r1: db/COPYING pyzy-0.1.0/COPYING
# net-misc/strongswan-5.0.2-r4: strongswan-5.0.2/COPYING
# strongswan-5.0.2/LICENSE
# sys-process/procps-3.2.8_p11: debian/copyright procps-3.2.8/COPYING
logging.info('License(s) for %s: %s', self.fullnamerev,
' '.join(license_files))
for license_file in sorted(license_files):
# Joy and pink ponies. Some license_files are encoded as latin1 while
# others are utf-8 and of course you can't know but only guess.
license_path = os.path.join(src_dir, license_file)
license_txt = ReadUnknownEncodedFile(license_path,
'Adding License')
self.license_text_scanned += [
'Scanned Source License %s:\n\n%s' %
(license_file, license_txt)
]