def _spawn(self, args, fd_pipes, **kwargs):
"""
Extend the superclass _spawn method to perform some pre-fork and
post-fork actions.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(
elog_reader_fd,
fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK,
)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(
self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd,
)
fd_pipes[elog_writer_fd] = elog_writer_fd
self.scheduler.add_reader(elog_reader_fd, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
if not self.unmerge:
self._counter = self.vartree.dbapi.counter_tick()
self._dblink = mylink
self._elog_reader_fd = elog_reader_fd
pids = super(MergeProcess, self)._spawn(args, fd_pipes, **kwargs)
os.close(elog_writer_fd)
self._buf = ""
self._elog_keys = set()
# Discard messages which will be collected by the subprocess,
# in order to avoid duplicates (bug #446136).
portage.elog.messages.collect_messages(key=mylink.mycpv)
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
return pids
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Ignore packages that are part of the target sysroot.
if pkg in ('kernel', 'libc'):
continue
# Skip Go compiler from redistributable packages.
# The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
# into it. Due to this, the toolchain cannot be unpacked anywhere
# else and be readily useful. To enable packaging Go, we need to:
# -) Tweak the wrappers/environment to override GOROOT
# automatically based on the unpack location.
# -) Make sure the ELF dependency checking and wrapping logic
# below skips the Go toolchain executables and libraries.
# -) Make sure the packaging process maintains the relative
# timestamps of precompiled standard library packages.
# (see dev-lang/go ebuild for details).
if pkg == 'ex_go':
continue
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom, root=root)[0]
logging.info('packaging %s-%s', atom, ver)
# pylint: disable=E1101
dblink = portage.dblink(cat,
pn + '-' + ver,
myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
elfs.add(obj)
paths.add(obj)
return paths, elfs
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Ignore packages that are part of the target sysroot.
if pkg in ('kernel', 'libc'):
continue
# Skip Go compiler from redistributable packages.
# The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
# into it. Due to this, the toolchain cannot be unpacked anywhere
# else and be readily useful. To enable packaging Go, we need to:
# -) Tweak the wrappers/environment to override GOROOT
# automatically based on the unpack location.
# -) Make sure the ELF dependency checking and wrapping logic
# below skips the Go toolchain executables and libraries.
# -) Make sure the packaging process maintains the relative
# timestamps of precompiled standard library packages.
# (see dev-lang/go ebuild for details).
if pkg == 'ex_go':
continue
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom, root=root)[0]
logging.info('packaging %s-%s', atom, ver)
# pylint: disable=E1101
dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
elfs.add(obj)
paths.add(obj)
return paths, elfs
def dblink(self):
"""Instantiate a L{portage.dbapi.vartree.dblink} object here."""
if self._dblink is None:
self._dblink = portage.dblink(
self.category, "%s-%s" % (self.name, self.fullversion),
self._settings["ROOT"], self._settings)
return self._dblink
def UpdatePackageContents(change_report, package_cp, portage_root=None):
"""
Add newly created files/directors to package contents.
Given an ItemizedChangeReport, add the newly created files and directories
to the CONTENTS of an installed portage package, such that these files are
considered owned by that package.
Arguments:
changereport: ItemizedChangeReport object for the changes to be
made to the package.
package_cp: A string similar to 'chromeos-base/autotest-tests' giving
the package category and name of the package to be altered.
portage_root: Portage root path, corresponding to the board that
we are working on. Defaults to '/'
"""
if portage_root is None:
portage_root = portage.root # pylint: disable-msg=E1101
# Ensure that portage_root ends with trailing slash.
portage_root = os.path.join(portage_root, "")
# Create vartree object corresponding to portage_root
trees = portage.create_trees(portage_root, portage_root)
vartree = trees[portage_root]["vartree"]
# List matching installed packages in cpv format
matching_packages = vartree.dbapi.cp_list(package_cp)
if not matching_packages:
raise ValueError("No matching package for %s in portage_root %s" % (package_cp, portage_root))
if len(matching_packages) > 1:
raise ValueError("Too many matching packages for %s in portage_root " "%s" % (package_cp, portage_root))
# Convert string match to package dblink
package_cpv = matching_packages[0]
package_split = portage_utilities.SplitCPV(package_cpv)
package = portage.dblink(
package_split.category, # pylint: disable-msg=E1101
package_split.pv,
settings=vartree.settings,
vartree=vartree,
)
# Append new contents to package contents dictionary
contents = package.getcontents().copy()
for _, filename in change_report.new_files:
contents.setdefault(filename, (u"obj", "0", "0"))
for _, dirname in change_report.new_directories:
# String trailing slashes if present.
dirname = dirname.rstrip("/")
contents.setdefault(dirname, (u"dir",))
# Write new contents dictionary to file
vartree.dbapi.writeContentsToContentsFile(package, contents)
def dblink(self):
"""Instantiate a L{portage.dbapi.vartree.dblink} object here."""
if self._dblink is None:
self._dblink = portage.dblink(
self.category,
"%s-%s" % (self.name, self.fullversion),
self._settings["ROOT"],
self._settings
)
return self._dblink
def GetPackageAPI(portage_root, package_cp):
"""Gets portage API handles for the given package.
Arguments:
portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy'
package_cp: A string similar to 'chromeos-base/autotest-tests'.
Returns:
Returns (package, vartree) tuple, where
package is of type portage.dbapi.vartree.dblink
vartree is of type portage.dbapi.vartree.vartree
"""
if portage_root is None:
# pylint: disable-msg=E1101
portage_root = portage.root
# Ensure that portage_root ends with trailing slash.
portage_root = os.path.join(portage_root, '')
# Create a vartree object corresponding to portage_root.
trees = portage.create_trees(portage_root, portage_root)
vartree = trees[portage_root]['vartree']
# List the matching installed packages in cpv format.
matching_packages = vartree.dbapi.cp_list(package_cp)
if not matching_packages:
raise PortagePackageAPIError(
'No matching package for %s in portage_root '
'%s' % (package_cp, portage_root))
if len(matching_packages) > 1:
raise PortagePackageAPIError('Too many matching packages for %s in '
'portage_root %s' %
(package_cp, portage_root))
# Convert string match to package dblink.
package_cpv = matching_packages[0]
package_split = portage_utilities.SplitCPV(package_cpv)
# pylint: disable-msg=E1101
package = portage.dblink(package_split.category,
package_split.pv,
settings=vartree.settings,
vartree=vartree)
return package, vartree
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Ignore packages that are part of the target sysroot.
if pkg in ('kernel', 'libc'):
continue
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom)[0]
cros_build_lib.Info('packaging %s-%s', atom, ver)
# pylint: disable=E1101
dblink = portage.dblink(cat,
pn + '-' + ver,
myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
elfs.add(obj)
paths.add(obj)
return paths, elfs
def GetPackageAPI(portage_root, package_cp):
"""Gets portage API handles for the given package.
Args:
portage_root: Root directory of portage tree. Eg '/' or '/build/lumpy'
package_cp: A string similar to 'chromeos-base/autotest-tests'.
Returns:
Returns (package, vartree) tuple, where
package is of type portage.dbapi.vartree.dblink
vartree is of type portage.dbapi.vartree.vartree
"""
if portage_root is None:
# pylint: disable=no-member
portage_root = portage.root
# Ensure that portage_root ends with trailing slash.
portage_root = os.path.join(portage_root, '')
# Create a vartree object corresponding to portage_root.
trees = portage.create_trees(portage_root, portage_root)
vartree = trees[portage_root]['vartree']
# List the matching installed packages in cpv format.
matching_packages = vartree.dbapi.cp_list(package_cp)
if not matching_packages:
raise PortagePackageAPIError('No matching package for %s in portage_root '
'%s' % (package_cp, portage_root))
if len(matching_packages) > 1:
raise PortagePackageAPIError('Too many matching packages for %s in '
'portage_root %s' % (package_cp,
portage_root))
# Convert string match to package dblink.
package_cpv = matching_packages[0]
package_split = portage_util.SplitCPV(package_cpv)
# pylint: disable=no-member
package = portage.dblink(package_split.category,
package_split.pv, settings=vartree.settings,
vartree=vartree)
return package, vartree
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Ignore packages that are part of the target sysroot.
if pkg in ('kernel', 'libc'):
continue
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom)[0]
cros_build_lib.Info('packaging %s-%s', atom, ver)
# pylint: disable=E1101
dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
elfs.add(obj)
paths.add(obj)
return paths, elfs
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(
elog_reader_fd, self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc,
self.infloc,
myebuild=self.myebuild,
mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes,
counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge(). TODO: Share code with ForkProcess.
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.io_add_watch(elog_reader_fd,
self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
parent_pid = os.getpid()
pid = None
try:
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# Use default signal handlers in order to avoid problems
# killing subprocesses as reported in bug #353239.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.locks._close_fds()
# We don't exec, so use close_fds=False
# (see _setup_pipes docstring).
portage.process._setup_pipes(fd_pipes, close_fds=False)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
os._exit(rval)
finally:
if pid == 0 or (pid is None and os.getpid() != parent_pid):
# Call os._exit() from a finally block in order
# to suppress any finally blocks from earlier
# in the call stack (see bug #345289). This
# finally block has to be setup before the fork
# in order to avoid a race condition.
os._exit(1)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge().
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat, self.mypkg, settings=self.settings,
treetype=self.treetype, vartree=self.vartree,
blockers=blockers, scheduler=self.scheduler,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.register(elog_reader_fd,
self._registered_events, self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError(
"fork returned non-integer: %s" % (repr(pid),))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers since the ones inherited
# from the parent process are irrelevant here.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# In this subprocess we want mylink._display_merge() to use
# stdout/stderr directly since they are pipes. This behavior
# is triggered when mylink._scheduler is None.
mylink._scheduler = None
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc, self.infloc,
myebuild=self.myebuild, mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes, counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def _spawn(self, args, fd_pipes, **kwargs):
"""
Fork a subprocess, apply local settings, and call
dblink.merge().
"""
elog_reader_fd, elog_writer_fd = os.pipe()
fcntl.fcntl(elog_reader_fd, fcntl.F_SETFL,
fcntl.fcntl(elog_reader_fd, fcntl.F_GETFL) | os.O_NONBLOCK)
blockers = None
if self.blockers is not None:
# Query blockers in the main process, since closing
# of file descriptors in the subprocess can prevent
# access to open database connections such as that
# used by the sqlite metadata cache module.
blockers = self.blockers()
mylink = portage.dblink(self.mycat,
self.mypkg,
settings=self.settings,
treetype=self.treetype,
vartree=self.vartree,
blockers=blockers,
scheduler=self.scheduler,
pipe=elog_writer_fd)
fd_pipes[elog_writer_fd] = elog_writer_fd
self._elog_reg_id = self.scheduler.register(elog_reader_fd,
self._registered_events,
self._elog_output_handler)
# If a concurrent emerge process tries to install a package
# in the same SLOT as this one at the same time, there is an
# extremely unlikely chance that the COUNTER values will not be
# ordered correctly unless we lock the vdb here.
# FEATURES=parallel-install skips this lock in order to
# improve performance, and the risk is practically negligible.
self._lock_vdb()
counter = None
if not self.unmerge:
counter = self.vartree.dbapi.counter_tick()
pid = os.fork()
if pid != 0:
if not isinstance(pid, int):
raise AssertionError("fork returned non-integer: %s" %
(repr(pid), ))
os.close(elog_writer_fd)
self._elog_reader_fd = elog_reader_fd
self._buf = ""
self._elog_keys = set()
# invalidate relevant vardbapi caches
if self.vartree.dbapi._categories is not None:
self.vartree.dbapi._categories = None
self.vartree.dbapi._pkgs_changed = True
self.vartree.dbapi._clear_pkg_cache(mylink)
portage.process.spawned_pids.append(pid)
return [pid]
os.close(elog_reader_fd)
# TODO: Find out why PyPy 1.8 with close_fds=True triggers
# "[Errno 9] Bad file descriptor" in subprocesses. It could
# be due to garbage collection of file objects that were not
# closed before going out of scope, since PyPy's garbage
# collector does not support the refcounting semantics that
# CPython does.
close_fds = platform.python_implementation() != 'PyPy'
portage.process._setup_pipes(fd_pipes, close_fds=close_fds)
# Use default signal handlers since the ones inherited
# from the parent process are irrelevant here.
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
portage.output.havecolor = self.settings.get('NOCOLOR') \
not in ('yes', 'true')
# In this subprocess we want mylink._display_merge() to use
# stdout/stderr directly since they are pipes. This behavior
# is triggered when mylink._scheduler is None.
mylink._scheduler = None
# Avoid wastful updates of the vdb cache.
self.vartree.dbapi._flush_cache_enabled = False
# In this subprocess we don't want PORTAGE_BACKGROUND to
# suppress stdout/stderr output since they are pipes. We
# also don't want to open PORTAGE_LOG_FILE, since it will
# already be opened by the parent process, so we set the
# "subprocess" value for use in conditional logging code
# involving PORTAGE_LOG_FILE.
if not self.unmerge:
# unmerge phases have separate logs
if self.settings.get("PORTAGE_BACKGROUND") == "1":
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "1"
else:
self.settings["PORTAGE_BACKGROUND_UNMERGE"] = "0"
self.settings.backup_changes("PORTAGE_BACKGROUND_UNMERGE")
self.settings["PORTAGE_BACKGROUND"] = "subprocess"
self.settings.backup_changes("PORTAGE_BACKGROUND")
rval = 1
try:
if self.unmerge:
if not mylink.exists():
rval = os.EX_OK
elif mylink.unmerge(
ldpath_mtimes=self.prev_mtimes) == os.EX_OK:
mylink.lockdb()
try:
mylink.delete()
finally:
mylink.unlockdb()
rval = os.EX_OK
else:
rval = mylink.merge(self.pkgloc,
self.infloc,
myebuild=self.myebuild,
mydbapi=self.mydbapi,
prev_mtimes=self.prev_mtimes,
counter=counter)
except SystemExit:
raise
except:
traceback.print_exc()
finally:
# Call os._exit() from finally block, in order to suppress any
# finally blocks from earlier in the call stack. See bug #345289.
os._exit(rval)
def pkgmerge(mytbz2, myroot, mysettings, mydbapi=None, vartree=None, prev_mtimes=None, blockers=None):
"""will merge a .tbz2 file, returning a list of runtime dependencies
that must be satisfied, or None if there was a merge error. This
code assumes the package exists."""
warnings.warn("portage.pkgmerge() is deprecated", DeprecationWarning, stacklevel=2)
if mydbapi is None:
mydbapi = portage.db[myroot]["bintree"].dbapi
if vartree is None:
vartree = portage.db[myroot]["vartree"]
if mytbz2[-5:] != ".tbz2":
print(_("!!! Not a .tbz2 file"))
return 1
tbz2_lock = None
mycat = None
mypkg = None
did_merge_phase = False
success = False
try:
""" Don't lock the tbz2 file because the filesytem could be readonly or
shared by a cluster."""
# tbz2_lock = portage.locks.lockfile(mytbz2, wantnewlockfile=1)
mypkg = os.path.basename(mytbz2)[:-5]
xptbz2 = portage.xpak.tbz2(mytbz2)
mycat = xptbz2.getfile(_unicode_encode("CATEGORY", encoding=_encodings["repo.content"]))
if not mycat:
writemsg(_("!!! CATEGORY info missing from info chunk, aborting...\n"), noiselevel=-1)
return 1
mycat = _unicode_decode(mycat, encoding=_encodings["repo.content"], errors="replace")
mycat = mycat.strip()
# These are the same directories that would be used at build time.
builddir = os.path.join(mysettings["PORTAGE_TMPDIR"], "portage", mycat, mypkg)
catdir = os.path.dirname(builddir)
pkgloc = os.path.join(builddir, "image")
infloc = os.path.join(builddir, "build-info")
myebuild = os.path.join(infloc, os.path.basename(mytbz2)[:-4] + "ebuild")
portage.util.ensure_dirs(os.path.dirname(catdir), uid=portage_uid, gid=portage_gid, mode=0o70, mask=0)
portage.util.ensure_dirs(catdir, uid=portage_uid, gid=portage_gid, mode=0o70, mask=0)
try:
shutil.rmtree(builddir)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
del e
for mydir in (builddir, pkgloc, infloc):
portage.util.ensure_dirs(mydir, uid=portage_uid, gid=portage_gid, mode=0o755)
writemsg_stdout(_(">>> Extracting info\n"))
xptbz2.unpackinfo(infloc)
mysettings.setcpv(mycat + "/" + mypkg, mydb=mydbapi)
# Store the md5sum in the vdb.
fp = open(_unicode_encode(os.path.join(infloc, "BINPKGMD5")), "w")
fp.write(str(portage.checksum.perform_md5(mytbz2)) + "\n")
fp.close()
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
mysettings["PORTAGE_BINPKG_FILE"] = mytbz2
mysettings.backup_changes("PORTAGE_BINPKG_FILE")
debug = mysettings.get("PORTAGE_DEBUG", "") == "1"
# Eventually we'd like to pass in the saved ebuild env here.
retval = portage.doebuild(
myebuild, "setup", myroot, mysettings, debug=debug, tree="bintree", mydbapi=mydbapi, vartree=vartree
)
if retval != os.EX_OK:
writemsg(_("!!! Setup failed: %s\n") % retval, noiselevel=-1)
return retval
writemsg_stdout(_(">>> Extracting %s\n") % mypkg)
retval = portage.process.spawn_bash(
"bzip2 -dqc -- '%s' | tar -xp -C '%s' -f -" % (mytbz2, pkgloc), env=mysettings.environ()
)
if retval != os.EX_OK:
writemsg(_("!!! Error Extracting '%s'\n") % mytbz2, noiselevel=-1)
return retval
# portage.locks.unlockfile(tbz2_lock)
# tbz2_lock = None
mylink = portage.dblink(
mycat, mypkg, myroot, mysettings, vartree=vartree, treetype="bintree", blockers=blockers
)
retval = mylink.merge(pkgloc, infloc, myroot, myebuild, cleanup=0, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
did_merge_phase = True
success = retval == os.EX_OK
return retval
finally:
mysettings.pop("PORTAGE_BINPKG_FILE", None)
if tbz2_lock:
portage.locks.unlockfile(tbz2_lock)
if True:
if not did_merge_phase:
# The merge phase handles this already. Callers don't know how
# far this function got, so we have to call elog_process() here
# so that it's only called once.
from portage.elog import elog_process
elog_process(mycat + "/" + mypkg, mysettings)
try:
if success:
shutil.rmtree(builddir)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
del e
def get_contents(cpv):
cpv = portage.catsplit(cpv)
return set(portage.dblink(cpv[0], cpv[1], root, settings).getcontents().keys())
def pkgmerge(mytbz2,
myroot,
mysettings,
mydbapi=None,
vartree=None,
prev_mtimes=None,
blockers=None):
"""will merge a .tbz2 file, returning a list of runtime dependencies
that must be satisfied, or None if there was a merge error. This
code assumes the package exists."""
warnings.warn("portage.pkgmerge() is deprecated",
DeprecationWarning,
stacklevel=2)
if mydbapi is None:
mydbapi = portage.db[myroot]["bintree"].dbapi
if vartree is None:
vartree = portage.db[myroot]["vartree"]
if mytbz2[-5:] != ".tbz2":
print(_("!!! Not a .tbz2 file"))
return 1
tbz2_lock = None
mycat = None
mypkg = None
did_merge_phase = False
success = False
try:
""" Don't lock the tbz2 file because the filesytem could be readonly or
shared by a cluster."""
#tbz2_lock = portage.locks.lockfile(mytbz2, wantnewlockfile=1)
mypkg = os.path.basename(mytbz2)[:-5]
xptbz2 = portage.xpak.tbz2(mytbz2)
mycat = xptbz2.getfile(
_unicode_encode("CATEGORY", encoding=_encodings['repo.content']))
if not mycat:
writemsg(
_("!!! CATEGORY info missing from info chunk, aborting...\n"),
noiselevel=-1)
return 1
mycat = _unicode_decode(mycat,
encoding=_encodings['repo.content'],
errors='replace')
mycat = mycat.strip()
# These are the same directories that would be used at build time.
builddir = os.path.join(mysettings["PORTAGE_TMPDIR"], "portage", mycat,
mypkg)
catdir = os.path.dirname(builddir)
pkgloc = os.path.join(builddir, "image")
infloc = os.path.join(builddir, "build-info")
myebuild = os.path.join(infloc,
os.path.basename(mytbz2)[:-4] + "ebuild")
portage.util.ensure_dirs(os.path.dirname(catdir),
uid=portage_uid,
gid=portage_gid,
mode=0o70,
mask=0)
portage.util.ensure_dirs(catdir,
uid=portage_uid,
gid=portage_gid,
mode=0o70,
mask=0)
try:
shutil.rmtree(builddir)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
del e
for mydir in (builddir, pkgloc, infloc):
portage.util.ensure_dirs(mydir,
uid=portage_uid,
gid=portage_gid,
mode=0o755)
writemsg_stdout(_(">>> Extracting info\n"))
xptbz2.unpackinfo(infloc)
mysettings.setcpv(mycat + "/" + mypkg, mydb=mydbapi)
# Store the md5sum in the vdb.
fp = open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5')), 'w')
fp.write(str(portage.checksum.perform_md5(mytbz2)) + "\n")
fp.close()
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
mysettings["PORTAGE_BINPKG_FILE"] = mytbz2
mysettings.backup_changes("PORTAGE_BINPKG_FILE")
debug = mysettings.get("PORTAGE_DEBUG", "") == "1"
# Eventually we'd like to pass in the saved ebuild env here.
retval = portage.doebuild(myebuild,
"setup",
myroot,
mysettings,
debug=debug,
tree="bintree",
mydbapi=mydbapi,
vartree=vartree)
if retval != os.EX_OK:
writemsg(_("!!! Setup failed: %s\n") % retval, noiselevel=-1)
return retval
writemsg_stdout(_(">>> Extracting %s\n") % mypkg)
retval = portage.process.spawn_bash(
"bzip2 -dqc -- '%s' | tar -xp -C '%s' -f -" % (mytbz2, pkgloc),
env=mysettings.environ())
if retval != os.EX_OK:
writemsg(_("!!! Error Extracting '%s'\n") % mytbz2, noiselevel=-1)
return retval
#portage.locks.unlockfile(tbz2_lock)
#tbz2_lock = None
mylink = portage.dblink(mycat,
mypkg,
myroot,
mysettings,
vartree=vartree,
treetype="bintree",
blockers=blockers)
retval = mylink.merge(pkgloc,
infloc,
myroot,
myebuild,
cleanup=0,
mydbapi=mydbapi,
prev_mtimes=prev_mtimes)
did_merge_phase = True
success = retval == os.EX_OK
return retval
finally:
mysettings.pop("PORTAGE_BINPKG_FILE", None)
if tbz2_lock:
portage.locks.unlockfile(tbz2_lock)
if True:
if not did_merge_phase:
# The merge phase handles this already. Callers don't know how
# far this function got, so we have to call elog_process() here
# so that it's only called once.
from portage.elog import elog_process
elog_process(mycat + "/" + mypkg, mysettings)
try:
if success:
shutil.rmtree(builddir)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT:
raise
del e
def get_contents(cpv):
cpv = portage.catsplit(cpv)
return set(
portage.dblink(cpv[0], cpv[1], root, settings).getcontents().keys())
def get_contents(self, c, p, v):
dbl = portage.dblink(c, '%s-%s' % (p, v), self._settings['ROOT'],
self._settings)
return dbl.getcontents()
def _GetFilesForTarget(target, root='/'):
"""Locate all the files to package for |target|
This does not cover ELF dependencies.
Args:
target: The toolchain target name
root: The root path to pull all packages from
Returns:
A tuple of a set of all packable paths, and a set of all paths which
are also native ELFs
"""
paths = set()
elfs = set()
# Find all the files owned by the packages for this target.
for pkg in GetTargetPackages(target):
# Skip Go compiler from redistributable packages.
# The "go" executable has GOROOT=/usr/lib/go/${CTARGET} hardcoded
# into it. Due to this, the toolchain cannot be unpacked anywhere
# else and be readily useful. To enable packaging Go, we need to:
# -) Tweak the wrappers/environment to override GOROOT
# automatically based on the unpack location.
# -) Make sure the ELF dependency checking and wrapping logic
# below skips the Go toolchain executables and libraries.
# -) Make sure the packaging process maintains the relative
# timestamps of precompiled standard library packages.
# (see dev-lang/go ebuild for details).
if pkg == 'ex_go':
continue
# Use armv7a-cros-linux-gnueabi/compiler-rt for
# armv7a-cros-linux-gnueabihf/compiler-rt.
# Currently the armv7a-cros-linux-gnueabi is actually
# the same as armv7a-cros-linux-gnueabihf with different names.
# Because of that, for compiler-rt, it generates the same binary in
# the same location. To avoid the installation conflict, we do not
# install anything for 'armv7a-cros-linux-gnueabihf'. This would cause
# problem if other people try to use standalone armv7a-cros-linux-gnueabihf
# toolchain.
if 'compiler-rt' in pkg and 'armv7a-cros-linux-gnueabi' in target:
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom, root=root)[0]
dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
if not contents:
if 'hf' in target:
new_target = 'armv7a-cros-linux-gnueabi'
else:
new_target = 'armv7a-cros-linux-gnueabihf'
atom = GetPortagePackage(new_target, pkg)
else:
atom = GetPortagePackage(target, pkg)
cat, pn = atom.split('/')
ver = GetInstalledPackageVersions(atom, root=root)[0]
logging.info('packaging %s-%s', atom, ver)
dblink = portage.dblink(cat, pn + '-' + ver, myroot=root,
settings=portage.settings)
contents = dblink.getcontents()
for obj in contents:
ptype = contents[obj][0]
if not IsPathPackagable(ptype, obj):
continue
if ptype == 'obj':
# For native ELFs, we need to pull in their dependencies too.
if FileIsCrosSdkElf(obj):
logging.debug('Adding ELF %s', obj)
elfs.add(obj)
logging.debug('Adding path %s', obj)
paths.add(obj)
return paths, elfs
