def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_task(
AsyncTaskFuture(future=self._builddir_lock.async_lock()),
self._start_unmerge,
)
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
if retval == os.EX_OK:
self._current_task = None
self.returncode = os.EX_OK
self._async_wait()
else:
# For pretend mode, the convention it to execute
# pkg_nofetch and return a successful exitcode.
self._start_task(
SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=self.pkg.root_config.trees[
self._tree].dbapi,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings), self._default_final_exit)
return
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath, "prerm",
settings=self.settings, db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(
settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts, "unmerge", [self.pkg.cpv], clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,))
unmerge_task = MergeProcess(
mycat=cat, mypkg=pf, settings=self.settings,
treetype="vartree", vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler, background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self.wait()
return
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
SpawnProcess._start(self)
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
else:
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
class Binpkg(CompositeTask):
__slots__ = ("find_blockers",
"ldpath_mtimes", "logger", "opts",
"pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \
("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg",
"_image_dir", "_infloc", "_pkg_path", "_tree", "_verify")
def _writemsg_level(self, msg, level=0, noiselevel=0):
if not self.background:
portage.util.writemsg_level(msg,
level=level, noiselevel=noiselevel)
log_path = self.settings.get("PORTAGE_LOG_FILE")
if log_path is not None:
f = codecs.open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
try:
f.write(msg)
finally:
f.close()
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
pkg = self.pkg
pkg_count = self.pkg_count
if not (self.opts.pretend or self.opts.fetchonly):
self._build_dir.lock()
# If necessary, discard old log so that we don't
# append to it.
self._build_dir.clean_log()
# Initialze PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
fetcher = BinpkgFetcher(background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
pretend=self.opts.pretend, scheduler=self.scheduler)
pkg_path = fetcher.pkg_path
self._pkg_path = pkg_path
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Fetch" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
self._start_task(fetcher, self._fetcher_exit)
return
self._fetcher_exit(fetcher)
def _fetcher_exit(self, fetcher):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
if fetcher.returncode is not None:
self._fetched_pkg = True
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.pretend:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
verifier = None
if self._verify:
logfile = None
if self.background:
logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(background=self.background,
logfile=logfile, pkg=self.pkg)
self._start_task(verifier, self._verifier_exit)
return
self._verifier_exit(verifier)
def _verifier_exit(self, verifier):
if verifier is not None and \
self._default_exit(verifier) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
pkg_path = self._pkg_path
if self._fetched_pkg:
self._bintree.inject(pkg.cpv, filename=pkg_path)
logfile = self.settings.get("PORTAGE_LOG_FILE")
if logfile is not None and os.path.isfile(logfile):
# Remove fetch log after successful fetch.
try:
os.unlink(logfile)
except OSError:
pass
if self.opts.fetchonly:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
msg = " === (%s of %s) Merging Binary (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Merge Binary" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
phase = "clean"
settings = self.settings
ebuild_phase = EbuildPhase(background=self.background,
pkg=pkg, phase=phase, scheduler=self.scheduler,
settings=settings, tree=self._tree)
self._start_task(ebuild_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
dir_path = self._build_dir.dir_path
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid,
gid=portage.data.portage_gid, mode=dir_mode)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
pkg_xpak = portage.xpak.tbz2(self._pkg_path)
check_missing_metadata = ("CATEGORY", "PF")
missing_metadata = set()
for k in check_missing_metadata:
v = pkg_xpak.getfile(_unicode_encode(k,
encoding=_encodings['repo.content']))
if not v:
missing_metadata.add(k)
pkg_xpak.unpackinfo(infloc)
for k in missing_metadata:
if k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = codecs.open(_unicode_encode(os.path.join(infloc, k),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='replace')
try:
f.write(v + "\n")
finally:
f.close()
# Store the md5sum in the vdb.
f = codecs.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='strict')
try:
f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n")
finally:
f.close()
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
settings = self.settings
settings.setcpv(self.pkg)
settings["PORTAGE_BINPKG_FILE"] = pkg_path
settings.backup_changes("PORTAGE_BINPKG_FILE")
phase = "setup"
setup_phase = EbuildPhase(background=self.background,
pkg=self.pkg, phase=phase, scheduler=self.scheduler,
settings=settings, tree=self._tree)
setup_phase.addExitListener(self._setup_exit)
self._current_task = setup_phase
self.scheduler.scheduleSetup(setup_phase)
def _setup_exit(self, setup_phase):
if self._default_exit(setup_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
extractor = BinpkgExtractorAsync(background=self.background,
image_dir=self._image_dir,
pkg=self.pkg, pkg_path=self._pkg_path,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
self._start_task(extractor, self._extractor_exit)
def _extractor_exit(self, extractor):
if self._final_exit(extractor) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self.wait()
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def install(self):
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
settings = self.settings
settings["PORTAGE_BINPKG_FILE"] = self._pkg_path
settings.backup_changes("PORTAGE_BINPKG_FILE")
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes, logger=self.logger,
pkg=self.pkg, pkg_count=self.pkg_count,
pkg_path=self._pkg_path, scheduler=self.scheduler,
settings=settings, tree=self._tree, world_atom=self.world_atom)
try:
retval = merge.execute()
finally:
settings.pop("PORTAGE_BINPKG_FILE", None)
self._unlock_builddir()
return retval
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path,
'setup',
settings=self.settings,
db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
logger = self.logger
opts = self.opts
pkg = self.pkg
settings = self.settings
world_atom = self.world_atom
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
ebuild_path = portdb.findname(pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
fetcher = EbuildFetcher(config_pool=self.config_pool,
fetchall=opts.fetch_all_uri,
fetchonly=opts.fetchonly,
background=self.background,
pkg=pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetch_exit)
def _fetch_exit(self, fetcher):
opts = self.opts
pkg = self.pkg
fetch_failed = False
if opts.fetchonly:
fetch_failed = self._final_exit(fetcher) != os.EX_OK
else:
fetch_failed = self._default_exit(fetcher) != os.EX_OK
if fetch_failed and fetcher.logfile is not None and \
os.path.exists(fetcher.logfile):
self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile
if fetch_failed or opts.fetchonly:
self.wait()
return
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning is triggered before the setup
# phase, in portage.doebuild().
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
opts = self.opts
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
log_path = self.settings.get("PORTAGE_LOG_FILE")
if log_path is not None:
log_file = codecs.open(_unicode_encode(
log_path, encoding=_encodings['fs'], errors='strict'),
mode='a',
encoding=_encodings['content'],
errors='replace')
try:
log_file.write(msg)
finally:
log_file.close()
if not self.background:
portage.writemsg_stdout(msg, noiselevel=-1)
packager = EbuildBinpkg(background=self.background,
pkg=self.pkg,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(packager, self._buildpkg_exit)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase],
phase=phase,
pkg=self.pkg,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
pkg=self.pkg,
phase=phase,
scheduler=self.scheduler,
settings=self.settings,
tree=self._tree)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def install(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
find_blockers = self.find_blockers
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes,
logger=logger,
pkg=pkg,
pkg_count=pkg_count,
pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings,
tree=tree,
world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
try:
rval = merge.execute()
finally:
self._unlock_builddir()
return rval
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
if self.prefetch and \
self._prefetch_size_ok(portdb, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self.wait()
return
# In prefetch mode, logging goes to emerge-fetch.log and the builddir
# should not be touched since otherwise it could interfere with
# another instance of the same cpv concurrently being built for a
# different $ROOT (currently, builds only cooperate with prefetchers
# that are spawned for the same $ROOT).
if not self.prefetch:
self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings)
self._build_dir.lock()
self._build_dir.clean_log()
cleanup=1
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, cleanup)
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
phase = "fetch"
if self.fetchall:
phase = "fetchall"
# If any incremental variables have been overridden
# via the environment, those values need to be passed
# along here so that they are correctly considered by
# the config instance in the subproccess.
fetch_env = os.environ.copy()
fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT']
nocolor = settings.get("NOCOLOR")
if nocolor is not None:
fetch_env["NOCOLOR"] = nocolor
fetch_env["PORTAGE_NICENESS"] = "0"
if self.prefetch:
fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
ebuild_binary = os.path.join(
settings["PORTAGE_BIN_PATH"], "ebuild")
fetch_args = [ebuild_binary, ebuild_path, phase]
debug = settings.get("PORTAGE_DEBUG") == "1"
if debug:
fetch_args.append("--debug")
if not self.background and nocolor not in ('yes', 'true'):
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
fetch_args.append('--color=y')
self.args = fetch_args
self.env = fetch_env
if self._build_dir is None:
# Free settings now since we only have a local reference.
self.config_pool.deallocate(settings)
SpawnProcess._start(self)
class Binpkg(CompositeTask):
__slots__ = ("find_blockers",
"ldpath_mtimes", "logger", "opts",
"pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \
("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg",
"_image_dir", "_infloc", "_pkg_path", "_tree", "_verify")
def _writemsg_level(self, msg, level=0, noiselevel=0):
self.scheduler.output(msg, level=level, noiselevel=noiselevel,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
pkg = self.pkg
pkg_count = self.pkg_count
if not (self.opts.pretend or self.opts.fetchonly):
self._build_dir.lock()
# Initialize PORTAGE_LOG_FILE (clean_log won't work without it).
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
# If necessary, discard old log so that we don't
# append to it.
self._build_dir.clean_log()
fetcher = BinpkgFetcher(background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
pretend=self.opts.pretend, scheduler=self.scheduler)
pkg_path = fetcher.pkg_path
self._pkg_path = pkg_path
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Fetch" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
self._start_task(fetcher, self._fetcher_exit)
return
self._fetcher_exit(fetcher)
def _fetcher_exit(self, fetcher):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
if fetcher.returncode is not None:
self._fetched_pkg = True
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.pretend:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
verifier = None
if self._verify:
logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(background=self.background,
logfile=logfile, pkg=self.pkg, scheduler=self.scheduler)
self._start_task(verifier, self._verifier_exit)
return
self._verifier_exit(verifier)
def _verifier_exit(self, verifier):
if verifier is not None and \
self._default_exit(verifier) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
pkg_path = self._pkg_path
if self._fetched_pkg:
self._bintree.inject(pkg.cpv, filename=pkg_path)
logfile = self.settings.get("PORTAGE_LOG_FILE")
if logfile is not None and os.path.isfile(logfile):
# Remove fetch log after successful fetch.
try:
os.unlink(logfile)
except OSError:
pass
if self.opts.fetchonly:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
msg = " === (%s of %s) Merging Binary (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Merge Binary" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
phase = "clean"
settings = self.settings
ebuild_phase = EbuildPhase(background=self.background,
phase=phase, scheduler=self.scheduler,
settings=settings)
self._start_task(ebuild_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
dir_path = self._build_dir.dir_path
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid,
gid=portage.data.portage_gid, mode=dir_mode)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
pkg_xpak = portage.xpak.tbz2(self._pkg_path)
check_missing_metadata = ("CATEGORY", "PF")
missing_metadata = set()
for k in check_missing_metadata:
v = pkg_xpak.getfile(_unicode_encode(k,
encoding=_encodings['repo.content']))
if not v:
missing_metadata.add(k)
pkg_xpak.unpackinfo(infloc)
for k in missing_metadata:
if k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = codecs.open(_unicode_encode(os.path.join(infloc, k),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='replace')
try:
f.write(v + "\n")
finally:
f.close()
# Store the md5sum in the vdb.
f = codecs.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='strict')
try:
f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n")
finally:
f.close()
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
self._start_task(env_extractor, self._env_extractor_exit)
def _env_extractor_exit(self, env_extractor):
if self._default_exit(env_extractor) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
settings = self.settings
settings.setcpv(self.pkg)
settings["PORTAGE_BINPKG_FILE"] = self._pkg_path
settings.backup_changes("PORTAGE_BINPKG_FILE")
setup_phase = EbuildPhase(background=self.background,
phase="setup", scheduler=self.scheduler,
settings=settings)
setup_phase.addExitListener(self._setup_exit)
self._current_task = setup_phase
self.scheduler.scheduleSetup(setup_phase)
def _setup_exit(self, setup_phase):
if self._default_exit(setup_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
extractor = BinpkgExtractorAsync(background=self.background,
image_dir=self._image_dir,
pkg=self.pkg, pkg_path=self._pkg_path,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
self._start_task(extractor, self._extractor_exit)
def _extractor_exit(self, extractor):
if self._final_exit(extractor) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self.wait()
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def install(self):
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
settings = self.settings
settings["PORTAGE_BINPKG_FILE"] = self._pkg_path
settings.backup_changes("PORTAGE_BINPKG_FILE")
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes, logger=self.logger,
pkg=self.pkg, pkg_count=self.pkg_count,
pkg_path=self._pkg_path, scheduler=self.scheduler,
settings=settings, tree=self._tree, world_atom=self.world_atom)
try:
retval = merge.execute()
finally:
settings.pop("PORTAGE_BINPKG_FILE", None)
self._unlock_builddir()
return retval
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
if not self.opts.fetchonly:
rval = _check_temp_dir(self.settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
# First get the SRC_URI metadata (it's not cached in self.pkg.metadata
# because some packages have an extremely large SRC_URI value).
self._start_task(
AsyncTaskFuture(
future=self.pkg.root_config.trees["porttree"].dbapi.\
async_aux_get(self.pkg.cpv, ["SRC_URI"], myrepo=self.pkg.repo,
loop=self.scheduler)),
self._start_with_metadata)
def _start_with_metadata(self, aux_get_task):
self._assert_current(aux_get_task)
if aux_get_task.cancelled:
self._default_final_exit(aux_get_task)
return
pkg = self.pkg
settings = self.settings
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result()
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path,
'setup',
settings=self.settings,
db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching files in the background.',
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features and \
'digest' not in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
if retval == os.EX_OK:
self._current_task = None
self.returncode = os.EX_OK
self._async_wait()
else:
# For pretend mode, the convention it to execute
# pkg_nofetch and return a successful exitcode.
self._start_task(
SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=self.pkg.root_config.trees[
self._tree].dbapi,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings), self._default_final_exit)
return
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _start_pre_clean(self, lock_task):
self._assert_current(lock_task)
if lock_task.cancelled:
self._default_final_exit(lock_task)
return
lock_task.future.result()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
self._final_exit(fetcher)
if self.returncode != os.EX_OK:
self.returncode = None
portdb = self.pkg.root_config.trees[self._tree].dbapi
self._start_task(
SpawnNofetchWithoutBuilddir(background=self.background,
portdb=portdb,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings),
self._nofetch_without_builddir_exit)
return
self.wait()
def _nofetch_without_builddir_exit(self, nofetch):
self._final_exit(nofetch)
self.returncode = 1
self.wait()
def _pre_clean_exit(self, pre_clean_phase):
if self._default_exit(pre_clean_phase) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(
AsyncTaskFuture(
future=fetcher.async_already_fetched(self.settings)),
functools.partial(self._start_fetch, fetcher))
def _start_fetch(self, fetcher, already_fetched_task):
self._assert_current(already_fetched_task)
if already_fetched_task.cancelled:
self._default_final_exit(already_fetched_task)
return
try:
already_fetched = already_fetched_task.future.result()
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
fetcher._eerror(msg_lines)
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._async_unlock_builddir(returncode=1)
return
if already_fetched:
# This case is optimized to skip the fetch queue.
fetcher = None
self._fetch_exit(fetcher)
return
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetch_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
def _fetch_exit(self, fetcher):
if fetcher is not None and \
self._default_exit(fetcher) != os.EX_OK:
self._fetch_failed()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
"buildpkg" not in features and \
opts.buildpkg != 'n'
if ("buildpkg" in features or self._issyspkg) \
and not self.opts.buildpkg_exclude.findAtomForPackage(pkg):
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
def _fetch_failed(self):
# We only call the pkg_nofetch phase if either RESTRICT=fetch
# is set or the package has explicitly overridden the default
# pkg_nofetch implementation. This allows specialized messages
# to be displayed for problematic packages even though they do
# not set RESTRICT=fetch (bug #336499).
if 'fetch' not in self.pkg.restrict and \
'nofetch' not in self.pkg.defined_phases:
self._async_unlock_builddir(returncode=self.returncode)
return
self.returncode = None
nofetch_phase = EbuildPhase(background=self.background,
phase='nofetch',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(nofetch_phase, self._nofetch_exit)
def _nofetch_exit(self, nofetch_phase):
self._final_exit(nofetch_phase)
self._async_unlock_builddir(returncode=1)
def _async_unlock_builddir(self, returncode=None):
"""
Release the lock asynchronously, and if a returncode parameter
is given then set self.returncode and notify exit listeners.
"""
if returncode is not None:
# The returncode will be set after unlock is complete.
self.returncode = None
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._start_task(
AsyncTaskFuture(future=self._build_dir.async_unlock()),
functools.partial(self._unlock_builddir_exit,
returncode=returncode))
def _unlock_builddir_exit(self, unlock_task, returncode=None):
self._assert_current(unlock_task)
if unlock_task.cancelled and returncode is not None:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.cancelled() or unlock_task.future.result()
if returncode is not None:
self.returncode = returncode
self._async_wait()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))
binpkg_tasks = TaskSequence()
requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT",
"tar").split()
for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS:
if pkg_fmt in requested_binpkg_formats:
if pkg_fmt == "rpm":
binpkg_tasks.add(
EbuildPhase(background=self.background,
phase="rpm",
scheduler=self.scheduler,
settings=self.settings))
else:
task = EbuildBinpkg(background=self.background,
pkg=self.pkg,
scheduler=self.scheduler,
settings=self.settings)
binpkg_tasks.add(task)
# Guarantee that _record_binpkg_info is called
# immediately after EbuildBinpkg. Note that
# task.addExitListener does not provide the
# necessary guarantee (see bug 578204).
binpkg_tasks.add(
self._RecordBinpkgInfo(ebuild_binpkg=task,
ebuild_build=self))
if binpkg_tasks:
self._start_task(binpkg_tasks, self._buildpkg_exit)
return
self._final_exit(build)
self.wait()
class _RecordBinpkgInfo(AsynchronousTask):
"""
This class wraps the EbuildBuild _record_binpkg_info method
with an AsynchronousTask interface, so that it can be
scheduled as a member of a TaskSequence.
"""
__slots__ = (
'ebuild_binpkg',
'ebuild_build',
)
def _start(self):
self.ebuild_build._record_binpkg_info(self.ebuild_binpkg)
AsynchronousTask._start(self)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase],
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _record_binpkg_info(self, task):
if task.returncode != os.EX_OK:
return
# Save info about the created binary package, so that
# identifying information can be passed to the install
# task, to be recorded in the installed package database.
pkg = task.get_binpkg_info()
infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info")
info = {
"BINPKGMD5": "%s\n" % pkg._metadata["MD5"],
}
if pkg.build_id is not None:
info["BUILD_ID"] = "%s\n" % pkg.build_id
for k, v in info.items():
with io.open(_unicode_encode(os.path.join(infoloc, k),
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='strict') as f:
f.write(v)
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._async_unlock_builddir(returncode=self.returncode)
else:
self.wait()
def create_install_task(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes,
logger=logger,
pkg=pkg,
pkg_count=pkg_count,
pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings,
tree=tree,
world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
return task
def _install_exit(self, task):
"""
@returns: Future, result is the returncode from an
EbuildBuildDir.async_unlock() task
"""
self._async_unlock_builddir()
if self._current_task is None:
result = self.scheduler.create_future()
self.scheduler.call_soon(result.set_result, os.EX_OK)
else:
result = self._current_task.async_wait()
return result
class EbuildFetcher(SpawnProcess):
__slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \
("_build_dir",)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
if self.prefetch and \
self._prefetch_size_ok(portdb, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self.wait()
return
# In prefetch mode, logging goes to emerge-fetch.log and the builddir
# should not be touched since otherwise it could interfere with
# another instance of the same cpv concurrently being built for a
# different $ROOT (currently, builds only cooperate with prefetchers
# that are spawned for the same $ROOT).
if not self.prefetch:
self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings)
self._build_dir.lock()
self._build_dir.clean_log()
portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, 0)
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
phase = "fetch"
if self.fetchall:
phase = "fetchall"
# If any incremental variables have been overridden
# via the environment, those values need to be passed
# along here so that they are correctly considered by
# the config instance in the subproccess.
fetch_env = os.environ.copy()
fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT']
nocolor = settings.get("NOCOLOR")
if nocolor is not None:
fetch_env["NOCOLOR"] = nocolor
fetch_env["PORTAGE_NICENESS"] = "0"
if self.prefetch:
fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
ebuild_binary = os.path.join(
settings["PORTAGE_BIN_PATH"], "ebuild")
fetch_args = [ebuild_binary, ebuild_path, phase]
debug = settings.get("PORTAGE_DEBUG") == "1"
if debug:
fetch_args.append("--debug")
if not self.background and nocolor not in ('yes', 'true'):
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
fetch_args.append('--color=y')
self.args = fetch_args
self.env = fetch_env
if self._build_dir is None:
# Free settings now since we only have a local reference.
self.config_pool.deallocate(settings)
SpawnProcess._start(self)
def _prefetch_size_ok(self, portdb, settings, ebuild_path):
pkgdir = os.path.dirname(ebuild_path)
mytree = os.path.dirname(os.path.dirname(pkgdir))
distdir = settings["DISTDIR"]
use = None
if not self.fetchall:
use = self.pkg.use.enabled
try:
uri_map = portdb.getFetchMap(self.pkg.cpv,
useflags=use, mytree=mytree)
except portage.exception.InvalidDependString as e:
return False
sizes = {}
for filename in uri_map:
# Use stat rather than lstat since portage.fetch() creates
# symlinks when PORTAGE_RO_DISTDIRS is used.
try:
st = os.stat(os.path.join(distdir, filename))
except OSError:
return False
if st.st_size == 0:
return False
sizes[filename] = st.st_size
digests = portage.Manifest(pkgdir, distdir).getTypeDigests("DIST")
for filename, actual_size in sizes.items():
size = digests.get(filename, {}).get('size')
if size is None:
continue
if size != actual_size:
return False
# All files are present and sizes are ok. In this case the normal
# fetch code will be skipped, so we need to generate equivalent
# output here.
if self.logfile is not None:
f = codecs.open(_unicode_encode(self.logfile,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
for filename in uri_map:
f.write((' * %s size ;-) ...' % \
filename).ljust(73) + '[ ok ]\n')
f.close()
return True
def _pipe(self, fd_pipes):
"""When appropriate, use a pty so that fetcher progress bars,
like wget has, will work properly."""
if self.background or not sys.stdout.isatty():
# When the output only goes to a log file,
# there's no point in creating a pty.
return os.pipe()
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
# Collect elog messages that might have been
# created by the pkg_nofetch phase.
if self._build_dir is not None:
# Skip elog messages for prefetch, in order to avoid duplicates.
if not self.prefetch and self.returncode != os.EX_OK:
elog_out = None
if self.logfile is not None:
if self.background:
elog_out = codecs.open(_unicode_encode(self.logfile,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
msg = "Fetch failed for '%s'" % (self.pkg.cpv,)
if self.logfile is not None:
msg += ", Log file:"
eerror(msg, phase="unpack", key=self.pkg.cpv, out=elog_out)
if self.logfile is not None:
eerror(" '%s'" % (self.logfile,),
phase="unpack", key=self.pkg.cpv, out=elog_out)
if elog_out is not None:
elog_out.close()
if not self.prefetch:
portage.elog.elog_process(self.pkg.cpv, self._build_dir.settings)
features = self._build_dir.settings.features
if self.returncode == os.EX_OK:
self._build_dir.clean_log()
self._build_dir.unlock()
self.config_pool.deallocate(self._build_dir.settings)
self._build_dir = None
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
pkg = self.pkg
settings = self.settings
if not self.opts.fetchonly:
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path, 'setup',
settings=self.settings, db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log')
msg = (
'Fetching files in the background.',
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features and \
'digest' not in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(
fetch_all=opts.fetch_all_uri,
pkg=pkg, pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
else:
fetcher = EbuildFetcher(
config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean', scheduler=self.scheduler, settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
self._final_exit(fetcher)
if self.returncode != os.EX_OK:
self.returncode = None
portdb = self.pkg.root_config.trees[self._tree].dbapi
self._start_task(SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=portdb,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings),
self._nofetch_without_builddir_exit)
return
self.wait()
def _nofetch_without_builddir_exit(self, nofetch):
self._final_exit(nofetch)
self.returncode = 1
self.wait()
def _pre_clean_exit(self, pre_clean_phase):
if self._default_exit(pre_clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg, scheduler=self.scheduler)
try:
already_fetched = fetcher.already_fetched(self.settings)
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
fetcher._eerror(msg_lines)
portage.elog.elog_process(self.pkg.cpv, self.settings)
self.returncode = 1
self._current_task = None
self._unlock_builddir()
self.wait()
return
if already_fetched:
# This case is optimized to skip the fetch queue.
fetcher = None
self._fetch_exit(fetcher)
return
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetch_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
def _fetch_exit(self, fetcher):
if fetcher is not None and \
self._default_exit(fetcher) != os.EX_OK:
self._fetch_failed()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
"buildpkg" not in features and \
opts.buildpkg != 'n'
if ("buildpkg" in features or self._issyspkg) \
and not self.opts.buildpkg_exclude.findAtomForPackage(pkg):
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background, pkg=pkg,
scheduler=scheduler, settings=settings)
self._start_task(build, self._build_exit)
def _fetch_failed(self):
# We only call the pkg_nofetch phase if either RESTRICT=fetch
# is set or the package has explicitly overridden the default
# pkg_nofetch implementation. This allows specialized messages
# to be displayed for problematic packages even though they do
# not set RESTRICT=fetch (bug #336499).
if 'fetch' not in self.pkg.restrict and \
'nofetch' not in self.pkg.defined_phases:
self._unlock_builddir()
self.wait()
return
self.returncode = None
nofetch_phase = EbuildPhase(background=self.background,
phase='nofetch', scheduler=self.scheduler, settings=self.settings)
self._start_task(nofetch_phase, self._nofetch_exit)
def _nofetch_exit(self, nofetch_phase):
self._final_exit(nofetch_phase)
self._unlock_builddir()
self.returncode = 1
self.wait()
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(msg,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
binpkg_tasks = TaskSequence()
requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split()
for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS:
if pkg_fmt in requested_binpkg_formats:
if pkg_fmt == "rpm":
binpkg_tasks.add(EbuildPhase(background=self.background,
phase="rpm", scheduler=self.scheduler,
settings=self.settings))
else:
task = EbuildBinpkg(
background=self.background,
pkg=self.pkg, scheduler=self.scheduler,
settings=self.settings)
binpkg_tasks.add(task)
# Guarantee that _record_binpkg_info is called
# immediately after EbuildBinpkg. Note that
# task.addExitListener does not provide the
# necessary guarantee (see bug 578204).
binpkg_tasks.add(self._RecordBinpkgInfo(
ebuild_binpkg=task, ebuild_build=self))
if binpkg_tasks:
self._start_task(binpkg_tasks, self._buildpkg_exit)
return
self._final_exit(build)
self.wait()
class _RecordBinpkgInfo(AsynchronousTask):
"""
This class wraps the EbuildBuild _record_binpkg_info method
with an AsynchronousTask interface, so that it can be
scheduled as a member of a TaskSequence.
"""
__slots__ = ('ebuild_binpkg', 'ebuild_build',)
def _start(self):
self.ebuild_build._record_binpkg_info(self.ebuild_binpkg)
AsynchronousTask._start(self)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(
background=self.background,
commands=[phase], phase=phase,
scheduler=self.scheduler, settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _record_binpkg_info(self, task):
if task.returncode != os.EX_OK:
return
# Save info about the created binary package, so that
# identifying information can be passed to the install
# task, to be recorded in the installed package database.
pkg = task.get_binpkg_info()
infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"],
"build-info")
info = {
"BINPKGMD5": "%s\n" % pkg._metadata["MD5"],
}
if pkg.build_id is not None:
info["BUILD_ID"] = "%s\n" % pkg.build_id
for k, v in info.items():
with io.open(_unicode_encode(os.path.join(infoloc, k),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['repo.content'],
errors='strict') as f:
f.write(v)
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase, scheduler=self.scheduler, settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def create_install_task(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg,
pkg_count=pkg_count, pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings, tree=tree, world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
return task
def _install_exit(self, task):
self._unlock_builddir()
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit)
def testIpcDaemon(self):
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
task_scheduler = TaskScheduler(max_jobs=2)
build_dir = EbuildBuildDir(
scheduler=task_scheduler.sched_iface,
settings=env)
build_dir.lock()
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit' : exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode],
env=env, scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
proc.cancel()
daemon.cancel()
exit_command.reply_hook = exit_command_callback
task_scheduler.add(daemon)
task_scheduler.add(proc)
start_time = time.time()
task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
task_scheduler.clear()
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for QueueScheduler.run()
sleep_time_s = 10 # 10.000 seconds
short_timeout_ms = 10 # 0.010 seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit' : exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(
args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s],
env=env, scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
proc.cancel()
daemon.cancel()
exit_command.reply_hook = exit_command_callback
task_scheduler.add(daemon)
task_scheduler.add(proc)
start_time = time.time()
task_scheduler.run(timeout=short_timeout_ms)
task_scheduler.clear()
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
build_dir.unlock()
shutil.rmtree(tmpdir)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self.wait()
return
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
if retval == os.EX_OK:
self._current_task = None
self.returncode = os.EX_OK
self._async_wait()
else:
# For pretend mode, the convention it to execute
# pkg_nofetch and return a successful exitcode.
self._start_task(
SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=self.pkg.root_config.trees[
self._tree].dbapi,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings), self._default_final_exit)
return
quiet_setting = settings.get("PORTAGE_QUIET", False)
fetch_log = None
logwrite_access = False
if quiet_setting:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
"emerge-fetch.log")
logwrite_access = os.access(first_existing(fetch_log), os.W_OK)
fetcher = EbuildFetcher(
config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=quiet_setting if logwrite_access else False,
logfile=fetch_log if logwrite_access else None,
pkg=self.pkg,
scheduler=self.scheduler,
)
if fetch_log and logwrite_access:
fetcher.addExitListener(self._fetchonly_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher, force_queue=True)
else:
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts,
"unmerge", [self.pkg.cpv],
clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, ))
unmerge_task = MergeProcess(
mycat=cat,
mypkg=pf,
settings=self.settings,
treetype="vartree",
vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler,
background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
pkg = self.pkg
settings = self.settings
if not self.opts.fetchonly:
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path, 'setup',
settings=self.settings, db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
EPREFIX + "/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features and \
'digest' not in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(
fetch_all=opts.fetch_all_uri,
pkg=pkg, pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
else:
fetcher = EbuildFetcher(
config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean', scheduler=self.scheduler, settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
self._final_exit(fetcher)
if self.returncode != os.EX_OK:
portdb = self.pkg.root_config.trees[self._tree].dbapi
spawn_nofetch(portdb, self._ebuild_path, settings=self.settings)
elif 'digest' in self.settings.features:
if not digestgen(mysettings=self.settings,
myportdb=self.pkg.root_config.trees[self._tree].dbapi):
self.returncode = 1
self.wait()
def _pre_clean_exit(self, pre_clean_phase):
if self._default_exit(pre_clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg, scheduler=self.scheduler)
try:
already_fetched = fetcher.already_fetched(self.settings)
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
fetcher._eerror(msg_lines)
portage.elog.elog_process(self.pkg.cpv, self.settings)
self.returncode = 1
self._current_task = None
self._unlock_builddir()
self.wait()
return
if already_fetched:
# This case is optimized to skip the fetch queue.
fetcher = None
self._fetch_exit(fetcher)
return
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetch_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
def _fetch_exit(self, fetcher):
if fetcher is not None and \
self._default_exit(fetcher) != os.EX_OK:
self._fetch_failed()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
"buildpkg" not in features and \
opts.buildpkg != 'n'
if ("buildpkg" in features or self._issyspkg) \
and not self.opts.buildpkg_exclude.findAtomForPackage(pkg):
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background, pkg=pkg,
scheduler=scheduler, settings=settings)
self._start_task(build, self._build_exit)
def _fetch_failed(self):
# We only call the pkg_nofetch phase if either RESTRICT=fetch
# is set or the package has explicitly overridden the default
# pkg_nofetch implementation. This allows specialized messages
# to be displayed for problematic packages even though they do
# not set RESTRICT=fetch (bug #336499).
if 'fetch' not in self.pkg.restrict and \
'nofetch' not in self.pkg.defined_phases:
self._unlock_builddir()
self.wait()
return
self.returncode = None
nofetch_phase = EbuildPhase(background=self.background,
phase='nofetch', scheduler=self.scheduler, settings=self.settings)
self._start_task(nofetch_phase, self._nofetch_exit)
def _nofetch_exit(self, nofetch_phase):
self._final_exit(nofetch_phase)
self._unlock_builddir()
self.returncode = 1
self.wait()
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(msg,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
binpkg_tasks = TaskSequence()
requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split()
for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS:
if pkg_fmt in requested_binpkg_formats:
if pkg_fmt == "rpm":
binpkg_tasks.add(EbuildPhase(background=self.background,
phase="rpm", scheduler=self.scheduler,
settings=self.settings))
else:
binpkg_tasks.add(EbuildBinpkg(background=self.background,
pkg=self.pkg, scheduler=self.scheduler,
settings=self.settings))
if binpkg_tasks:
self._start_task(binpkg_tasks, self._buildpkg_exit)
return
self._final_exit(build)
self.wait()
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(
background=self.background,
commands=[phase], phase=phase,
scheduler=self.scheduler, settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase, scheduler=self.scheduler, settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def create_install_task(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg,
pkg_count=pkg_count, pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings, tree=tree, world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
return task
def _install_exit(self, task):
self._unlock_builddir()
class PackageUninstall(CompositeTask):
"""
Uninstall a package asynchronously in a subprocess. When
both parallel-install and ebuild-locks FEATURES are enabled,
it is essential for the ebuild-locks code to execute in a
subprocess, since the portage.locks module does not behave
as desired if we try to lock the same file multiple times
concurrently from the same process for ebuild-locks phases
such as pkg_setup, pkg_prerm, and pkg_postrm.
"""
__slots__ = (
"world_atom",
"ldpath_mtimes",
"opts",
"pkg",
"settings",
"_builddir_lock",
)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_task(
AsyncTaskFuture(future=self._builddir_lock.async_lock()),
self._start_unmerge,
)
def _start_unmerge(self, lock_task):
self._assert_current(lock_task)
if lock_task.cancelled:
self._default_final_exit(lock_task)
return
lock_task.future.result()
portage.prepare_build_dirs(settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, _ = _unmerge_display(
self.pkg.root_config,
self.opts,
"unmerge",
[self.pkg.cpv],
clean_delay=0,
writemsg_level=self._writemsg_level,
)
if retval != os.EX_OK:
self._async_unlock_builddir(returncode=retval)
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, ))
cat, pf = portage.catsplit(self.pkg.cpv)
unmerge_task = MergeProcess(
mycat=cat,
mypkg=pf,
settings=self.settings,
treetype="vartree",
vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler,
background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
unmerge=True,
)
self._start_task(unmerge_task, self._unmerge_exit)
def _unmerge_exit(self, unmerge_task):
if self._final_exit(unmerge_task) != os.EX_OK:
self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv, ))
else:
self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv, ))
self.world_atom(self.pkg)
self._async_unlock_builddir(returncode=self.returncode)
def _async_unlock_builddir(self, returncode=None):
"""
Release the lock asynchronously, and if a returncode parameter
is given then set self.returncode and notify exit listeners.
"""
if returncode is not None:
# The returncode will be set after unlock is complete.
self.returncode = None
self._start_task(
AsyncTaskFuture(future=self._builddir_lock.async_unlock()),
functools.partial(self._unlock_builddir_exit,
returncode=returncode),
)
def _unlock_builddir_exit(self, unlock_task, returncode=None):
self._assert_current(unlock_task)
if unlock_task.cancelled and returncode is not None:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.cancelled() or unlock_task.future.result()
if returncode is not None:
self.returncode = returncode
self._async_wait()
def _emergelog(self, msg):
emergelog("notitles" not in self.settings.features, msg)
def _writemsg_level(self, msg, level=0, noiselevel=0):
log_path = self.settings.get("PORTAGE_LOG_FILE")
background = self.background
if log_path is None:
if not (background and level < logging.WARNING):
portage.util.writemsg_level(msg,
level=level,
noiselevel=noiselevel)
else:
self.scheduler.output(msg,
log_path=log_path,
level=level,
noiselevel=noiselevel)
class Binpkg(CompositeTask):
__slots__ = (
"find_blockers",
"ldpath_mtimes",
"logger",
"opts",
"pkg",
"pkg_count",
"prefetcher",
"settings",
"world_atom",
) + (
"_bintree",
"_build_dir",
"_build_prefix",
"_ebuild_path",
"_fetched_pkg",
"_image_dir",
"_infloc",
"_pkg_path",
"_tree",
"_verify",
)
def _writemsg_level(self, msg, level=0, noiselevel=0):
self.scheduler.output(
msg,
level=level,
noiselevel=noiselevel,
log_path=self.settings.get("PORTAGE_LOG_FILE"),
)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(
os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage",
pkg.category,
pkg.pf,
)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(
self._ebuild_path, "setup", settings=self.settings, db=self._bintree.dbapi
)
if dir_path != self.settings["PORTAGE_BUILDDIR"]:
raise AssertionError(
"'%s' != '%s'" % (dir_path, self.settings["PORTAGE_BUILDDIR"])
)
self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(
portage.versions.cpv_getversion(x)
for x in vardb.match(self.pkg.slot_atom)
+ vardb.match("=" + self.pkg.cpv)
)
)
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, "emerge-fetch.log"
)
msg = (
"Fetching in the background:",
prefetcher.pkg_path,
"To view fetch progress, run in another terminal:",
"tail -f %s" % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
if not (self.opts.pretend or self.opts.fetchonly):
self._start_task(
AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_fetcher,
)
else:
self._start_fetcher()
def _start_fetcher(self, lock_task=None):
if lock_task is not None:
self._assert_current(lock_task)
if lock_task.cancelled:
self._default_final_exit(lock_task)
return
lock_task.future.result()
# Initialize PORTAGE_LOG_FILE (clean_log won't work without it).
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
# If necessary, discard old log so that we don't
# append to it.
self._build_dir.clean_log()
pkg = self.pkg
pkg_count = self.pkg_count
fetcher = None
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
fetcher = BinpkgFetcher(
background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
pkg=self.pkg,
pretend=self.opts.pretend,
scheduler=self.scheduler,
)
msg = " --- (%s of %s) Fetching Binary (%s::%s)" % (
pkg_count.curval,
pkg_count.maxval,
pkg.cpv,
fetcher.pkg_path,
)
short_msg = "emerge: (%s of %s) %s Fetch" % (
pkg_count.curval,
pkg_count.maxval,
pkg.cpv,
)
self.logger.log(msg, short_msg=short_msg)
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetcher_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
return
self._fetcher_exit(fetcher)
def _fetcher_exit(self, fetcher):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
if fetcher is not None:
self._fetched_pkg = fetcher.pkg_path
if self._default_exit(fetcher) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
if self.opts.pretend:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
verifier = None
if self._verify:
if self._fetched_pkg:
path = self._fetched_pkg
else:
path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv)
logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(
background=self.background,
logfile=logfile,
pkg=self.pkg,
scheduler=self.scheduler,
_pkg_path=path,
)
self._start_task(verifier, self._verifier_exit)
return
self._verifier_exit(verifier)
def _verifier_exit(self, verifier):
if verifier is not None and self._default_exit(verifier) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
if self._fetched_pkg:
pkg_path = self._bintree.getname(
self._bintree.inject(pkg.cpv, filename=self._fetched_pkg),
allocate_new=False,
)
else:
pkg_path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv)
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
if pkg_path is not None:
self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
self._pkg_path = pkg_path
logfile = self.settings.get("PORTAGE_LOG_FILE")
if logfile is not None and os.path.isfile(logfile):
# Remove fetch log after successful fetch.
try:
os.unlink(logfile)
except OSError:
pass
if self.opts.fetchonly:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
msg = " === (%s of %s) Merging Binary (%s::%s)" % (
pkg_count.curval,
pkg_count.maxval,
pkg.cpv,
pkg_path,
)
short_msg = "emerge: (%s of %s) %s Merge Binary" % (
pkg_count.curval,
pkg_count.maxval,
pkg.cpv,
)
logger.log(msg, short_msg=short_msg)
phase = "clean"
settings = self.settings
ebuild_phase = EbuildPhase(
background=self.background,
phase=phase,
scheduler=self.scheduler,
settings=settings,
)
self._start_task(ebuild_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
self._start_task(
AsyncTaskFuture(future=self._unpack_metadata(loop=self.scheduler)),
self._unpack_metadata_exit,
)
async def _unpack_metadata(self, loop=None):
dir_path = self.settings["PORTAGE_BUILDDIR"]
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(
mydir,
uid=portage.data.portage_uid,
gid=portage.data.portage_gid,
mode=dir_mode,
)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
await self._bintree.dbapi.unpack_metadata(
self.settings, infloc, loop=self.scheduler
)
check_missing_metadata = ("CATEGORY", "PF")
for k, v in zip(
check_missing_metadata,
self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata),
):
if v:
continue
elif k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = io.open(
_unicode_encode(
os.path.join(infloc, k), encoding=_encodings["fs"], errors="strict"
),
mode="w",
encoding=_encodings["content"],
errors="backslashreplace",
)
try:
f.write(_unicode_decode(v + "\n"))
finally:
f.close()
# Store the md5sum in the vdb.
if pkg_path is not None:
(md5sum,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["MD5"])
if not md5sum:
md5sum = portage.checksum.perform_md5(pkg_path)
with io.open(
_unicode_encode(
os.path.join(infloc, "BINPKGMD5"),
encoding=_encodings["fs"],
errors="strict",
),
mode="w",
encoding=_encodings["content"],
errors="strict",
) as f:
f.write(_unicode_decode("{}\n".format(md5sum)))
env_extractor = BinpkgEnvExtractor(
background=self.background, scheduler=self.scheduler, settings=self.settings
)
env_extractor.start()
await env_extractor.async_wait()
if env_extractor.returncode != os.EX_OK:
raise portage.exception.PortageException(
"failed to extract environment for {}".format(self.pkg.cpv)
)
def _unpack_metadata_exit(self, unpack_metadata):
if self._default_exit(unpack_metadata) != os.EX_OK:
unpack_metadata.future.result()
self._async_unlock_builddir(returncode=self.returncode)
return
setup_phase = EbuildPhase(
background=self.background,
phase="setup",
scheduler=self.scheduler,
settings=self.settings,
)
setup_phase.addExitListener(self._setup_exit)
self._task_queued(setup_phase)
self.scheduler.scheduleSetup(setup_phase)
def _setup_exit(self, setup_phase):
if self._default_exit(setup_phase) != os.EX_OK:
self._async_unlock_builddir(returncode=self.returncode)
return
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
self._start_task(
AsyncTaskFuture(
future=self._bintree.dbapi.unpack_contents(
self.settings, self._image_dir, loop=self.scheduler
)
),
self._unpack_contents_exit,
)
def _unpack_contents_exit(self, unpack_contents):
if self._default_exit(unpack_contents) != os.EX_OK:
unpack_contents.future.result()
self._writemsg_level(
"!!! Error Extracting '%s'\n" % self._pkg_path,
noiselevel=-1,
level=logging.ERROR,
)
self._async_unlock_builddir(returncode=self.returncode)
return
# Before anything else, let's do an integrity check.
(provides,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["PROVIDES"])
if check_dyn_libs_inconsistent(self.settings["D"], provides):
self._writemsg_level(
colorize(
"BAD",
"!!! Error! Installing dynamic libraries (.so) with blank PROVIDES!",
),
noiselevel=-1,
level=logging.ERROR,
)
try:
with io.open(
_unicode_encode(
os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings["fs"],
errors="strict",
),
mode="r",
encoding=_encodings["repo.content"],
errors="replace",
) as f:
self._build_prefix = f.read().rstrip("\n")
except IOError:
self._build_prefix = ""
if self._build_prefix == self.settings["EPREFIX"]:
ensure_dirs(self.settings["ED"])
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
env = self.settings.environ()
env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"]
chpathtool = SpawnProcess(
args=[
portage._python_interpreter,
os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"),
self.settings["D"],
self._build_prefix,
self.settings["EPREFIX"],
],
background=self.background,
env=env,
scheduler=self.scheduler,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
)
self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"])
self._start_task(chpathtool, self._chpathtool_exit)
def _chpathtool_exit(self, chpathtool):
if self._final_exit(chpathtool) != os.EX_OK:
self._writemsg_level(
"!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],),
noiselevel=-1,
level=logging.ERROR,
)
self._async_unlock_builddir(returncode=self.returncode)
return
# We want to install in "our" prefix, not the binary one
with io.open(
_unicode_encode(
os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings["fs"],
errors="strict",
),
mode="w",
encoding=_encodings["repo.content"],
errors="strict",
) as f:
f.write(self.settings["EPREFIX"] + "\n")
# Move the files to the correct location for merge.
image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"], "image_tmp")
build_d = os.path.join(
self.settings["D"], self._build_prefix.lstrip(os.sep)
).rstrip(os.sep)
if not os.path.isdir(build_d):
# Assume this is a virtual package or something.
shutil.rmtree(self._image_dir)
ensure_dirs(self.settings["ED"])
else:
os.rename(build_d, image_tmp_dir)
if build_d != self._image_dir:
shutil.rmtree(self._image_dir)
ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep)))
os.rename(image_tmp_dir, self.settings["ED"])
self.wait()
def _async_unlock_builddir(self, returncode=None):
"""
Release the lock asynchronously, and if a returncode parameter
is given then set self.returncode and notify exit listeners.
"""
if self.opts.pretend or self.opts.fetchonly:
if returncode is not None:
self.returncode = returncode
self._async_wait()
return
if returncode is not None:
# The returncode will be set after unlock is complete.
self.returncode = None
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._start_task(
AsyncTaskFuture(future=self._build_dir.async_unlock()),
functools.partial(self._unlock_builddir_exit, returncode=returncode),
)
def _unlock_builddir_exit(self, unlock_task, returncode=None):
self._assert_current(unlock_task)
if unlock_task.cancelled and returncode is not None:
self._default_final_exit(unlock_task)
return
# Normally, async_unlock should not raise an exception here.
unlock_task.future.cancelled() or unlock_task.future.result()
if returncode is not None:
self.returncode = returncode
self._async_wait()
def create_install_task(self):
task = EbuildMerge(
exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes,
logger=self.logger,
pkg=self.pkg,
pkg_count=self.pkg_count,
pkg_path=self._pkg_path,
scheduler=self.scheduler,
settings=self.settings,
tree=self._tree,
world_atom=self.world_atom,
)
return task
def _install_exit(self, task):
"""
@returns: Future, result is the returncode from an
EbuildBuildDir.async_unlock() task
"""
self.settings.pop("PORTAGE_BINPKG_FILE", None)
if (
task.returncode == os.EX_OK
and "binpkg-logs" not in self.settings.features
and self.settings.get("PORTAGE_LOG_FILE")
):
try:
os.unlink(self.settings["PORTAGE_LOG_FILE"])
except OSError:
pass
self._async_unlock_builddir()
if self._current_task is None:
result = self.scheduler.create_future()
self.scheduler.call_soon(result.set_result, os.EX_OK)
else:
result = self._current_task.async_wait()
return result
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')
_phases_without_builddir = (
'clean',
'cleanrm',
'depend',
'help',
)
_phases_interactive_whitelist = ('config', )
_phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm',
'config')
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
'mount', '-t', 'tmpfs', '-o',
'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
cgroup_root
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
'mount', '-t', 'cgroup', '-o',
'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
cgroup_portage
])
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix='%s:%s.' %
(self.settings["CATEGORY"], self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
'available_eclasses': query_command,
'best_version': query_command,
'eclass_path': query_command,
'exit': self._exit_command,
'has_version': query_command,
'license_path': query_command,
'master_repositories': query_command,
'repository_path': query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = \
self.scheduler.timeout_add(self._exit_timeout,
self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = \
self.scheduler.timeout_add(self._cancel_timeout,
self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
return False # only run once
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self.wait()
return False # only run once
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (self.phase, signum)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = _(
"The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog('eerror', lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _log_poll_exception(self, event):
self._elog("eerror",
["%s received strange poll event: %s\n" % \
(self.__class__.__name__, event,)])
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.cgroup is not None:
try:
shutil.rmtree(self.cgroup)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if self._exit_timeout_id is not None:
self.scheduler.source_remove(self._exit_timeout_id)
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
if self._build_dir is not None:
self._build_dir.unlock()
self._build_dir = None
elif not self.cancelled:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
pkg = self.pkg
settings = self.settings
if not self.opts.fetchonly:
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path,
'setup',
settings=self.settings,
db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features and \
'digest' not in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
else:
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _fetchonly_exit(self, fetcher):
self._final_exit(fetcher)
if self.returncode != os.EX_OK:
portdb = self.pkg.root_config.trees[self._tree].dbapi
spawn_nofetch(portdb, self._ebuild_path, settings=self.settings)
elif 'digest' in self.settings.features:
if not digestgen(
mysettings=self.settings,
myportdb=self.pkg.root_config.trees[self._tree].dbapi):
self.returncode = 1
self.wait()
def _pre_clean_exit(self, pre_clean_phase):
if self._default_exit(pre_clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg,
scheduler=self.scheduler)
try:
already_fetched = fetcher.already_fetched(self.settings)
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
fetcher._eerror(msg_lines)
portage.elog.elog_process(self.pkg.cpv, self.settings)
self.returncode = 1
self._current_task = None
self._unlock_builddir()
self.wait()
return
if already_fetched:
# This case is optimized to skip the fetch queue.
fetcher = None
self._fetch_exit(fetcher)
return
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetch_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
def _fetch_exit(self, fetcher):
if fetcher is not None and \
self._default_exit(fetcher) != os.EX_OK:
self._fetch_failed()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
"buildpkg" not in features and \
opts.buildpkg != 'n'
if ("buildpkg" in features or self._issyspkg) \
and not self.opts.buildpkg_exclude.findAtomForPackage(pkg):
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
def _fetch_failed(self):
# We only call the pkg_nofetch phase if either RESTRICT=fetch
# is set or the package has explicitly overridden the default
# pkg_nofetch implementation. This allows specialized messages
# to be displayed for problematic packages even though they do
# not set RESTRICT=fetch (bug #336499).
if 'fetch' not in self.pkg.restrict and \
'nofetch' not in self.pkg.defined_phases:
self._unlock_builddir()
self.wait()
return
self.returncode = None
nofetch_phase = EbuildPhase(background=self.background,
phase='nofetch',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(nofetch_phase, self._nofetch_exit)
def _nofetch_exit(self, nofetch_phase):
self._final_exit(nofetch_phase)
self._unlock_builddir()
self.returncode = 1
self.wait()
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))
binpkg_tasks = TaskSequence()
requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT",
"tar").split()
for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS:
if pkg_fmt in requested_binpkg_formats:
if pkg_fmt == "rpm":
binpkg_tasks.add(
EbuildPhase(background=self.background,
phase="rpm",
scheduler=self.scheduler,
settings=self.settings))
else:
binpkg_tasks.add(
EbuildBinpkg(background=self.background,
pkg=self.pkg,
scheduler=self.scheduler,
settings=self.settings))
if binpkg_tasks:
self._start_task(binpkg_tasks, self._buildpkg_exit)
return
self._final_exit(build)
self.wait()
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase],
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def create_install_task(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes,
logger=logger,
pkg=pkg,
pkg_count=pkg_count,
pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings,
tree=tree,
world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
return task
def _install_exit(self, task):
self._unlock_builddir()
def testIpcDaemon(self):
event_loop = global_event_loop()
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
env['PYTHONDONTWRITEBYTECODE'] = os.environ.get(
'PYTHONDONTWRITEBYTECODE', '')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
build_dir = EbuildBuildDir(scheduler=event_loop, settings=env)
event_loop.run_until_complete(build_dir.async_lock())
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for EventLoop/AsyncScheduler.
# Use a ridiculously long sleep_time_s in case the user's
# system is heavily loaded (see bug #436334).
sleep_time_s = 600 # seconds
short_timeout_s = 0.010 # seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SleepProcess(seconds=sleep_time_s)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, short_timeout_s)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
event_loop.run_until_complete(build_dir.async_unlock())
shutil.rmtree(tmpdir)
def _fetch_exit(self, fetcher):
opts = self.opts
pkg = self.pkg
fetch_failed = False
if opts.fetchonly:
fetch_failed = self._final_exit(fetcher) != os.EX_OK
else:
fetch_failed = self._default_exit(fetcher) != os.EX_OK
if fetch_failed and fetcher.logfile is not None and \
os.path.exists(fetcher.logfile):
self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile
if fetch_failed or opts.fetchonly:
self.wait()
return
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning is triggered before the setup
# phase, in portage.doebuild().
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
pkg = self.pkg
settings = self.settings
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self.wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
ebuild_path = portdb.findname(pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self.wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(
fetch_all=opts.fetch_all_uri,
pkg=pkg, pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean', scheduler=self.scheduler, settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _pre_clean_exit(self, pre_clean_phase):
if self._final_exit(pre_clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg, scheduler=self.scheduler)
self._start_task(fetcher, self._fetch_exit)
def _fetch_exit(self, fetcher):
portage.elog.elog_process(self.pkg.cpv, self.settings)
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background, pkg=pkg,
scheduler=scheduler, settings=settings)
self._start_task(build, self._build_exit)
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(msg,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
packager = EbuildBinpkg(background=self.background, pkg=self.pkg,
scheduler=self.scheduler, settings=self.settings)
self._start_task(packager, self._buildpkg_exit)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(
background=self.background,
commands=[phase], phase=phase,
scheduler=self.scheduler, settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase, scheduler=self.scheduler, settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def install(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg,
pkg_count=pkg_count, pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings, tree=tree, world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
try:
rval = merge.execute()
finally:
self._unlock_builddir()
return rval
class EbuildFetcher(SpawnProcess):
__slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \
("_build_dir",)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
if self.prefetch and \
self._prefetch_size_ok(portdb, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self.wait()
return
# In prefetch mode, logging goes to emerge-fetch.log and the builddir
# should not be touched since otherwise it could interfere with
# another instance of the same cpv concurrently being built for a
# different $ROOT (currently, builds only cooperate with prefetchers
# that are spawned for the same $ROOT).
if not self.prefetch:
self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings)
self._build_dir.lock()
self._build_dir.clean_log()
cleanup=1
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, cleanup)
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
phase = "fetch"
if self.fetchall:
phase = "fetchall"
# If any incremental variables have been overridden
# via the environment, those values need to be passed
# along here so that they are correctly considered by
# the config instance in the subproccess.
fetch_env = os.environ.copy()
fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT']
nocolor = settings.get("NOCOLOR")
if nocolor is not None:
fetch_env["NOCOLOR"] = nocolor
fetch_env["PORTAGE_NICENESS"] = "0"
if self.prefetch:
fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
ebuild_binary = os.path.join(
settings["PORTAGE_BIN_PATH"], "ebuild")
fetch_args = [ebuild_binary, ebuild_path, phase]
debug = settings.get("PORTAGE_DEBUG") == "1"
if debug:
fetch_args.append("--debug")
if not self.background and nocolor not in ('yes', 'true'):
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
fetch_args.append('--color=y')
self.args = fetch_args
self.env = fetch_env
if self._build_dir is None:
# Free settings now since we only have a local reference.
self.config_pool.deallocate(settings)
SpawnProcess._start(self)
def _prefetch_size_ok(self, portdb, settings, ebuild_path):
pkgdir = os.path.dirname(ebuild_path)
mytree = os.path.dirname(os.path.dirname(pkgdir))
distdir = settings["DISTDIR"]
use = None
if not self.fetchall:
use = self.pkg.use.enabled
try:
uri_map = portdb.getFetchMap(self.pkg.cpv,
useflags=use, mytree=mytree)
except portage.exception.InvalidDependString as e:
return False
sizes = {}
for filename in uri_map:
# Use stat rather than lstat since portage.fetch() creates
# symlinks when PORTAGE_RO_DISTDIRS is used.
try:
st = os.stat(os.path.join(distdir, filename))
except OSError:
return False
if st.st_size == 0:
return False
sizes[filename] = st.st_size
digests = portage.Manifest(pkgdir, distdir).getTypeDigests("DIST")
for filename, actual_size in sizes.items():
size = digests.get(filename, {}).get('size')
if size is None:
continue
if size != actual_size:
return False
# All files are present and sizes are ok. In this case the normal
# fetch code will be skipped, so we need to generate equivalent
# output here.
if self.logfile is not None:
f = codecs.open(_unicode_encode(self.logfile,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
for filename in uri_map:
f.write((' * %s size ;-) ...' % \
filename).ljust(73) + '[ ok ]\n')
f.close()
return True
def _pipe(self, fd_pipes):
"""When appropriate, use a pty so that fetcher progress bars,
like wget has, will work properly."""
if self.background or not sys.stdout.isatty():
# When the output only goes to a log file,
# there's no point in creating a pty.
return os.pipe()
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
# Collect elog messages that might have been
# created by the pkg_nofetch phase.
if self._build_dir is not None:
# Skip elog messages for prefetch, in order to avoid duplicates.
if not self.prefetch and self.returncode != os.EX_OK:
elog_out = None
if self.logfile is not None:
if self.background:
elog_out = codecs.open(_unicode_encode(self.logfile,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
msg = "Fetch failed for '%s'" % (self.pkg.cpv,)
if self.logfile is not None:
msg += ", Log file:"
eerror(msg, phase="unpack", key=self.pkg.cpv, out=elog_out)
if self.logfile is not None:
eerror(" '%s'" % (self.logfile,),
phase="unpack", key=self.pkg.cpv, out=elog_out)
if elog_out is not None:
elog_out.close()
if not self.prefetch:
portage.elog.elog_process(self.pkg.cpv, self._build_dir.settings)
features = self._build_dir.settings.features
if self.returncode == os.EX_OK:
self._build_dir.clean_log()
self._build_dir.unlock()
self.config_pool.deallocate(self._build_dir.settings)
self._build_dir = None
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
'mount', '-t', 'tmpfs', '-o',
'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
cgroup_root
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
'mount', '-t', 'cgroup', '-o',
'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
cgroup_portage
])
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix='%s:%s.' %
(self.settings["CATEGORY"], self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path, 'setup',
settings=self.settings, db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
class PackageUninstall(CompositeTask):
"""
Uninstall a package asynchronously in a subprocess. When
both parallel-install and ebuild-locks FEATURES are enabled,
it is essential for the ebuild-locks code to execute in a
subprocess, since the portage.locks module does not behave
as desired if we try to lock the same file multiple times
concurrently from the same process for ebuild-locks phases
such as pkg_setup, pkg_prerm, and pkg_postrm.
"""
__slots__ = ("world_atom", "ldpath_mtimes", "opts",
"pkg", "settings", "_builddir_lock")
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath, "prerm",
settings=self.settings, db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(
settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts, "unmerge", [self.pkg.cpv], clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,))
unmerge_task = MergeProcess(
mycat=cat, mypkg=pf, settings=self.settings,
treetype="vartree", vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler, background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
def _unmerge_exit(self, unmerge_task):
if self._final_exit(unmerge_task) != os.EX_OK:
self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv,))
else:
self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv,))
self.world_atom(self.pkg)
self._builddir_lock.unlock()
self.wait()
def _emergelog(self, msg):
emergelog("notitles" not in self.settings.features, msg)
def _writemsg_level(self, msg, level=0, noiselevel=0):
log_path = self.settings.get("PORTAGE_LOG_FILE")
background = self.background
if log_path is None:
if not (background and level < logging.WARNING):
portage.util.writemsg_level(msg,
level=level, noiselevel=noiselevel)
else:
self.scheduler.output(msg, log_path=log_path,
level=level, noiselevel=noiselevel)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not os.path.isdir(
self.settings["PORTAGE_BUILDDIR"]):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % (
self.phase, self.settings["PORTAGE_BUILDDIR"])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == "Linux"
and "cgroup" in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = "/sys/fs/cgroup"
cgroup_portage = os.path.join(cgroup_root, "portage")
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
"mount",
"-t",
"tmpfs",
"-o",
"rw,nosuid,nodev,noexec,mode=0755",
"tmpfs",
cgroup_root,
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
"mount",
"-t",
"cgroup",
"-o",
"rw,nosuid,nodev,noexec,none,name=portage",
"tmpfs",
cgroup_portage,
])
with open(os.path.join(cgroup_portage, "release_agent"),
"w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
with open(
os.path.join(cgroup_portage, "notify_on_release"),
"w") as f:
f.write("1")
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(cgroup_portage,
"release_agent")
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip("\n")
except EnvironmentError:
release_agent_path = None
if release_agent_path is None or not os.path.exists(
release_agent_path):
with open(release_agent, "w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix="%s:%s." %
(self.settings["CATEGORY"], self.settings["PF"]),
)
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings["NOCOLOR"] = "true"
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(
self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon,
))
return
else:
self.settings.pop("PORTAGE_IPC_DAEMON", None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop("PORTAGE_IPC_DAEMON", None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".exit_status")
self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
if self.prefetch and \
self._prefetch_size_ok(portdb, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self.wait()
return
# In prefetch mode, logging goes to emerge-fetch.log and the builddir
# should not be touched since otherwise it could interfere with
# another instance of the same cpv concurrently being built for a
# different $ROOT (currently, builds only cooperate with prefetchers
# that are spawned for the same $ROOT).
if not self.prefetch:
self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings)
self._build_dir.lock()
self._build_dir.clean_log()
portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, 0)
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
phase = "fetch"
if self.fetchall:
phase = "fetchall"
# If any incremental variables have been overridden
# via the environment, those values need to be passed
# along here so that they are correctly considered by
# the config instance in the subproccess.
fetch_env = os.environ.copy()
fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT']
nocolor = settings.get("NOCOLOR")
if nocolor is not None:
fetch_env["NOCOLOR"] = nocolor
fetch_env["PORTAGE_NICENESS"] = "0"
if self.prefetch:
fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
ebuild_binary = os.path.join(
settings["PORTAGE_BIN_PATH"], "ebuild")
fetch_args = [ebuild_binary, ebuild_path, phase]
debug = settings.get("PORTAGE_DEBUG") == "1"
if debug:
fetch_args.append("--debug")
if not self.background and nocolor not in ('yes', 'true'):
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
fetch_args.append('--color=y')
self.args = fetch_args
self.env = fetch_env
if self._build_dir is None:
# Free settings now since we only have a local reference.
self.config_pool.deallocate(settings)
SpawnProcess._start(self)
class Binpkg(CompositeTask):
__slots__ = ("find_blockers",
"ldpath_mtimes", "logger", "opts",
"pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \
("_bintree", "_build_dir", "_build_prefix",
"_ebuild_path", "_fetched_pkg",
"_image_dir", "_infloc", "_pkg_path", "_tree", "_verify")
def _writemsg_level(self, msg, level=0, noiselevel=0):
self.scheduler.output(msg, level=level, noiselevel=noiselevel,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path, 'setup',
settings=self.settings, db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
pkg = self.pkg
pkg_count = self.pkg_count
if not (self.opts.pretend or self.opts.fetchonly):
self._build_dir.lock()
# Initialize PORTAGE_LOG_FILE (clean_log won't work without it).
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
# If necessary, discard old log so that we don't
# append to it.
self._build_dir.clean_log()
fetcher = BinpkgFetcher(background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg,
pretend=self.opts.pretend, scheduler=self.scheduler)
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv,
fetcher.pkg_path)
short_msg = "emerge: (%s of %s) %s Fetch" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetcher_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
return
self._fetcher_exit(fetcher)
def _fetcher_exit(self, fetcher):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
if fetcher.returncode is not None:
self._fetched_pkg = fetcher.pkg_path
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.pretend:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
verifier = None
if self._verify:
if self._fetched_pkg:
path = self._fetched_pkg
else:
path = self.pkg.root_config.trees["bintree"].getname(
self.pkg.cpv)
logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(background=self.background,
logfile=logfile, pkg=self.pkg, scheduler=self.scheduler,
_pkg_path=path)
self._start_task(verifier, self._verifier_exit)
return
self._verifier_exit(verifier)
def _verifier_exit(self, verifier):
if verifier is not None and \
self._default_exit(verifier) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
if self._fetched_pkg:
pkg_path = self._bintree.getname(
self._bintree.inject(pkg.cpv,
filename=self._fetched_pkg),
allocate_new=False)
else:
pkg_path = self.pkg.root_config.trees["bintree"].getname(
self.pkg.cpv)
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
self._pkg_path = pkg_path
logfile = self.settings.get("PORTAGE_LOG_FILE")
if logfile is not None and os.path.isfile(logfile):
# Remove fetch log after successful fetch.
try:
os.unlink(logfile)
except OSError:
pass
if self.opts.fetchonly:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
msg = " === (%s of %s) Merging Binary (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Merge Binary" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
phase = "clean"
settings = self.settings
ebuild_phase = EbuildPhase(background=self.background,
phase=phase, scheduler=self.scheduler,
settings=settings)
self._start_task(ebuild_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
dir_path = self.settings['PORTAGE_BUILDDIR']
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid,
gid=portage.data.portage_gid, mode=dir_mode)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
pkg_xpak = portage.xpak.tbz2(self._pkg_path)
check_missing_metadata = ("CATEGORY", "PF")
missing_metadata = set()
for k in check_missing_metadata:
v = pkg_xpak.getfile(_unicode_encode(k,
encoding=_encodings['repo.content']))
if not v:
missing_metadata.add(k)
pkg_xpak.unpackinfo(infloc)
for k in missing_metadata:
if k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = io.open(_unicode_encode(os.path.join(infloc, k),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'],
errors='backslashreplace')
try:
f.write(_unicode_decode(v + "\n"))
finally:
f.close()
# Store the md5sum in the vdb.
f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'),
encoding=_encodings['fs'], errors='strict'),
mode='w', encoding=_encodings['content'], errors='strict')
try:
f.write(_unicode_decode(
str(portage.checksum.perform_md5(pkg_path)) + "\n"))
finally:
f.close()
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler, settings=self.settings)
self._start_task(env_extractor, self._env_extractor_exit)
def _env_extractor_exit(self, env_extractor):
if self._default_exit(env_extractor) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
setup_phase = EbuildPhase(background=self.background,
phase="setup", scheduler=self.scheduler,
settings=self.settings)
setup_phase.addExitListener(self._setup_exit)
self._task_queued(setup_phase)
self.scheduler.scheduleSetup(setup_phase)
def _setup_exit(self, setup_phase):
if self._default_exit(setup_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
extractor = BinpkgExtractorAsync(background=self.background,
env=self.settings.environ(),
features=self.settings.features,
image_dir=self._image_dir,
pkg=self.pkg, pkg_path=self._pkg_path,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
self._start_task(extractor, self._extractor_exit)
def _extractor_exit(self, extractor):
if self._default_exit(extractor) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self.wait()
return
try:
with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings['fs'], errors='strict'), mode='r',
encoding=_encodings['repo.content'], errors='replace') as f:
self._build_prefix = f.read().rstrip('\n')
except IOError:
self._build_prefix = ""
if self._build_prefix == self.settings["EPREFIX"]:
ensure_dirs(self.settings["ED"])
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
env = self.settings.environ()
env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"]
chpathtool = SpawnProcess(
args=[portage._python_interpreter,
os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"),
self.settings["D"], self._build_prefix, self.settings["EPREFIX"]],
background=self.background, env=env,
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"])
self._start_task(chpathtool, self._chpathtool_exit)
def _chpathtool_exit(self, chpathtool):
if self._final_exit(chpathtool) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Adjusting Prefix to %s\n" %
(self.settings["EPREFIX"],),
noiselevel=-1, level=logging.ERROR)
self.wait()
return
# We want to install in "our" prefix, not the binary one
with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings['fs'], errors='strict'), mode='w',
encoding=_encodings['repo.content'], errors='strict') as f:
f.write(self.settings["EPREFIX"] + "\n")
# Move the files to the correct location for merge.
image_tmp_dir = os.path.join(
self.settings["PORTAGE_BUILDDIR"], "image_tmp")
build_d = os.path.join(self.settings["D"],
self._build_prefix.lstrip(os.sep))
if not os.path.isdir(build_d):
# Assume this is a virtual package or something.
shutil.rmtree(self._image_dir)
ensure_dirs(self.settings["ED"])
else:
os.rename(build_d, image_tmp_dir)
shutil.rmtree(self._image_dir)
ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep)))
os.rename(image_tmp_dir, self.settings["ED"])
self.wait()
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def create_install_task(self):
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes, logger=self.logger,
pkg=self.pkg, pkg_count=self.pkg_count,
pkg_path=self._pkg_path, scheduler=self.scheduler,
settings=self.settings, tree=self._tree,
world_atom=self.world_atom)
return task
def _install_exit(self, task):
self.settings.pop("PORTAGE_BINPKG_FILE", None)
self._unlock_builddir()
if task.returncode == os.EX_OK and \
'binpkg-logs' not in self.settings.features and \
self.settings.get("PORTAGE_LOG_FILE"):
try:
os.unlink(self.settings["PORTAGE_LOG_FILE"])
except OSError:
pass
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
pkg = self.pkg
settings = self.settings
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self.wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
ebuild_path = portdb.findname(pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self.wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _check_manifest(self):
success = True
settings = self.settings
if 'strict' in settings.features:
settings['O'] = os.path.dirname(self._ebuild_path)
quiet_setting = settings.get('PORTAGE_QUIET')
settings['PORTAGE_QUIET'] = '1'
try:
success = digestcheck([], settings, strict=True)
finally:
if quiet_setting:
settings['PORTAGE_QUIET'] = quiet_setting
else:
del settings['PORTAGE_QUIET']
return success
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _pre_clean_exit(self, pre_clean_phase):
if self._final_exit(pre_clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# for log handling
portage.prepare_build_dirs(self.pkg.root, self.settings, 1)
fetcher = EbuildFetcher(config_pool=self.config_pool,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=self.background,
logfile=self.settings.get('PORTAGE_LOG_FILE'),
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetch_exit)
def _fetch_exit(self, fetcher):
portage.elog.elog_process(self.pkg.cpv, self.settings)
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
# discard successful fetch log
self._build_dir.clean_log()
pkg = self.pkg
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))
packager = EbuildBinpkg(background=self.background,
pkg=self.pkg,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(packager, self._buildpkg_exit)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(background=self.background,
commands=[phase],
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
phase=phase,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def install(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes,
logger=logger,
pkg=pkg,
pkg_count=pkg_count,
pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings,
tree=tree,
world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
try:
rval = merge.execute()
finally:
self._unlock_builddir()
return rval
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')
_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
_phases_interactive_whitelist = ('config',)
_phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config')
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770, stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
'available_eclasses' : query_command,
'best_version' : query_command,
'eclass_path' : query_command,
'exit' : self._exit_command,
'has_version' : query_command,
'license_path' : query_command,
'master_repositories' : query_command,
'repository_path' : query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = \
self.scheduler.timeout_add(self._exit_timeout,
self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = \
self.scheduler.timeout_add(self._cancel_timeout,
self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
return False # only run once
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self.wait()
return False # only run once
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (self.phase, signum)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = _("The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog('eerror', lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _log_poll_exception(self, event):
self._elog("eerror",
["%s received strange poll event: %s\n" % \
(self.__class__.__name__, event,)])
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.cgroup is not None:
try:
shutil.rmtree(self.cgroup)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if self._exit_timeout_id is not None:
self.scheduler.source_remove(self._exit_timeout_id)
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
if self._build_dir is not None:
self._build_dir.unlock()
self._build_dir = None
elif not self.cancelled:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
class EbuildBuild(CompositeTask):
__slots__ = ("args_set", "config_pool", "find_blockers",
"ldpath_mtimes", "logger", "opts", "pkg", "pkg_count",
"prefetcher", "settings", "world_atom") + \
("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree")
def _start(self):
logger = self.logger
opts = self.opts
pkg = self.pkg
settings = self.settings
world_atom = self.world_atom
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
ebuild_path = portdb.findname(pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(
fetch_all=opts.fetch_all_uri,
pkg=pkg, pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
fetcher = EbuildFetcher(config_pool=self.config_pool,
fetchall=opts.fetch_all_uri,
fetchonly=opts.fetchonly,
background=self.background,
pkg=pkg, scheduler=self.scheduler)
self._start_task(fetcher, self._fetch_exit)
def _fetch_exit(self, fetcher):
opts = self.opts
pkg = self.pkg
fetch_failed = False
if opts.fetchonly:
fetch_failed = self._final_exit(fetcher) != os.EX_OK
else:
fetch_failed = self._default_exit(fetcher) != os.EX_OK
if fetch_failed and fetcher.logfile is not None and \
os.path.exists(fetcher.logfile):
self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile
if fetch_failed or opts.fetchonly:
self.wait()
return
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning is triggered before the setup
# phase, in portage.doebuild().
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background, pkg=pkg,
scheduler=scheduler, settings=settings)
self._start_task(build, self._build_exit)
def _unlock_builddir(self):
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def _build_exit(self, build):
if self._default_exit(build) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
opts = self.opts
buildpkg = self._buildpkg
if not buildpkg:
self._final_exit(build)
self.wait()
return
if self._issyspkg:
msg = ">>> This is a system package, " + \
"let's pack a rescue tarball.\n"
log_path = self.settings.get("PORTAGE_LOG_FILE")
if log_path is not None:
log_file = codecs.open(_unicode_encode(log_path,
encoding=_encodings['fs'], errors='strict'),
mode='a', encoding=_encodings['content'], errors='replace')
try:
log_file.write(msg)
finally:
log_file.close()
if not self.background:
portage.writemsg_stdout(msg, noiselevel=-1)
packager = EbuildBinpkg(background=self.background, pkg=self.pkg,
scheduler=self.scheduler, settings=self.settings)
self._start_task(packager, self._buildpkg_exit)
def _buildpkg_exit(self, packager):
"""
Released build dir lock when there is a failure or
when in buildpkgonly mode. Otherwise, the lock will
be released when merge() is called.
"""
if self._default_exit(packager) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.buildpkgonly:
phase = 'success_hooks'
success_hooks = MiscFunctionsProcess(
background=self.background,
commands=[phase], phase=phase, pkg=self.pkg,
scheduler=self.scheduler, settings=self.settings)
self._start_task(success_hooks,
self._buildpkgonly_success_hook_exit)
return
# Continue holding the builddir lock until
# after the package has been installed.
self._current_task = None
self.returncode = packager.returncode
self.wait()
def _buildpkgonly_success_hook_exit(self, success_hooks):
self._default_exit(success_hooks)
self.returncode = None
# Need to call "clean" phase for buildpkgonly mode
portage.elog.elog_process(self.pkg.cpv, self.settings)
phase = 'clean'
clean_phase = EbuildPhase(background=self.background,
pkg=self.pkg, phase=phase,
scheduler=self.scheduler, settings=self.settings,
tree=self._tree)
self._start_task(clean_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._final_exit(clean_phase) != os.EX_OK or \
self.opts.buildpkgonly:
self._unlock_builddir()
self.wait()
def install(self):
"""
Install the package and then clean up and release locks.
Only call this after the build has completed successfully
and neither fetchonly nor buildpkgonly mode are enabled.
"""
find_blockers = self.find_blockers
ldpath_mtimes = self.ldpath_mtimes
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
settings = self.settings
world_atom = self.world_atom
ebuild_path = self._ebuild_path
tree = self._tree
merge = EbuildMerge(find_blockers=self.find_blockers,
ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg,
pkg_count=pkg_count, pkg_path=ebuild_path,
scheduler=self.scheduler,
settings=settings, tree=tree, world_atom=world_atom)
msg = " === (%s of %s) Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval,
pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Merge" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
try:
rval = merge.execute()
finally:
self._unlock_builddir()
return rval
class AbstractEbuildProcess(SpawnProcess):
__slots__ = (
"phase",
"settings",
) + (
"_build_dir",
"_build_dir_unlock",
"_ipc_daemon",
"_exit_command",
"_exit_timeout_id",
"_start_future",
)
_phases_without_builddir = (
"clean",
"cleanrm",
"depend",
"help",
)
_phases_interactive_whitelist = ("config", )
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10 # seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = "other"
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not os.path.isdir(
self.settings["PORTAGE_BUILDDIR"]):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % (
self.phase, self.settings["PORTAGE_BUILDDIR"])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == "Linux"
and "cgroup" in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = "/sys/fs/cgroup"
cgroup_portage = os.path.join(cgroup_root, "portage")
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
"mount",
"-t",
"tmpfs",
"-o",
"rw,nosuid,nodev,noexec,mode=0755",
"tmpfs",
cgroup_root,
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
"mount",
"-t",
"cgroup",
"-o",
"rw,nosuid,nodev,noexec,none,name=portage",
"tmpfs",
cgroup_portage,
])
with open(os.path.join(cgroup_portage, "release_agent"),
"w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
with open(
os.path.join(cgroup_portage, "notify_on_release"),
"w") as f:
f.write("1")
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(cgroup_portage,
"release_agent")
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip("\n")
except EnvironmentError:
release_agent_path = None
if release_agent_path is None or not os.path.exists(
release_agent_path):
with open(release_agent, "w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix="%s:%s." %
(self.settings["CATEGORY"], self.settings["PF"]),
)
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings["NOCOLOR"] = "true"
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(
self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon,
))
return
else:
self.settings.pop("PORTAGE_IPC_DAEMON", None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop("PORTAGE_IPC_DAEMON", None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".exit_status")
self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start_post_builddir_lock(self,
lock_future=None,
start_ipc_daemon=False):
if lock_future is not None:
if lock_future is not self._start_future:
raise AssertionError("lock_future is not self._start_future")
self._start_future = None
if lock_future.cancelled():
self._build_dir = None
self.cancelled = True
self._was_cancelled()
self._async_wait()
return
lock_future.result()
if start_ipc_daemon:
self.settings["PORTAGE_IPC_DAEMON"] = "1"
self._start_ipc_daemon()
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if (0 not in self.fd_pipes
and self.phase not in self._phases_interactive_whitelist
and "interactive" not in self.settings.get("PROPERTIES",
"").split()):
null_fd = os.open("/dev/null", os.O_RDONLY)
self.fd_pipes[0] = null_fd
self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(
p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st,
)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
"available_eclasses": query_command,
"best_version": query_command,
"eclass_path": query_command,
"exit": self._exit_command,
"has_version": query_command,
"license_path": query_command,
"master_repositories": query_command,
"repository_path": query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(
commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler,
)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = self.scheduler.call_later(
self._exit_timeout, self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = self.scheduler.call_later(
self._cancel_timeout, self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self._async_waitpid()
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = _create_pty_or_pipe(
copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ("sesandbox" in self.settings.features
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (
self.phase,
signum,
)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = (_(
"The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase)
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog("eerror", lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = self.settings.get(
"NOCOLOR", "false").lower() in ("no", "false")
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _async_waitpid_cb(self, *args, **kwargs):
"""
Override _async_waitpid_cb to perform cleanup that is
not necessarily idempotent.
"""
SpawnProcess._async_waitpid_cb(self, *args, **kwargs)
if self._exit_timeout_id is not None:
self._exit_timeout_id.cancel()
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
elif not self.cancelled:
exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE")
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
def _async_wait(self):
"""
Override _async_wait to asynchronously unlock self._build_dir
when necessary.
"""
if self._build_dir is None:
SpawnProcess._async_wait(self)
elif self._build_dir_unlock is None:
if self.returncode is None:
raise asyncio.InvalidStateError("Result is not ready for %s" %
(self, ))
self._async_unlock_builddir(returncode=self.returncode)
def _async_unlock_builddir(self, returncode=None):
"""
Release the lock asynchronously, and if a returncode parameter
is given then set self.returncode and notify exit listeners.
"""
if self._build_dir_unlock is not None:
raise AssertionError("unlock already in progress")
if returncode is not None:
# The returncode will be set after unlock is complete.
self.returncode = None
self._build_dir_unlock = self._build_dir.async_unlock()
# Unlock only once.
self._build_dir = None
self._build_dir_unlock.add_done_callback(
functools.partial(self._unlock_builddir_exit,
returncode=returncode))
def _unlock_builddir_exit(self, unlock_future, returncode=None):
# Normally, async_unlock should not raise an exception here.
unlock_future.cancelled() or unlock_future.result()
if returncode is not None:
if unlock_future.cancelled():
self.cancelled = True
self._was_cancelled()
else:
self.returncode = returncode
SpawnProcess._async_wait(self)
def _fetch_exit(self, fetcher): opts = self.opts pkg = self.pkg fetch_failed = False if opts.fetchonly: fetch_failed = self._final_exit(fetcher) != os.EX_OK else: fetch_failed = self._default_exit(fetcher) != os.EX_OK if fetch_failed and fetcher.logfile is not None and \ os.path.exists(fetcher.logfile): self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile if fetch_failed or opts.fetchonly: self.wait() return logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning is triggered before the setup # phase, in portage.doebuild(). msg = " === (%s of %s) Cleaning (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit)
class Binpkg(CompositeTask):
__slots__ = ("find_blockers",
"ldpath_mtimes", "logger", "opts",
"pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \
("_bintree", "_build_dir", "_build_prefix",
"_ebuild_path", "_fetched_pkg",
"_image_dir", "_infloc", "_pkg_path", "_tree", "_verify")
def _writemsg_level(self, msg, level=0, noiselevel=0):
self.scheduler.output(msg,
level=level,
noiselevel=noiselevel,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path,
'setup',
settings=self.settings,
db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
pkg = self.pkg
pkg_count = self.pkg_count
if not (self.opts.pretend or self.opts.fetchonly):
self._build_dir.lock()
# Initialize PORTAGE_LOG_FILE (clean_log won't work without it).
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
# If necessary, discard old log so that we don't
# append to it.
self._build_dir.clean_log()
fetcher = BinpkgFetcher(background=self.background,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
pkg=self.pkg,
pretend=self.opts.pretend,
scheduler=self.scheduler)
if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\
(pkg_count.curval, pkg_count.maxval, pkg.cpv,
fetcher.pkg_path)
short_msg = "emerge: (%s of %s) %s Fetch" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
# Allow the Scheduler's fetch queue to control the
# number of concurrent fetchers.
fetcher.addExitListener(self._fetcher_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher)
return
self._fetcher_exit(fetcher)
def _fetcher_exit(self, fetcher):
# The fetcher only has a returncode when
# --getbinpkg is enabled.
if fetcher.returncode is not None:
self._fetched_pkg = fetcher.pkg_path
if self._default_exit(fetcher) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
if self.opts.pretend:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
verifier = None
if self._verify:
if self._fetched_pkg:
path = self._fetched_pkg
else:
path = self.pkg.root_config.trees["bintree"].getname(
self.pkg.cpv)
logfile = self.settings.get("PORTAGE_LOG_FILE")
verifier = BinpkgVerifier(background=self.background,
logfile=logfile,
pkg=self.pkg,
scheduler=self.scheduler,
_pkg_path=path)
self._start_task(verifier, self._verifier_exit)
return
self._verifier_exit(verifier)
def _verifier_exit(self, verifier):
if verifier is not None and \
self._default_exit(verifier) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
logger = self.logger
pkg = self.pkg
pkg_count = self.pkg_count
if self._fetched_pkg:
pkg_path = self._bintree.getname(self._bintree.inject(
pkg.cpv, filename=self._fetched_pkg),
allocate_new=False)
else:
pkg_path = self.pkg.root_config.trees["bintree"].getname(
self.pkg.cpv)
# This gives bashrc users an opportunity to do various things
# such as remove binary packages after they're installed.
self.settings["PORTAGE_BINPKG_FILE"] = pkg_path
self._pkg_path = pkg_path
logfile = self.settings.get("PORTAGE_LOG_FILE")
if logfile is not None and os.path.isfile(logfile):
# Remove fetch log after successful fetch.
try:
os.unlink(logfile)
except OSError:
pass
if self.opts.fetchonly:
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
msg = " === (%s of %s) Merging Binary (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path)
short_msg = "emerge: (%s of %s) %s Merge Binary" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
phase = "clean"
settings = self.settings
ebuild_phase = EbuildPhase(background=self.background,
phase=phase,
scheduler=self.scheduler,
settings=settings)
self._start_task(ebuild_phase, self._clean_exit)
def _clean_exit(self, clean_phase):
if self._default_exit(clean_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
dir_path = self.settings['PORTAGE_BUILDDIR']
infloc = self._infloc
pkg = self.pkg
pkg_path = self._pkg_path
dir_mode = 0o755
for mydir in (dir_path, self._image_dir, infloc):
portage.util.ensure_dirs(mydir,
uid=portage.data.portage_uid,
gid=portage.data.portage_gid,
mode=dir_mode)
# This initializes PORTAGE_LOG_FILE.
portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1)
self._writemsg_level(">>> Extracting info\n")
pkg_xpak = portage.xpak.tbz2(self._pkg_path)
check_missing_metadata = ("CATEGORY", "PF")
missing_metadata = set()
for k in check_missing_metadata:
v = pkg_xpak.getfile(
_unicode_encode(k, encoding=_encodings['repo.content']))
if not v:
missing_metadata.add(k)
pkg_xpak.unpackinfo(infloc)
for k in missing_metadata:
if k == "CATEGORY":
v = pkg.category
elif k == "PF":
v = pkg.pf
else:
continue
f = io.open(_unicode_encode(os.path.join(infloc, k),
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['content'],
errors='backslashreplace')
try:
f.write(_unicode_decode(v + "\n"))
finally:
f.close()
# Store the md5sum in the vdb.
f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'),
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['content'],
errors='strict')
try:
f.write(
_unicode_decode(
str(portage.checksum.perform_md5(pkg_path)) + "\n"))
finally:
f.close()
env_extractor = BinpkgEnvExtractor(background=self.background,
scheduler=self.scheduler,
settings=self.settings)
self._start_task(env_extractor, self._env_extractor_exit)
def _env_extractor_exit(self, env_extractor):
if self._default_exit(env_extractor) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
setup_phase = EbuildPhase(background=self.background,
phase="setup",
scheduler=self.scheduler,
settings=self.settings)
setup_phase.addExitListener(self._setup_exit)
self._task_queued(setup_phase)
self.scheduler.scheduleSetup(setup_phase)
def _setup_exit(self, setup_phase):
if self._default_exit(setup_phase) != os.EX_OK:
self._unlock_builddir()
self.wait()
return
extractor = BinpkgExtractorAsync(
background=self.background,
env=self.settings.environ(),
features=self.settings.features,
image_dir=self._image_dir,
pkg=self.pkg,
pkg_path=self._pkg_path,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
scheduler=self.scheduler)
self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv)
self._start_task(extractor, self._extractor_exit)
def _extractor_exit(self, extractor):
if self._default_exit(extractor) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Extracting '%s'\n" % \
self._pkg_path, noiselevel=-1, level=logging.ERROR)
self.wait()
return
try:
with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings['fs'],
errors='strict'),
mode='r',
encoding=_encodings['repo.content'],
errors='replace') as f:
self._build_prefix = f.read().rstrip('\n')
except IOError:
self._build_prefix = ""
if self._build_prefix == self.settings["EPREFIX"]:
ensure_dirs(self.settings["ED"])
self._current_task = None
self.returncode = os.EX_OK
self.wait()
return
env = self.settings.environ()
env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"]
chpathtool = SpawnProcess(
args=[
portage._python_interpreter,
os.path.join(self.settings["PORTAGE_BIN_PATH"],
"chpathtool.py"), self.settings["D"],
self._build_prefix, self.settings["EPREFIX"]
],
background=self.background,
env=env,
scheduler=self.scheduler,
logfile=self.settings.get('PORTAGE_LOG_FILE'))
self._writemsg_level(">>> Adjusting Prefix to %s\n" %
self.settings["EPREFIX"])
self._start_task(chpathtool, self._chpathtool_exit)
def _chpathtool_exit(self, chpathtool):
if self._final_exit(chpathtool) != os.EX_OK:
self._unlock_builddir()
self._writemsg_level("!!! Error Adjusting Prefix to %s\n" %
(self.settings["EPREFIX"], ),
noiselevel=-1,
level=logging.ERROR)
self.wait()
return
# We want to install in "our" prefix, not the binary one
with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"),
encoding=_encodings['fs'],
errors='strict'),
mode='w',
encoding=_encodings['repo.content'],
errors='strict') as f:
f.write(self.settings["EPREFIX"] + "\n")
# Move the files to the correct location for merge.
image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"],
"image_tmp")
build_d = os.path.join(self.settings["D"],
self._build_prefix.lstrip(os.sep))
if not os.path.isdir(build_d):
# Assume this is a virtual package or something.
shutil.rmtree(self._image_dir)
ensure_dirs(self.settings["ED"])
else:
os.rename(build_d, image_tmp_dir)
shutil.rmtree(self._image_dir)
ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep)))
os.rename(image_tmp_dir, self.settings["ED"])
self.wait()
def _unlock_builddir(self):
if self.opts.pretend or self.opts.fetchonly:
return
portage.elog.elog_process(self.pkg.cpv, self.settings)
self._build_dir.unlock()
def create_install_task(self):
task = EbuildMerge(exit_hook=self._install_exit,
find_blockers=self.find_blockers,
ldpath_mtimes=self.ldpath_mtimes,
logger=self.logger,
pkg=self.pkg,
pkg_count=self.pkg_count,
pkg_path=self._pkg_path,
scheduler=self.scheduler,
settings=self.settings,
tree=self._tree,
world_atom=self.world_atom)
return task
def _install_exit(self, task):
self.settings.pop("PORTAGE_BINPKG_FILE", None)
self._unlock_builddir()
if task.returncode == os.EX_OK and \
'binpkg-logs' not in self.settings.features and \
self.settings.get("PORTAGE_LOG_FILE"):
try:
os.unlink(self.settings["PORTAGE_LOG_FILE"])
except OSError:
pass
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
with open(os.path.join(
cgroup_portage, 'release_agent'), 'w') as f:
f.write(os.path.join(self.settings['PORTAGE_BIN_PATH'],
'cgroup-release-agent'))
with open(os.path.join(
cgroup_portage, 'notify_on_release'), 'w') as f:
f.write('1')
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(
cgroup_portage, 'release_agent')
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip('\n')
except EnvironmentError:
release_agent_path = None
if (release_agent_path is None or
not os.path.exists(release_agent_path)):
with open(release_agent, 'w') as f:
f.write(os.path.join(
self.settings['PORTAGE_BIN_PATH'],
'cgroup-release-agent'))
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon))
return
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def testIpcDaemon(self):
event_loop = global_event_loop()
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
build_dir = EbuildBuildDir(
scheduler=event_loop,
settings=env)
build_dir.lock()
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit' : exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode],
env=env)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2, event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for EventLoop/AsyncScheduler.
# Use a ridiculously long sleep_time_s in case the user's
# system is heavily loaded (see bug #436334).
sleep_time_s = 600 #600.000 seconds
short_timeout_ms = 10 # 0.010 seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit' : exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SleepProcess(seconds=sleep_time_s)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2, event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, short_timeout_ms)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
build_dir.unlock()
shutil.rmtree(tmpdir)
def testIpcDaemon(self):
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
task_scheduler = TaskScheduler(max_jobs=2)
build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface,
settings=env)
build_dir.lock()
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for QueueScheduler.run()
sleep_time_s = 10 # 10.000 seconds
short_timeout_ms = 10 # 0.010 seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
'exec sleep %d' % sleep_time_s],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=short_timeout_ms)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
build_dir.unlock()
shutil.rmtree(tmpdir)
