class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_build_prefix", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output(msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return if not (self.opts.pretend or self.opts.fetchonly): self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_fetcher) else: self._start_fetcher() def _start_fetcher(self, lock_task=None): if lock_task is not None: self._assert_current(lock_task) lock_task.future.result() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() pkg = self.pkg pkg_count = self.pkg_count fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, fetcher.pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetcher_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = fetcher.pkg_path if self._default_exit(fetcher) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: if self._fetched_pkg: path = self._fetched_pkg else: path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, _pkg_path=path) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count if self._fetched_pkg: pkg_path = self._bintree.getname(self._bintree.inject( pkg.cpv, filename=self._fetched_pkg), allocate_new=False) else: pkg_path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. self.settings["PORTAGE_BINPKG_FILE"] = pkg_path self._pkg_path = pkg_path logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=settings) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return dir_path = self.settings['PORTAGE_BUILDDIR'] infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile( _unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = io.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='backslashreplace') try: f.write(_unicode_decode(v + "\n")) finally: f.close() # Store the md5sum in the vdb. f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write( _unicode_decode( str(portage.checksum.perform_md5(pkg_path)) + "\n")) finally: f.close() env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) self._start_task(env_extractor, self._env_extractor_exit) def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return setup_phase = EbuildPhase(background=self.background, phase="setup", scheduler=self.scheduler, settings=self.settings) setup_phase.addExitListener(self._setup_exit) self._task_queued(setup_phase) self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return extractor = BinpkgExtractorAsync( background=self.background, env=self.settings.environ(), features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._default_exit(extractor) != os.EX_OK: self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self._async_unlock_builddir(returncode=self.returncode) return try: with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._build_prefix = f.read().rstrip('\n') except IOError: self._build_prefix = "" if self._build_prefix == self.settings["EPREFIX"]: ensure_dirs(self.settings["ED"]) self._current_task = None self.returncode = os.EX_OK self.wait() return env = self.settings.environ() env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[ portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"] ], background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) self._start_task(chpathtool, self._chpathtool_exit) def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._writemsg_level("!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"], ), noiselevel=-1, level=logging.ERROR) self._async_unlock_builddir(returncode=self.returncode) return # We want to install in "our" prefix, not the binary one with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(self.settings["EPREFIX"] + "\n") # Move the files to the correct location for merge. image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"], "image_tmp") build_d = os.path.join(self.settings["D"], self._build_prefix.lstrip(os.sep)) if not os.path.isdir(build_d): # Assume this is a virtual package or something. shutil.rmtree(self._image_dir) ensure_dirs(self.settings["ED"]) else: os.rename(build_d, image_tmp_dir) shutil.rmtree(self._image_dir) ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) os.rename(image_tmp_dir, self.settings["ED"]) self.wait() def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self.opts.pretend or self.opts.fetchonly: if returncode is not None: self.returncode = returncode self._async_wait() return if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) # Normally, async_unlock should not raise an exception here. unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def create_install_task(self): task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self.settings.pop("PORTAGE_BINPKG_FILE", None) if task.returncode == os.EX_OK and \ 'binpkg-logs' not in self.settings.features and \ self.settings.get("PORTAGE_LOG_FILE"): try: os.unlink(self.settings["PORTAGE_LOG_FILE"]) except OSError: pass self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') env['PYTHONDONTWRITEBYTECODE'] = os.environ.get( 'PYTHONDONTWRITEBYTECODE', '') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir(scheduler=event_loop, settings=env) event_loop.run_until_complete(build_dir.async_lock()) ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 # seconds short_timeout_s = 0.010 # seconds for i in range(3): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_s) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: event_loop.run_until_complete(build_dir.async_unlock()) shutil.rmtree(tmpdir)
class AbstractEbuildProcess(SpawnProcess): __slots__ = ( "phase", "settings", ) + ( "_build_dir", "_build_dir_unlock", "_ipc_daemon", "_exit_command", "_exit_timeout_id", "_start_future", ) _phases_without_builddir = ( "clean", "cleanrm", "depend", "help", ) _phases_interactive_whitelist = ("config", ) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10 # seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = "other" self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and not os.path.isdir( self.settings["PORTAGE_BUILDDIR"]): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % ( self.phase, self.settings["PORTAGE_BUILDDIR"]) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == "Linux" and "cgroup" in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = "/sys/fs/cgroup" cgroup_portage = os.path.join(cgroup_root, "portage") try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ "mount", "-t", "tmpfs", "-o", "rw,nosuid,nodev,noexec,mode=0755", "tmpfs", cgroup_root, ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ "mount", "-t", "cgroup", "-o", "rw,nosuid,nodev,noexec,none,name=portage", "tmpfs", cgroup_portage, ]) with open(os.path.join(cgroup_portage, "release_agent"), "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) with open( os.path.join(cgroup_portage, "notify_on_release"), "w") as f: f.write("1") else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join(cgroup_portage, "release_agent") try: with open(release_agent) as f: release_agent_path = f.readline().rstrip("\n") except EnvironmentError: release_agent_path = None if release_agent_path is None or not os.path.exists( release_agent_path): with open(release_agent, "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix="%s:%s." % (self.settings["CATEGORY"], self.settings["PF"]), ) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings["NOCOLOR"] = "true" start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial( self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon, )) return else: self.settings.pop("PORTAGE_IPC_DAEMON", None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop("PORTAGE_IPC_DAEMON", None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".exit_status") self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon) def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False): if lock_future is not None: if lock_future is not self._start_future: raise AssertionError("lock_future is not self._start_future") self._start_future = None if lock_future.cancelled(): self._build_dir = None self.cancelled = True self._was_cancelled() self._async_wait() return lock_future.result() if start_ipc_daemon: self.settings["PORTAGE_IPC_DAEMON"] = "1" self._start_ipc_daemon() if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if (0 not in self.fd_pipes and self.phase not in self._phases_interactive_whitelist and "interactive" not in self.settings.get("PROPERTIES", "").split()): null_fd = os.open("/dev/null", os.O_RDONLY) self.fd_pipes[0] = null_fd self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in") output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_out") for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions( p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st, ) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { "available_eclasses": query_command, "best_version": query_command, "eclass_path": query_command, "exit": self._exit_command, "has_version": query_command, "license_path": query_command, "master_repositories": query_command, "repository_path": query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon( commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler, ) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = self.scheduler.call_later( self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = self.scheduler.call_later( self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None def _cancel_timeout_cb(self): self._exit_timeout_id = None self._async_waitpid() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = _create_pty_or_pipe( copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ("sesandbox" in self.settings.features and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % ( self.phase, signum, ) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = (_( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase) self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog("eerror", lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = self.settings.get( "NOCOLOR", "false").lower() in ("no", "false") for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _async_waitpid_cb(self, *args, **kwargs): """ Override _async_waitpid_cb to perform cleanup that is not necessarily idempotent. """ SpawnProcess._async_waitpid_cb(self, *args, **kwargs) if self._exit_timeout_id is not None: self._exit_timeout_id.cancel() self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() elif not self.cancelled: exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE") if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir when necessary. """ if self._build_dir is None: SpawnProcess._async_wait(self) elif self._build_dir_unlock is None: if self.returncode is None: raise asyncio.InvalidStateError("Result is not ready for %s" % (self, )) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self._build_dir_unlock is not None: raise AssertionError("unlock already in progress") if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._build_dir_unlock = self._build_dir.async_unlock() # Unlock only once. self._build_dir = None self._build_dir_unlock.add_done_callback( functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_future, returncode=None): # Normally, async_unlock should not raise an exception here. unlock_future.cancelled() or unlock_future.result() if returncode is not None: if unlock_future.cancelled(): self.cancelled = True self._was_cancelled() else: self.returncode = returncode SpawnProcess._async_wait(self)
class PackageUninstall(CompositeTask): """ Uninstall a package asynchronously in a subprocess. When both parallel-install and ebuild-locks FEATURES are enabled, it is essential for the ebuild-locks code to execute in a subprocess, since the portage.locks module does not behave as desired if we try to lock the same file multiple times concurrently from the same process for ebuild-locks phases such as pkg_setup, pkg_prerm, and pkg_postrm. """ __slots__ = ( "world_atom", "ldpath_mtimes", "opts", "pkg", "settings", "_builddir_lock", ) def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge, ) def _start_unmerge(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() portage.prepare_build_dirs(settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, _ = _unmerge_display( self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level, ) if retval != os.EX_OK: self._async_unlock_builddir(returncode=retval) return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, )) cat, pf = portage.catsplit(self.pkg.cpv) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True, ) self._start_task(unmerge_task, self._unmerge_exit) def _unmerge_exit(self, unmerge_task): if self._final_exit(unmerge_task) != os.EX_OK: self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv, )) else: self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv, )) self.world_atom(self.pkg) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode), ) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _emergelog(self, msg): emergelog("notitles" not in self.settings.features, msg) def _writemsg_level(self, msg, level=0, noiselevel=0): log_path = self.settings.get("PORTAGE_LOG_FILE") background = self.background if log_path is None: if not (background and level < logging.WARNING): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: self.scheduler.output(msg, log_path=log_path, level=level, noiselevel=noiselevel)
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): if not self.opts.fetchonly: rval = _check_temp_dir(self.settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return # First get the SRC_URI metadata (it's not cached in self.pkg.metadata # because some packages have an extremely large SRC_URI value). self._start_task( AsyncTaskFuture( future=self.pkg.root_config.trees["porttree"].dbapi.\ async_aux_get(self.pkg.cpv, ["SRC_URI"], myrepo=self.pkg.repo, loop=self.scheduler)), self._start_with_metadata) def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) if aux_get_task.cancelled: self._default_final_exit(aux_get_task) return pkg = self.pkg settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task( SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[ self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean) def _start_pre_clean(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: self.returncode = None portdb = self.pkg.root_config.trees[self._tree].dbapi self._start_task( SpawnNofetchWithoutBuilddir(background=self.background, portdb=portdb, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._nofetch_without_builddir_exit) return self.wait() def _nofetch_without_builddir_exit(self, nofetch): self._final_exit(nofetch) self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) self._start_task( AsyncTaskFuture( future=fetcher.async_already_fetched(self.settings)), functools.partial(self._start_fetch, fetcher)) def _start_fetch(self, fetcher, already_fetched_task): self._assert_current(already_fetched_task) if already_fetched_task.cancelled: self._default_final_exit(already_fetched_task) return try: already_fetched = already_fetched_task.future.result() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self._async_unlock_builddir(returncode=1) return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._async_unlock_builddir(returncode=self.returncode) return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._async_unlock_builddir(returncode=1) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add( EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: task = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) binpkg_tasks.add(task) # Guarantee that _record_binpkg_info is called # immediately after EbuildBinpkg. Note that # task.addExitListener does not provide the # necessary guarantee (see bug 578204). binpkg_tasks.add( self._RecordBinpkgInfo(ebuild_binpkg=task, ebuild_build=self)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() class _RecordBinpkgInfo(AsynchronousTask): """ This class wraps the EbuildBuild _record_binpkg_info method with an AsynchronousTask interface, so that it can be scheduled as a member of a TaskSequence. """ __slots__ = ( 'ebuild_binpkg', 'ebuild_build', ) def _start(self): self.ebuild_build._record_binpkg_info(self.ebuild_binpkg) AsynchronousTask._start(self) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess(background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _record_binpkg_info(self, task): if task.returncode != os.EX_OK: return # Save info about the created binary package, so that # identifying information can be passed to the install # task, to be recorded in the installed package database. pkg = task.get_binpkg_info() infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info") info = { "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], } if pkg.build_id is not None: info["BUILD_ID"] = "%s\n" % pkg.build_id for k, v in info.items(): with io.open(_unicode_encode(os.path.join(infoloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(v) def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._async_unlock_builddir(returncode=self.returncode) else: self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_build_dir_unlock', '_ipc_daemon', '_exit_command', '_exit_timeout_id', '_start_future') _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) _phases_interactive_whitelist = ('config',) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10 # seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call(['mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call(['mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage]) with open(os.path.join( cgroup_portage, 'release_agent'), 'w') as f: f.write(os.path.join(self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) with open(os.path.join( cgroup_portage, 'notify_on_release'), 'w') as f: f.write('1') else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join( cgroup_portage, 'release_agent') try: with open(release_agent) as f: release_agent_path = f.readline().rstrip('\n') except EnvironmentError: release_agent_path = None if (release_agent_path is None or not os.path.exists(release_agent_path)): with open(release_agent, 'w') as f: f.write(os.path.join( self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) cgroup_path = tempfile.mkdtemp(dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial(self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon)) return else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon) def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False): if lock_future is not None: if lock_future is not self._start_future: raise AssertionError('lock_future is not self._start_future') self._start_future = None if lock_future.cancelled(): self._build_dir = None self.cancelled = True self._was_cancelled() self._async_wait() return lock_future.result() if start_ipc_daemon: self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses' : query_command, 'best_version' : query_command, 'eclass_path' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, 'license_path' : query_command, 'master_repositories' : query_command, 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = \ self.scheduler.call_later(self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = \ self.scheduler.call_later(self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None def _cancel_timeout_cb(self): self._exit_timeout_id = None self._async_waitpid() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = _("The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog('eerror', lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = \ self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false') for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _async_waitpid_cb(self, *args, **kwargs): """ Override _async_waitpid_cb to perform cleanup that is not necessarily idempotent. """ SpawnProcess._async_waitpid_cb(self, *args, **kwargs) if self._exit_timeout_id is not None: self._exit_timeout_id.cancel() self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() elif not self.cancelled: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir when necessary. """ if self._build_dir is None: SpawnProcess._async_wait(self) elif self._build_dir_unlock is None: if self.returncode is None: raise asyncio.InvalidStateError('Result is not ready.') self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self._build_dir_unlock is not None: raise AssertionError('unlock already in progress') if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._build_dir_unlock = self._build_dir.async_unlock() # Unlock only once. self._build_dir = None self._build_dir_unlock.add_done_callback( functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_future, returncode=None): # Normally, async_unlock should not raise an exception here. unlock_future.cancelled() or unlock_future.result() if returncode is not None: if unlock_future.cancelled(): self.cancelled = True self._was_cancelled() else: self.returncode = returncode SpawnProcess._async_wait(self)
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') env['PYTHONDONTWRITEBYTECODE'] = os.environ.get('PYTHONDONTWRITEBYTECODE', '') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir( scheduler=event_loop, settings=env) event_loop.run_until_complete(build_dir.async_lock()) ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess( args=[BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 # seconds short_timeout_s = 0.010 # seconds for i in range(3): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_s) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: event_loop.run_until_complete(build_dir.async_unlock()) shutil.rmtree(tmpdir)
class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_build_prefix", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output(msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return if not (self.opts.pretend or self.opts.fetchonly): self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_fetcher) else: self._start_fetcher() def _start_fetcher(self, lock_task=None): if lock_task is not None: self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() pkg = self.pkg pkg_count = self.pkg_count fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, fetcher.pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetcher_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = fetcher.pkg_path if self._default_exit(fetcher) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: if self._fetched_pkg: path = self._fetched_pkg else: path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, _pkg_path=path) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count if self._fetched_pkg: pkg_path = self._bintree.getname( self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), allocate_new=False) else: pkg_path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. self.settings["PORTAGE_BINPKG_FILE"] = pkg_path self._pkg_path = pkg_path logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=settings) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return dir_path = self.settings['PORTAGE_BUILDDIR'] infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile(_unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = io.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='backslashreplace') try: f.write(_unicode_decode(v + "\n")) finally: f.close() # Store the md5sum in the vdb. f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write(_unicode_decode( str(portage.checksum.perform_md5(pkg_path)) + "\n")) finally: f.close() env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) self._start_task(env_extractor, self._env_extractor_exit) def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return setup_phase = EbuildPhase(background=self.background, phase="setup", scheduler=self.scheduler, settings=self.settings) setup_phase.addExitListener(self._setup_exit) self._task_queued(setup_phase) self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return extractor = BinpkgExtractorAsync(background=self.background, env=self.settings.environ(), features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._default_exit(extractor) != os.EX_OK: self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self._async_unlock_builddir(returncode=self.returncode) return try: with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._build_prefix = f.read().rstrip('\n') except IOError: self._build_prefix = "" if self._build_prefix == self.settings["EPREFIX"]: ensure_dirs(self.settings["ED"]) self._current_task = None self.returncode = os.EX_OK self.wait() return env = self.settings.environ() env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"]], background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) self._start_task(chpathtool, self._chpathtool_exit) def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._writemsg_level("!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],), noiselevel=-1, level=logging.ERROR) self._async_unlock_builddir(returncode=self.returncode) return # We want to install in "our" prefix, not the binary one with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(self.settings["EPREFIX"] + "\n") # Move the files to the correct location for merge. image_tmp_dir = os.path.join( self.settings["PORTAGE_BUILDDIR"], "image_tmp") build_d = os.path.join(self.settings["D"], self._build_prefix.lstrip(os.sep)) if not os.path.isdir(build_d): # Assume this is a virtual package or something. shutil.rmtree(self._image_dir) ensure_dirs(self.settings["ED"]) else: os.rename(build_d, image_tmp_dir) shutil.rmtree(self._image_dir) ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) os.rename(image_tmp_dir, self.settings["ED"]) self.wait() def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self.opts.pretend or self.opts.fetchonly: if returncode is not None: self.returncode = returncode self._async_wait() return if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def create_install_task(self): task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self.settings.pop("PORTAGE_BINPKG_FILE", None) if task.returncode == os.EX_OK and \ 'binpkg-logs' not in self.settings.features and \ self.settings.get("PORTAGE_LOG_FILE"): try: os.unlink(self.settings["PORTAGE_LOG_FILE"]) except OSError: pass self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result
class PackageUninstall(CompositeTask): """ Uninstall a package asynchronously in a subprocess. When both parallel-install and ebuild-locks FEATURES are enabled, it is essential for the ebuild-locks code to execute in a subprocess, since the portage.locks module does not behave as desired if we try to lock the same file multiple times concurrently from the same process for ebuild-locks phases such as pkg_setup, pkg_prerm, and pkg_postrm. """ __slots__ = ("world_atom", "ldpath_mtimes", "opts", "pkg", "settings", "_builddir_lock") def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge) def _start_unmerge(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() portage.prepare_build_dirs( settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._async_unlock_builddir(returncode=retval) return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,)) cat, pf = portage.catsplit(self.pkg.cpv) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit) def _unmerge_exit(self, unmerge_task): if self._final_exit(unmerge_task) != os.EX_OK: self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv,)) else: self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv,)) self.world_atom(self.pkg) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _emergelog(self, msg): emergelog("notitles" not in self.settings.features, msg) def _writemsg_level(self, msg, level=0, noiselevel=0): log_path = self.settings.get("PORTAGE_LOG_FILE") background = self.background if log_path is None: if not (background and level < logging.WARNING): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: self.scheduler.output(msg, log_path=log_path, level=level, noiselevel=noiselevel)
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): if not self.opts.fetchonly: rval = _check_temp_dir(self.settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return # First get the SRC_URI metadata (it's not cached in self.pkg.metadata # because some packages have an extremely large SRC_URI value). self._start_task( AsyncTaskFuture( future=self.pkg.root_config.trees["porttree"].dbapi.\ async_aux_get(self.pkg.cpv, ["SRC_URI"], myrepo=self.pkg.repo, loop=self.scheduler)), self._start_with_metadata) def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) if aux_get_task.cancelled: self._default_final_exit(aux_get_task) return pkg = self.pkg settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean) def _start_pre_clean(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: self.returncode = None portdb = self.pkg.root_config.trees[self._tree].dbapi self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=portdb, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._nofetch_without_builddir_exit) return self.wait() def _nofetch_without_builddir_exit(self, nofetch): self._final_exit(nofetch) self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) self._start_task(AsyncTaskFuture( future=fetcher.async_already_fetched(self.settings)), functools.partial(self._start_fetch, fetcher)) def _start_fetch(self, fetcher, already_fetched_task): self._assert_current(already_fetched_task) if already_fetched_task.cancelled: self._default_final_exit(already_fetched_task) return try: already_fetched = already_fetched_task.future.result() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self._async_unlock_builddir(returncode=1) return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._async_unlock_builddir(returncode=self.returncode) return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._async_unlock_builddir(returncode=1) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add(EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: task = EbuildBinpkg( background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) binpkg_tasks.add(task) # Guarantee that _record_binpkg_info is called # immediately after EbuildBinpkg. Note that # task.addExitListener does not provide the # necessary guarantee (see bug 578204). binpkg_tasks.add(self._RecordBinpkgInfo( ebuild_binpkg=task, ebuild_build=self)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() class _RecordBinpkgInfo(AsynchronousTask): """ This class wraps the EbuildBuild _record_binpkg_info method with an AsynchronousTask interface, so that it can be scheduled as a member of a TaskSequence. """ __slots__ = ('ebuild_binpkg', 'ebuild_build',) def _start(self): self.ebuild_build._record_binpkg_info(self.ebuild_binpkg) AsynchronousTask._start(self) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _record_binpkg_info(self, task): if task.returncode != os.EX_OK: return # Save info about the created binary package, so that # identifying information can be passed to the install # task, to be recorded in the installed package database. pkg = task.get_binpkg_info() infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info") info = { "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], } if pkg.build_id is not None: info["BUILD_ID"] = "%s\n" % pkg.build_id for k, v in info.items(): with io.open(_unicode_encode(os.path.join(infoloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(v) def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._async_unlock_builddir(returncode=self.returncode) else: self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result