def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge, )
def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit)
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend dir_path = os.path.join(settings["PORTAGE_TMPDIR"], "portage", pkg.category, pkg.pf) self._build_dir = EbuildBuildDir(dir_path=dir_path, pkg=pkg, settings=settings) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path debug = settings.get("PORTAGE_DEBUG") == "1" portage.doebuild_environment(self._ebuild_path, "setup", settings["ROOT"], settings, debug, 1, self._bintree.dbapi) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal.") % prefetcher.pkg_path msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task( SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[ self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean)
def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._builddir_lock.lock() portage.prepare_build_dirs( settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,)) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit)
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self.wait() return if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) SpawnProcess._start(self)
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean)
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return else: fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean)
class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): if not self.background: portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path is not None: f = codecs.open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') try: f.write(msg) finally: f.close() def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend dir_path = os.path.join(settings["PORTAGE_TMPDIR"], "portage", pkg.category, pkg.pf) self._build_dir = EbuildBuildDir(dir_path=dir_path, pkg=pkg, settings=settings) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path debug = settings.get("PORTAGE_DEBUG") == "1" portage.doebuild_environment(self._ebuild_path, "setup", settings["ROOT"], settings, debug, 1, self._bintree.dbapi) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal.") % prefetcher.pkg_path msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): pkg = self.pkg pkg_count = self.pkg_count if not (self.opts.pretend or self.opts.fetchonly): self._build_dir.lock() # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() # Initialze PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) pkg_path = fetcher.pkg_path self._pkg_path = pkg_path if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) self._start_task(fetcher, self._fetcher_exit) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = True if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: logfile = None if self.background: logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._unlock_builddir() self.wait() return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count pkg_path = self._pkg_path if self._fetched_pkg: self._bintree.inject(pkg.cpv, filename=pkg_path) logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, pkg=pkg, phase=phase, scheduler=self.scheduler, settings=settings, tree=self._tree) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return dir_path = self._build_dir.dir_path infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile(_unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = codecs.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='replace') try: f.write(v + "\n") finally: f.close() # Store the md5sum in the vdb. f = codecs.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n") finally: f.close() # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. settings = self.settings settings.setcpv(self.pkg) settings["PORTAGE_BINPKG_FILE"] = pkg_path settings.backup_changes("PORTAGE_BINPKG_FILE") phase = "setup" setup_phase = EbuildPhase(background=self.background, pkg=self.pkg, phase=phase, scheduler=self.scheduler, settings=settings, tree=self._tree) setup_phase.addExitListener(self._setup_exit) self._current_task = setup_phase self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._unlock_builddir() self.wait() return extractor = BinpkgExtractorAsync(background=self.background, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._final_exit(extractor) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self.wait() def _unlock_builddir(self): if self.opts.pretend or self.opts.fetchonly: return portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def install(self): # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. settings = self.settings settings["PORTAGE_BINPKG_FILE"] = self._pkg_path settings.backup_changes("PORTAGE_BINPKG_FILE") merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=settings, tree=self._tree, world_atom=self.world_atom) try: retval = merge.execute() finally: settings.pop("PORTAGE_BINPKG_FILE", None) self._unlock_builddir() return retval
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): logger = self.logger opts = self.opts pkg = self.pkg settings = self.settings world_atom = self.world_atom root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name ebuild_path = portdb.findname(pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return fetcher = EbuildFetcher(config_pool=self.config_pool, fetchall=opts.fetch_all_uri, fetchonly=opts.fetchonly, background=self.background, pkg=pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetch_exit) def _fetch_exit(self, fetcher): opts = self.opts pkg = self.pkg fetch_failed = False if opts.fetchonly: fetch_failed = self._final_exit(fetcher) != os.EX_OK else: fetch_failed = self._default_exit(fetcher) != os.EX_OK if fetch_failed and fetcher.logfile is not None and \ os.path.exists(fetcher.logfile): self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile if fetch_failed or opts.fetchonly: self.wait() return logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning is triggered before the setup # phase, in portage.doebuild(). msg = " === (%s of %s) Cleaning (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return opts = self.opts buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path is not None: log_file = codecs.open(_unicode_encode( log_path, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') try: log_file.write(msg) finally: log_file.close() if not self.background: portage.writemsg_stdout(msg, noiselevel=-1) packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(packager, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess(background=self.background, commands=[phase], phase=phase, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, pkg=self.pkg, phase=phase, scheduler=self.scheduler, settings=self.settings, tree=self._tree) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def install(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ find_blockers = self.find_blockers ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) try: rval = merge.execute() finally: self._unlock_builddir() return rval
def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) settings = self.config_pool.allocate() settings.setcpv(self.pkg) if self.prefetch and \ self._prefetch_size_ok(portdb, settings, ebuild_path): self.config_pool.deallocate(settings) self.returncode = os.EX_OK self.wait() return # In prefetch mode, logging goes to emerge-fetch.log and the builddir # should not be touched since otherwise it could interfere with # another instance of the same cpv concurrently being built for a # different $ROOT (currently, builds only cooperate with prefetchers # that are spawned for the same $ROOT). if not self.prefetch: self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings) self._build_dir.lock() self._build_dir.clean_log() cleanup=1 # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, cleanup) if self.logfile is None: self.logfile = settings.get("PORTAGE_LOG_FILE") phase = "fetch" if self.fetchall: phase = "fetchall" # If any incremental variables have been overridden # via the environment, those values need to be passed # along here so that they are correctly considered by # the config instance in the subproccess. fetch_env = os.environ.copy() fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT'] nocolor = settings.get("NOCOLOR") if nocolor is not None: fetch_env["NOCOLOR"] = nocolor fetch_env["PORTAGE_NICENESS"] = "0" if self.prefetch: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" ebuild_binary = os.path.join( settings["PORTAGE_BIN_PATH"], "ebuild") fetch_args = [ebuild_binary, ebuild_path, phase] debug = settings.get("PORTAGE_DEBUG") == "1" if debug: fetch_args.append("--debug") if not self.background and nocolor not in ('yes', 'true'): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. fetch_args.append('--color=y') self.args = fetch_args self.env = fetch_env if self._build_dir is None: # Free settings now since we only have a local reference. self.config_pool.deallocate(settings) SpawnProcess._start(self)
class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output(msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend dir_path = os.path.join(settings["PORTAGE_TMPDIR"], "portage", pkg.category, pkg.pf) self._build_dir = EbuildBuildDir(dir_path=dir_path, pkg=pkg, settings=settings) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path debug = settings.get("PORTAGE_DEBUG") == "1" portage.doebuild_environment(self._ebuild_path, "setup", settings["ROOT"], settings, debug, 1, self._bintree.dbapi) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal.") % prefetcher.pkg_path msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): pkg = self.pkg pkg_count = self.pkg_count if not (self.opts.pretend or self.opts.fetchonly): self._build_dir.lock() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) pkg_path = fetcher.pkg_path self._pkg_path = pkg_path if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) self._start_task(fetcher, self._fetcher_exit) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = True if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._unlock_builddir() self.wait() return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count pkg_path = self._pkg_path if self._fetched_pkg: self._bintree.inject(pkg.cpv, filename=pkg_path) logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=settings) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return dir_path = self._build_dir.dir_path infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile(_unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = codecs.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='replace') try: f.write(v + "\n") finally: f.close() # Store the md5sum in the vdb. f = codecs.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write(str(portage.checksum.perform_md5(pkg_path)) + "\n") finally: f.close() env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) self._start_task(env_extractor, self._env_extractor_exit) def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self._unlock_builddir() self.wait() return # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. settings = self.settings settings.setcpv(self.pkg) settings["PORTAGE_BINPKG_FILE"] = self._pkg_path settings.backup_changes("PORTAGE_BINPKG_FILE") setup_phase = EbuildPhase(background=self.background, phase="setup", scheduler=self.scheduler, settings=settings) setup_phase.addExitListener(self._setup_exit) self._current_task = setup_phase self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._unlock_builddir() self.wait() return extractor = BinpkgExtractorAsync(background=self.background, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._final_exit(extractor) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self.wait() def _unlock_builddir(self): if self.opts.pretend or self.opts.fetchonly: return portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def install(self): # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. settings = self.settings settings["PORTAGE_BINPKG_FILE"] = self._pkg_path settings.backup_changes("PORTAGE_BINPKG_FILE") merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=settings, tree=self._tree, world_atom=self.world_atom) try: retval = merge.execute() finally: settings.pop("PORTAGE_BINPKG_FILE", None) self._unlock_builddir() return retval
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): if not self.opts.fetchonly: rval = _check_temp_dir(self.settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return # First get the SRC_URI metadata (it's not cached in self.pkg.metadata # because some packages have an extremely large SRC_URI value). self._start_task( AsyncTaskFuture( future=self.pkg.root_config.trees["porttree"].dbapi.\ async_aux_get(self.pkg.cpv, ["SRC_URI"], myrepo=self.pkg.repo, loop=self.scheduler)), self._start_with_metadata) def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) if aux_get_task.cancelled: self._default_final_exit(aux_get_task) return pkg = self.pkg settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task( SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[ self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean) def _start_pre_clean(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: self.returncode = None portdb = self.pkg.root_config.trees[self._tree].dbapi self._start_task( SpawnNofetchWithoutBuilddir(background=self.background, portdb=portdb, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._nofetch_without_builddir_exit) return self.wait() def _nofetch_without_builddir_exit(self, nofetch): self._final_exit(nofetch) self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) self._start_task( AsyncTaskFuture( future=fetcher.async_already_fetched(self.settings)), functools.partial(self._start_fetch, fetcher)) def _start_fetch(self, fetcher, already_fetched_task): self._assert_current(already_fetched_task) if already_fetched_task.cancelled: self._default_final_exit(already_fetched_task) return try: already_fetched = already_fetched_task.future.result() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self._async_unlock_builddir(returncode=1) return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._async_unlock_builddir(returncode=self.returncode) return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._async_unlock_builddir(returncode=1) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add( EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: task = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) binpkg_tasks.add(task) # Guarantee that _record_binpkg_info is called # immediately after EbuildBinpkg. Note that # task.addExitListener does not provide the # necessary guarantee (see bug 578204). binpkg_tasks.add( self._RecordBinpkgInfo(ebuild_binpkg=task, ebuild_build=self)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() class _RecordBinpkgInfo(AsynchronousTask): """ This class wraps the EbuildBuild _record_binpkg_info method with an AsynchronousTask interface, so that it can be scheduled as a member of a TaskSequence. """ __slots__ = ( 'ebuild_binpkg', 'ebuild_build', ) def _start(self): self.ebuild_build._record_binpkg_info(self.ebuild_binpkg) AsynchronousTask._start(self) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess(background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _record_binpkg_info(self, task): if task.returncode != os.EX_OK: return # Save info about the created binary package, so that # identifying information can be passed to the install # task, to be recorded in the installed package database. pkg = task.get_binpkg_info() infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info") info = { "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], } if pkg.build_id is not None: info["BUILD_ID"] = "%s\n" % pkg.build_id for k, v in info.items(): with io.open(_unicode_encode(os.path.join(infoloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(v) def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._async_unlock_builddir(returncode=self.returncode) else: self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result
class EbuildFetcher(SpawnProcess): __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \ ("_build_dir",) def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) settings = self.config_pool.allocate() settings.setcpv(self.pkg) if self.prefetch and \ self._prefetch_size_ok(portdb, settings, ebuild_path): self.config_pool.deallocate(settings) self.returncode = os.EX_OK self.wait() return # In prefetch mode, logging goes to emerge-fetch.log and the builddir # should not be touched since otherwise it could interfere with # another instance of the same cpv concurrently being built for a # different $ROOT (currently, builds only cooperate with prefetchers # that are spawned for the same $ROOT). if not self.prefetch: self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings) self._build_dir.lock() self._build_dir.clean_log() portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, 0) if self.logfile is None: self.logfile = settings.get("PORTAGE_LOG_FILE") phase = "fetch" if self.fetchall: phase = "fetchall" # If any incremental variables have been overridden # via the environment, those values need to be passed # along here so that they are correctly considered by # the config instance in the subproccess. fetch_env = os.environ.copy() fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT'] nocolor = settings.get("NOCOLOR") if nocolor is not None: fetch_env["NOCOLOR"] = nocolor fetch_env["PORTAGE_NICENESS"] = "0" if self.prefetch: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" ebuild_binary = os.path.join( settings["PORTAGE_BIN_PATH"], "ebuild") fetch_args = [ebuild_binary, ebuild_path, phase] debug = settings.get("PORTAGE_DEBUG") == "1" if debug: fetch_args.append("--debug") if not self.background and nocolor not in ('yes', 'true'): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. fetch_args.append('--color=y') self.args = fetch_args self.env = fetch_env if self._build_dir is None: # Free settings now since we only have a local reference. self.config_pool.deallocate(settings) SpawnProcess._start(self) def _prefetch_size_ok(self, portdb, settings, ebuild_path): pkgdir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkgdir)) distdir = settings["DISTDIR"] use = None if not self.fetchall: use = self.pkg.use.enabled try: uri_map = portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree) except portage.exception.InvalidDependString as e: return False sizes = {} for filename in uri_map: # Use stat rather than lstat since portage.fetch() creates # symlinks when PORTAGE_RO_DISTDIRS is used. try: st = os.stat(os.path.join(distdir, filename)) except OSError: return False if st.st_size == 0: return False sizes[filename] = st.st_size digests = portage.Manifest(pkgdir, distdir).getTypeDigests("DIST") for filename, actual_size in sizes.items(): size = digests.get(filename, {}).get('size') if size is None: continue if size != actual_size: return False # All files are present and sizes are ok. In this case the normal # fetch code will be skipped, so we need to generate equivalent # output here. if self.logfile is not None: f = codecs.open(_unicode_encode(self.logfile, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') for filename in uri_map: f.write((' * %s size ;-) ...' % \ filename).ljust(73) + '[ ok ]\n') f.close() return True def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" if self.background or not sys.stdout.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) # Collect elog messages that might have been # created by the pkg_nofetch phase. if self._build_dir is not None: # Skip elog messages for prefetch, in order to avoid duplicates. if not self.prefetch and self.returncode != os.EX_OK: elog_out = None if self.logfile is not None: if self.background: elog_out = codecs.open(_unicode_encode(self.logfile, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') msg = "Fetch failed for '%s'" % (self.pkg.cpv,) if self.logfile is not None: msg += ", Log file:" eerror(msg, phase="unpack", key=self.pkg.cpv, out=elog_out) if self.logfile is not None: eerror(" '%s'" % (self.logfile,), phase="unpack", key=self.pkg.cpv, out=elog_out) if elog_out is not None: elog_out.close() if not self.prefetch: portage.elog.elog_process(self.pkg.cpv, self._build_dir.settings) features = self._build_dir.settings.features if self.returncode == os.EX_OK: self._build_dir.clean_log() self._build_dir.unlock() self.config_pool.deallocate(self._build_dir.settings) self._build_dir = None
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): pkg = self.pkg settings = self.settings if not self.opts.fetchonly: rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: self.returncode = None portdb = self.pkg.root_config.trees[self._tree].dbapi self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=portdb, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._nofetch_without_builddir_exit) return self.wait() def _nofetch_without_builddir_exit(self, nofetch): self._final_exit(nofetch) self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) try: already_fetched = fetcher.already_fetched(self.settings) except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self.returncode = 1 self._current_task = None self._unlock_builddir() self.wait() return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._unlock_builddir() self.wait() return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._unlock_builddir() self.returncode = 1 self.wait() def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add(EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: task = EbuildBinpkg( background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) binpkg_tasks.add(task) # Guarantee that _record_binpkg_info is called # immediately after EbuildBinpkg. Note that # task.addExitListener does not provide the # necessary guarantee (see bug 578204). binpkg_tasks.add(self._RecordBinpkgInfo( ebuild_binpkg=task, ebuild_build=self)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() class _RecordBinpkgInfo(AsynchronousTask): """ This class wraps the EbuildBuild _record_binpkg_info method with an AsynchronousTask interface, so that it can be scheduled as a member of a TaskSequence. """ __slots__ = ('ebuild_binpkg', 'ebuild_build',) def _start(self): self.ebuild_build._record_binpkg_info(self.ebuild_binpkg) AsynchronousTask._start(self) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _record_binpkg_info(self, task): if task.returncode != os.EX_OK: return # Save info about the created binary package, so that # identifying information can be passed to the install # task, to be recorded in the installed package database. pkg = task.get_binpkg_info() infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], "build-info") info = { "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], } if pkg.build_id is not None: info["BUILD_ID"] = "%s\n" % pkg.build_id for k, v in info.items(): with io.open(_unicode_encode(os.path.join(infoloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(v) def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): self._unlock_builddir()
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task(SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit)
def testIpcDaemon(self): tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') task_scheduler = TaskScheduler(max_jobs=2) build_dir = EbuildBuildDir( scheduler=task_scheduler.sched_iface, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True proc.cancel() daemon.cancel() exit_command.reply_hook = exit_command_callback task_scheduler.add(daemon) task_scheduler.add(proc) start_time = time.time() task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT) task_scheduler.clear() self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for QueueScheduler.run() sleep_time_s = 10 # 10.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True proc.cancel() daemon.cancel() exit_command.reply_hook = exit_command_callback task_scheduler.add(daemon) task_scheduler.add(proc) start_time = time.time() task_scheduler.run(timeout=short_timeout_ms) task_scheduler.clear() self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self.wait() return if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd)
def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() if retval == os.EX_OK: self._current_task = None self.returncode = os.EX_OK self._async_wait() else: # For pretend mode, the convention it to execute # pkg_nofetch and return a successful exitcode. self._start_task( SpawnNofetchWithoutBuilddir( background=self.background, portdb=self.pkg.root_config.trees[ self._tree].dbapi, ebuild_path=self._ebuild_path, scheduler=self.scheduler, settings=self.settings), self._default_final_exit) return quiet_setting = settings.get("PORTAGE_QUIET", False) fetch_log = None logwrite_access = False if quiet_setting: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, "emerge-fetch.log") logwrite_access = os.access(first_existing(fetch_log), os.W_OK) fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=quiet_setting if logwrite_access else False, logfile=fetch_log if logwrite_access else None, pkg=self.pkg, scheduler=self.scheduler, ) if fetch_log and logwrite_access: fetcher.addExitListener(self._fetchonly_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher, force_queue=True) else: self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_pre_clean)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._builddir_lock.lock() portage.prepare_build_dirs(settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, )) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit)
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): pkg = self.pkg settings = self.settings if not self.opts.fetchonly: rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ EPREFIX + "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return else: fetcher = EbuildFetcher( config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: portdb = self.pkg.root_config.trees[self._tree].dbapi spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) elif 'digest' in self.settings.features: if not digestgen(mysettings=self.settings, myportdb=self.pkg.root_config.trees[self._tree].dbapi): self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) try: already_fetched = fetcher.already_fetched(self.settings) except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self.returncode = 1 self._current_task = None self._unlock_builddir() self.wait() return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._unlock_builddir() self.wait() return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._unlock_builddir() self.returncode = 1 self.wait() def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add(EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: binpkg_tasks.add(EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): self._unlock_builddir()
class PackageUninstall(CompositeTask): """ Uninstall a package asynchronously in a subprocess. When both parallel-install and ebuild-locks FEATURES are enabled, it is essential for the ebuild-locks code to execute in a subprocess, since the portage.locks module does not behave as desired if we try to lock the same file multiple times concurrently from the same process for ebuild-locks phases such as pkg_setup, pkg_prerm, and pkg_postrm. """ __slots__ = ( "world_atom", "ldpath_mtimes", "opts", "pkg", "settings", "_builddir_lock", ) def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge, ) def _start_unmerge(self, lock_task): self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() portage.prepare_build_dirs(settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, _ = _unmerge_display( self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level, ) if retval != os.EX_OK: self._async_unlock_builddir(returncode=retval) return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, )) cat, pf = portage.catsplit(self.pkg.cpv) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True, ) self._start_task(unmerge_task, self._unmerge_exit) def _unmerge_exit(self, unmerge_task): if self._final_exit(unmerge_task) != os.EX_OK: self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv, )) else: self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv, )) self.world_atom(self.pkg) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode), ) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def _emergelog(self, msg): emergelog("notitles" not in self.settings.features, msg) def _writemsg_level(self, msg, level=0, noiselevel=0): log_path = self.settings.get("PORTAGE_LOG_FILE") background = self.background if log_path is None: if not (background and level < logging.WARNING): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: self.scheduler.output(msg, log_path=log_path, level=level, noiselevel=noiselevel)
class Binpkg(CompositeTask): __slots__ = ( "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom", ) + ( "_bintree", "_build_dir", "_build_prefix", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify", ) def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output( msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE"), ) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join( os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf, ) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment( self._ebuild_path, "setup", settings=self.settings, db=self._bintree.dbapi ) if dir_path != self.settings["PORTAGE_BUILDDIR"]: raise AssertionError( "'%s' != '%s'" % (dir_path, self.settings["PORTAGE_BUILDDIR"]) ) self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set( portage.versions.cpv_getversion(x) for x in vardb.match(self.pkg.slot_atom) + vardb.match("=" + self.pkg.cpv) ) ) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, "emerge-fetch.log" ) msg = ( "Fetching in the background:", prefetcher.pkg_path, "To view fetch progress, run in another terminal:", "tail -f %s" % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return if not (self.opts.pretend or self.opts.fetchonly): self._start_task( AsyncTaskFuture(future=self._build_dir.async_lock()), self._start_fetcher, ) else: self._start_fetcher() def _start_fetcher(self, lock_task=None): if lock_task is not None: self._assert_current(lock_task) if lock_task.cancelled: self._default_final_exit(lock_task) return lock_task.future.result() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() pkg = self.pkg pkg_count = self.pkg_count fetcher = None if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): fetcher = BinpkgFetcher( background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler, ) msg = " --- (%s of %s) Fetching Binary (%s::%s)" % ( pkg_count.curval, pkg_count.maxval, pkg.cpv, fetcher.pkg_path, ) short_msg = "emerge: (%s of %s) %s Fetch" % ( pkg_count.curval, pkg_count.maxval, pkg.cpv, ) self.logger.log(msg, short_msg=short_msg) # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetcher_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher is not None: self._fetched_pkg = fetcher.pkg_path if self._default_exit(fetcher) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: if self._fetched_pkg: path = self._fetched_pkg else: path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv) logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier( background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, _pkg_path=path, ) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and self._default_exit(verifier) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count if self._fetched_pkg: pkg_path = self._bintree.getname( self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), allocate_new=False, ) else: pkg_path = self.pkg.root_config.trees["bintree"].getname(self.pkg.cpv) # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. if pkg_path is not None: self.settings["PORTAGE_BINPKG_FILE"] = pkg_path self._pkg_path = pkg_path logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % ( pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path, ) short_msg = "emerge: (%s of %s) %s Merge Binary" % ( pkg_count.curval, pkg_count.maxval, pkg.cpv, ) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase( background=self.background, phase=phase, scheduler=self.scheduler, settings=settings, ) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return self._start_task( AsyncTaskFuture(future=self._unpack_metadata(loop=self.scheduler)), self._unpack_metadata_exit, ) async def _unpack_metadata(self, loop=None): dir_path = self.settings["PORTAGE_BUILDDIR"] infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs( mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode, ) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") await self._bintree.dbapi.unpack_metadata( self.settings, infloc, loop=self.scheduler ) check_missing_metadata = ("CATEGORY", "PF") for k, v in zip( check_missing_metadata, self._bintree.dbapi.aux_get(self.pkg.cpv, check_missing_metadata), ): if v: continue elif k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = io.open( _unicode_encode( os.path.join(infloc, k), encoding=_encodings["fs"], errors="strict" ), mode="w", encoding=_encodings["content"], errors="backslashreplace", ) try: f.write(_unicode_decode(v + "\n")) finally: f.close() # Store the md5sum in the vdb. if pkg_path is not None: (md5sum,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["MD5"]) if not md5sum: md5sum = portage.checksum.perform_md5(pkg_path) with io.open( _unicode_encode( os.path.join(infloc, "BINPKGMD5"), encoding=_encodings["fs"], errors="strict", ), mode="w", encoding=_encodings["content"], errors="strict", ) as f: f.write(_unicode_decode("{}\n".format(md5sum))) env_extractor = BinpkgEnvExtractor( background=self.background, scheduler=self.scheduler, settings=self.settings ) env_extractor.start() await env_extractor.async_wait() if env_extractor.returncode != os.EX_OK: raise portage.exception.PortageException( "failed to extract environment for {}".format(self.pkg.cpv) ) def _unpack_metadata_exit(self, unpack_metadata): if self._default_exit(unpack_metadata) != os.EX_OK: unpack_metadata.future.result() self._async_unlock_builddir(returncode=self.returncode) return setup_phase = EbuildPhase( background=self.background, phase="setup", scheduler=self.scheduler, settings=self.settings, ) setup_phase.addExitListener(self._setup_exit) self._task_queued(setup_phase) self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._async_unlock_builddir(returncode=self.returncode) return self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task( AsyncTaskFuture( future=self._bintree.dbapi.unpack_contents( self.settings, self._image_dir, loop=self.scheduler ) ), self._unpack_contents_exit, ) def _unpack_contents_exit(self, unpack_contents): if self._default_exit(unpack_contents) != os.EX_OK: unpack_contents.future.result() self._writemsg_level( "!!! Error Extracting '%s'\n" % self._pkg_path, noiselevel=-1, level=logging.ERROR, ) self._async_unlock_builddir(returncode=self.returncode) return # Before anything else, let's do an integrity check. (provides,) = self._bintree.dbapi.aux_get(self.pkg.cpv, ["PROVIDES"]) if check_dyn_libs_inconsistent(self.settings["D"], provides): self._writemsg_level( colorize( "BAD", "!!! Error! Installing dynamic libraries (.so) with blank PROVIDES!", ), noiselevel=-1, level=logging.ERROR, ) try: with io.open( _unicode_encode( os.path.join(self._infloc, "EPREFIX"), encoding=_encodings["fs"], errors="strict", ), mode="r", encoding=_encodings["repo.content"], errors="replace", ) as f: self._build_prefix = f.read().rstrip("\n") except IOError: self._build_prefix = "" if self._build_prefix == self.settings["EPREFIX"]: ensure_dirs(self.settings["ED"]) self._current_task = None self.returncode = os.EX_OK self.wait() return env = self.settings.environ() env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[ portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"], ], background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get("PORTAGE_LOG_FILE"), ) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) self._start_task(chpathtool, self._chpathtool_exit) def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._writemsg_level( "!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],), noiselevel=-1, level=logging.ERROR, ) self._async_unlock_builddir(returncode=self.returncode) return # We want to install in "our" prefix, not the binary one with io.open( _unicode_encode( os.path.join(self._infloc, "EPREFIX"), encoding=_encodings["fs"], errors="strict", ), mode="w", encoding=_encodings["repo.content"], errors="strict", ) as f: f.write(self.settings["EPREFIX"] + "\n") # Move the files to the correct location for merge. image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"], "image_tmp") build_d = os.path.join( self.settings["D"], self._build_prefix.lstrip(os.sep) ).rstrip(os.sep) if not os.path.isdir(build_d): # Assume this is a virtual package or something. shutil.rmtree(self._image_dir) ensure_dirs(self.settings["ED"]) else: os.rename(build_d, image_tmp_dir) if build_d != self._image_dir: shutil.rmtree(self._image_dir) ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) os.rename(image_tmp_dir, self.settings["ED"]) self.wait() def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self.opts.pretend or self.opts.fetchonly: if returncode is not None: self.returncode = returncode self._async_wait() return if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None portage.elog.elog_process(self.pkg.cpv, self.settings) self._start_task( AsyncTaskFuture(future=self._build_dir.async_unlock()), functools.partial(self._unlock_builddir_exit, returncode=returncode), ) def _unlock_builddir_exit(self, unlock_task, returncode=None): self._assert_current(unlock_task) if unlock_task.cancelled and returncode is not None: self._default_final_exit(unlock_task) return # Normally, async_unlock should not raise an exception here. unlock_task.future.cancelled() or unlock_task.future.result() if returncode is not None: self.returncode = returncode self._async_wait() def create_install_task(self): task = EbuildMerge( exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom, ) return task def _install_exit(self, task): """ @returns: Future, result is the returncode from an EbuildBuildDir.async_unlock() task """ self.settings.pop("PORTAGE_BINPKG_FILE", None) if ( task.returncode == os.EX_OK and "binpkg-logs" not in self.settings.features and self.settings.get("PORTAGE_LOG_FILE") ): try: os.unlink(self.settings["PORTAGE_LOG_FILE"]) except OSError: pass self._async_unlock_builddir() if self._current_task is None: result = self.scheduler.create_future() self.scheduler.call_soon(result.set_result, os.EX_OK) else: result = self._current_task.async_wait() return result
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') _phases_without_builddir = ( 'clean', 'cleanrm', 'depend', 'help', ) _phases_interactive_whitelist = ('config', ) _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ 'mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ 'mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage ]) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses': query_command, 'best_version': query_command, 'eclass_path': query_command, 'exit': self._exit_command, 'has_version': query_command, 'license_path': query_command, 'master_repositories': query_command, 'repository_path': query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = \ self.scheduler.timeout_add(self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = \ self.scheduler.timeout_add(self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None return False # only run once def _cancel_timeout_cb(self): self._exit_timeout_id = None self.wait() return False # only run once def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = _( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog('eerror', lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = \ self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false') for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _log_poll_exception(self, event): self._elog("eerror", ["%s received strange poll event: %s\n" % \ (self.__class__.__name__, event,)]) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self.cgroup is not None: try: shutil.rmtree(self.cgroup) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if self._exit_timeout_id is not None: self.scheduler.source_remove(self._exit_timeout_id) self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() if self._build_dir is not None: self._build_dir.unlock() self._build_dir = None elif not self.cancelled: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit()
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): pkg = self.pkg settings = self.settings if not self.opts.fetchonly: rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features and \ 'digest' not in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: if opts.pretend: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return else: fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=False, logfile=None, pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetchonly_exit) return self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _fetchonly_exit(self, fetcher): self._final_exit(fetcher) if self.returncode != os.EX_OK: portdb = self.pkg.root_config.trees[self._tree].dbapi spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) elif 'digest' in self.settings.features: if not digestgen( mysettings=self.settings, myportdb=self.pkg.root_config.trees[self._tree].dbapi): self.returncode = 1 self.wait() def _pre_clean_exit(self, pre_clean_phase): if self._default_exit(pre_clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, ebuild_path=self._ebuild_path, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) try: already_fetched = fetcher.already_fetched(self.settings) except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) fetcher._eerror(msg_lines) portage.elog.elog_process(self.pkg.cpv, self.settings) self.returncode = 1 self._current_task = None self._unlock_builddir() self.wait() return if already_fetched: # This case is optimized to skip the fetch queue. fetcher = None self._fetch_exit(fetcher) return # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetch_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) def _fetch_exit(self, fetcher): if fetcher is not None and \ self._default_exit(fetcher) != os.EX_OK: self._fetch_failed() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ "buildpkg" not in features and \ opts.buildpkg != 'n' if ("buildpkg" in features or self._issyspkg) \ and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _fetch_failed(self): # We only call the pkg_nofetch phase if either RESTRICT=fetch # is set or the package has explicitly overridden the default # pkg_nofetch implementation. This allows specialized messages # to be displayed for problematic packages even though they do # not set RESTRICT=fetch (bug #336499). if 'fetch' not in self.pkg.restrict and \ 'nofetch' not in self.pkg.defined_phases: self._unlock_builddir() self.wait() return self.returncode = None nofetch_phase = EbuildPhase(background=self.background, phase='nofetch', scheduler=self.scheduler, settings=self.settings) self._start_task(nofetch_phase, self._nofetch_exit) def _nofetch_exit(self, nofetch_phase): self._final_exit(nofetch_phase) self._unlock_builddir() self.returncode = 1 self.wait() def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) binpkg_tasks = TaskSequence() requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: if pkg_fmt in requested_binpkg_formats: if pkg_fmt == "rpm": binpkg_tasks.add( EbuildPhase(background=self.background, phase="rpm", scheduler=self.scheduler, settings=self.settings)) else: binpkg_tasks.add( EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings)) if binpkg_tasks: self._start_task(binpkg_tasks, self._buildpkg_exit) return self._final_exit(build) self.wait() def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess(background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def create_install_task(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) return task def _install_exit(self, task): self._unlock_builddir()
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') env['PYTHONDONTWRITEBYTECODE'] = os.environ.get( 'PYTHONDONTWRITEBYTECODE', '') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir(scheduler=event_loop, settings=env) event_loop.run_until_complete(build_dir.async_lock()) ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 # seconds short_timeout_s = 0.010 # seconds for i in range(3): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_s) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: event_loop.run_until_complete(build_dir.async_unlock()) shutil.rmtree(tmpdir)
def _fetch_exit(self, fetcher): opts = self.opts pkg = self.pkg fetch_failed = False if opts.fetchonly: fetch_failed = self._final_exit(fetcher) != os.EX_OK else: fetch_failed = self._default_exit(fetcher) != os.EX_OK if fetch_failed and fetcher.logfile is not None and \ os.path.exists(fetcher.logfile): self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile if fetch_failed or opts.fetchonly: self.wait() return logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning is triggered before the setup # phase, in portage.doebuild(). msg = " === (%s of %s) Cleaning (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit)
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): pkg = self.pkg settings = self.settings rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self.wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name ebuild_path = portdb.findname(pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self.wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _pre_clean_exit(self, pre_clean_phase): if self._final_exit(pre_clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetch_exit) def _fetch_exit(self, fetcher): portage.elog.elog_process(self.pkg.cpv, self.settings) if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(packager, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def install(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) try: rval = merge.execute() finally: self._unlock_builddir() return rval
class EbuildFetcher(SpawnProcess): __slots__ = ("config_pool", "fetchonly", "fetchall", "pkg", "prefetch") + \ ("_build_dir",) def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) settings = self.config_pool.allocate() settings.setcpv(self.pkg) if self.prefetch and \ self._prefetch_size_ok(portdb, settings, ebuild_path): self.config_pool.deallocate(settings) self.returncode = os.EX_OK self.wait() return # In prefetch mode, logging goes to emerge-fetch.log and the builddir # should not be touched since otherwise it could interfere with # another instance of the same cpv concurrently being built for a # different $ROOT (currently, builds only cooperate with prefetchers # that are spawned for the same $ROOT). if not self.prefetch: self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings) self._build_dir.lock() self._build_dir.clean_log() cleanup=1 # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, cleanup) if self.logfile is None: self.logfile = settings.get("PORTAGE_LOG_FILE") phase = "fetch" if self.fetchall: phase = "fetchall" # If any incremental variables have been overridden # via the environment, those values need to be passed # along here so that they are correctly considered by # the config instance in the subproccess. fetch_env = os.environ.copy() fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT'] nocolor = settings.get("NOCOLOR") if nocolor is not None: fetch_env["NOCOLOR"] = nocolor fetch_env["PORTAGE_NICENESS"] = "0" if self.prefetch: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" ebuild_binary = os.path.join( settings["PORTAGE_BIN_PATH"], "ebuild") fetch_args = [ebuild_binary, ebuild_path, phase] debug = settings.get("PORTAGE_DEBUG") == "1" if debug: fetch_args.append("--debug") if not self.background and nocolor not in ('yes', 'true'): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. fetch_args.append('--color=y') self.args = fetch_args self.env = fetch_env if self._build_dir is None: # Free settings now since we only have a local reference. self.config_pool.deallocate(settings) SpawnProcess._start(self) def _prefetch_size_ok(self, portdb, settings, ebuild_path): pkgdir = os.path.dirname(ebuild_path) mytree = os.path.dirname(os.path.dirname(pkgdir)) distdir = settings["DISTDIR"] use = None if not self.fetchall: use = self.pkg.use.enabled try: uri_map = portdb.getFetchMap(self.pkg.cpv, useflags=use, mytree=mytree) except portage.exception.InvalidDependString as e: return False sizes = {} for filename in uri_map: # Use stat rather than lstat since portage.fetch() creates # symlinks when PORTAGE_RO_DISTDIRS is used. try: st = os.stat(os.path.join(distdir, filename)) except OSError: return False if st.st_size == 0: return False sizes[filename] = st.st_size digests = portage.Manifest(pkgdir, distdir).getTypeDigests("DIST") for filename, actual_size in sizes.items(): size = digests.get(filename, {}).get('size') if size is None: continue if size != actual_size: return False # All files are present and sizes are ok. In this case the normal # fetch code will be skipped, so we need to generate equivalent # output here. if self.logfile is not None: f = codecs.open(_unicode_encode(self.logfile, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') for filename in uri_map: f.write((' * %s size ;-) ...' % \ filename).ljust(73) + '[ ok ]\n') f.close() return True def _pipe(self, fd_pipes): """When appropriate, use a pty so that fetcher progress bars, like wget has, will work properly.""" if self.background or not sys.stdout.isatty(): # When the output only goes to a log file, # there's no point in creating a pty. return os.pipe() stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) # Collect elog messages that might have been # created by the pkg_nofetch phase. if self._build_dir is not None: # Skip elog messages for prefetch, in order to avoid duplicates. if not self.prefetch and self.returncode != os.EX_OK: elog_out = None if self.logfile is not None: if self.background: elog_out = codecs.open(_unicode_encode(self.logfile, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') msg = "Fetch failed for '%s'" % (self.pkg.cpv,) if self.logfile is not None: msg += ", Log file:" eerror(msg, phase="unpack", key=self.pkg.cpv, out=elog_out) if self.logfile is not None: eerror(" '%s'" % (self.logfile,), phase="unpack", key=self.pkg.cpv, out=elog_out) if elog_out is not None: elog_out.close() if not self.prefetch: portage.elog.elog_process(self.pkg.cpv, self._build_dir.settings) features = self._build_dir.settings.features if self.returncode == os.EX_OK: self._build_dir.clean_log() self._build_dir.unlock() self.config_pool.deallocate(self._build_dir.settings) self._build_dir = None
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ 'mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ 'mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage ]) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd)
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
class PackageUninstall(CompositeTask): """ Uninstall a package asynchronously in a subprocess. When both parallel-install and ebuild-locks FEATURES are enabled, it is essential for the ebuild-locks code to execute in a subprocess, since the portage.locks module does not behave as desired if we try to lock the same file multiple times concurrently from the same process for ebuild-locks phases such as pkg_setup, pkg_prerm, and pkg_postrm. """ __slots__ = ("world_atom", "ldpath_mtimes", "opts", "pkg", "settings", "_builddir_lock") def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._builddir_lock.lock() portage.prepare_build_dirs( settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,)) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit) def _unmerge_exit(self, unmerge_task): if self._final_exit(unmerge_task) != os.EX_OK: self._emergelog(" !!! unmerge FAILURE: %s" % (self.pkg.cpv,)) else: self._emergelog(" >>> unmerge success: %s" % (self.pkg.cpv,)) self.world_atom(self.pkg) self._builddir_lock.unlock() self.wait() def _emergelog(self, msg): emergelog("notitles" not in self.settings.features, msg) def _writemsg_level(self, msg, level=0, noiselevel=0): log_path = self.settings.get("PORTAGE_LOG_FILE") background = self.background if log_path is None: if not (background and level < logging.WARNING): portage.util.writemsg_level(msg, level=level, noiselevel=noiselevel) else: self.scheduler.output(msg, log_path=log_path, level=level, noiselevel=noiselevel)
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call(['mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call(['mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage]) cgroup_path = tempfile.mkdtemp(dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd)
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and not os.path.isdir( self.settings["PORTAGE_BUILDDIR"]): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % ( self.phase, self.settings["PORTAGE_BUILDDIR"]) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == "Linux" and "cgroup" in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = "/sys/fs/cgroup" cgroup_portage = os.path.join(cgroup_root, "portage") try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ "mount", "-t", "tmpfs", "-o", "rw,nosuid,nodev,noexec,mode=0755", "tmpfs", cgroup_root, ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ "mount", "-t", "cgroup", "-o", "rw,nosuid,nodev,noexec,none,name=portage", "tmpfs", cgroup_portage, ]) with open(os.path.join(cgroup_portage, "release_agent"), "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) with open( os.path.join(cgroup_portage, "notify_on_release"), "w") as f: f.write("1") else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join(cgroup_portage, "release_agent") try: with open(release_agent) as f: release_agent_path = f.readline().rstrip("\n") except EnvironmentError: release_agent_path = None if release_agent_path is None or not os.path.exists( release_agent_path): with open(release_agent, "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix="%s:%s." % (self.settings["CATEGORY"], self.settings["PF"]), ) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings["NOCOLOR"] = "true" start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial( self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon, )) return else: self.settings.pop("PORTAGE_IPC_DAEMON", None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop("PORTAGE_IPC_DAEMON", None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".exit_status") self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) settings = self.config_pool.allocate() settings.setcpv(self.pkg) if self.prefetch and \ self._prefetch_size_ok(portdb, settings, ebuild_path): self.config_pool.deallocate(settings) self.returncode = os.EX_OK self.wait() return # In prefetch mode, logging goes to emerge-fetch.log and the builddir # should not be touched since otherwise it could interfere with # another instance of the same cpv concurrently being built for a # different $ROOT (currently, builds only cooperate with prefetchers # that are spawned for the same $ROOT). if not self.prefetch: self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings) self._build_dir.lock() self._build_dir.clean_log() portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, 0) if self.logfile is None: self.logfile = settings.get("PORTAGE_LOG_FILE") phase = "fetch" if self.fetchall: phase = "fetchall" # If any incremental variables have been overridden # via the environment, those values need to be passed # along here so that they are correctly considered by # the config instance in the subproccess. fetch_env = os.environ.copy() fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT'] nocolor = settings.get("NOCOLOR") if nocolor is not None: fetch_env["NOCOLOR"] = nocolor fetch_env["PORTAGE_NICENESS"] = "0" if self.prefetch: fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1" ebuild_binary = os.path.join( settings["PORTAGE_BIN_PATH"], "ebuild") fetch_args = [ebuild_binary, ebuild_path, phase] debug = settings.get("PORTAGE_DEBUG") == "1" if debug: fetch_args.append("--debug") if not self.background and nocolor not in ('yes', 'true'): # Force consistent color output, in case we are capturing fetch # output through a normal pipe due to unavailability of ptys. fetch_args.append('--color=y') self.args = fetch_args self.env = fetch_env if self._build_dir is None: # Free settings now since we only have a local reference. self.config_pool.deallocate(settings) SpawnProcess._start(self)
class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_build_prefix", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output(msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return pkg = self.pkg pkg_count = self.pkg_count if not (self.opts.pretend or self.opts.fetchonly): self._build_dir.lock() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, fetcher.pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetcher_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = fetcher.pkg_path if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: if self._fetched_pkg: path = self._fetched_pkg else: path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, _pkg_path=path) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._unlock_builddir() self.wait() return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count if self._fetched_pkg: pkg_path = self._bintree.getname( self._bintree.inject(pkg.cpv, filename=self._fetched_pkg), allocate_new=False) else: pkg_path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. self.settings["PORTAGE_BINPKG_FILE"] = pkg_path self._pkg_path = pkg_path logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=settings) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return dir_path = self.settings['PORTAGE_BUILDDIR'] infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile(_unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = io.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='backslashreplace') try: f.write(_unicode_decode(v + "\n")) finally: f.close() # Store the md5sum in the vdb. f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write(_unicode_decode( str(portage.checksum.perform_md5(pkg_path)) + "\n")) finally: f.close() env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) self._start_task(env_extractor, self._env_extractor_exit) def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self._unlock_builddir() self.wait() return setup_phase = EbuildPhase(background=self.background, phase="setup", scheduler=self.scheduler, settings=self.settings) setup_phase.addExitListener(self._setup_exit) self._task_queued(setup_phase) self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._unlock_builddir() self.wait() return extractor = BinpkgExtractorAsync(background=self.background, env=self.settings.environ(), features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._default_exit(extractor) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self.wait() return try: with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._build_prefix = f.read().rstrip('\n') except IOError: self._build_prefix = "" if self._build_prefix == self.settings["EPREFIX"]: ensure_dirs(self.settings["ED"]) self._current_task = None self.returncode = os.EX_OK self.wait() return env = self.settings.environ() env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"]], background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) self._start_task(chpathtool, self._chpathtool_exit) def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"],), noiselevel=-1, level=logging.ERROR) self.wait() return # We want to install in "our" prefix, not the binary one with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(self.settings["EPREFIX"] + "\n") # Move the files to the correct location for merge. image_tmp_dir = os.path.join( self.settings["PORTAGE_BUILDDIR"], "image_tmp") build_d = os.path.join(self.settings["D"], self._build_prefix.lstrip(os.sep)) if not os.path.isdir(build_d): # Assume this is a virtual package or something. shutil.rmtree(self._image_dir) ensure_dirs(self.settings["ED"]) else: os.rename(build_d, image_tmp_dir) shutil.rmtree(self._image_dir) ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) os.rename(image_tmp_dir, self.settings["ED"]) self.wait() def _unlock_builddir(self): if self.opts.pretend or self.opts.fetchonly: return portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def create_install_task(self): task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom) return task def _install_exit(self, task): self.settings.pop("PORTAGE_BINPKG_FILE", None) self._unlock_builddir() if task.returncode == os.EX_OK and \ 'binpkg-logs' not in self.settings.features and \ self.settings.get("PORTAGE_LOG_FILE"): try: os.unlink(self.settings["PORTAGE_LOG_FILE"]) except OSError: pass
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): pkg = self.pkg settings = self.settings rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self.wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name ebuild_path = portdb.findname(pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self.wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _check_manifest(self): success = True settings = self.settings if 'strict' in settings.features: settings['O'] = os.path.dirname(self._ebuild_path) quiet_setting = settings.get('PORTAGE_QUIET') settings['PORTAGE_QUIET'] = '1' try: success = digestcheck([], settings, strict=True) finally: if quiet_setting: settings['PORTAGE_QUIET'] = quiet_setting else: del settings['PORTAGE_QUIET'] return success def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning needs to happen before fetch, since the build dir # is used for log handling. msg = " === (%s of %s) Cleaning (%s::%s)" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv, self._ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) self.logger.log(msg, short_msg=short_msg) pre_clean_phase = EbuildPhase(background=self.background, phase='clean', scheduler=self.scheduler, settings=self.settings) self._start_task(pre_clean_phase, self._pre_clean_exit) def _pre_clean_exit(self, pre_clean_phase): if self._final_exit(pre_clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return # for log handling portage.prepare_build_dirs(self.pkg.root, self.settings, 1) fetcher = EbuildFetcher(config_pool=self.config_pool, fetchall=self.opts.fetch_all_uri, fetchonly=self.opts.fetchonly, background=self.background, logfile=self.settings.get('PORTAGE_LOG_FILE'), pkg=self.pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetch_exit) def _fetch_exit(self, fetcher): portage.elog.elog_process(self.pkg.cpv, self.settings) if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return # discard successful fetch log self._build_dir.clean_log() pkg = self.pkg logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(packager, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess(background=self.background, commands=[phase], phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=self.settings) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def install(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) try: rval = merge.execute() finally: self._unlock_builddir() return rval
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) _phases_interactive_whitelist = ('config',) _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call(['mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call(['mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage]) cgroup_path = tempfile.mkdtemp(dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses' : query_command, 'best_version' : query_command, 'eclass_path' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, 'license_path' : query_command, 'master_repositories' : query_command, 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = \ self.scheduler.timeout_add(self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = \ self.scheduler.timeout_add(self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None return False # only run once def _cancel_timeout_cb(self): self._exit_timeout_id = None self.wait() return False # only run once def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = _("The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog('eerror', lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = \ self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false') for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _log_poll_exception(self, event): self._elog("eerror", ["%s received strange poll event: %s\n" % \ (self.__class__.__name__, event,)]) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self.cgroup is not None: try: shutil.rmtree(self.cgroup) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if self._exit_timeout_id is not None: self.scheduler.source_remove(self._exit_timeout_id) self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() if self._build_dir is not None: self._build_dir.unlock() self._build_dir = None elif not self.cancelled: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit()
class EbuildBuild(CompositeTask): __slots__ = ("args_set", "config_pool", "find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") def _start(self): logger = self.logger opts = self.opts pkg = self.pkg settings = self.settings world_atom = self.world_atom root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name ebuild_path = portdb.findname(pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): opts = self.opts pkg = self.pkg settings = self.settings if opts.fetchonly: fetcher = EbuildFetchonly( fetch_all=opts.fetch_all_uri, pkg=pkg, pretend=opts.pretend, settings=settings) retval = fetcher.execute() self.returncode = retval self.wait() return fetcher = EbuildFetcher(config_pool=self.config_pool, fetchall=opts.fetch_all_uri, fetchonly=opts.fetchonly, background=self.background, pkg=pkg, scheduler=self.scheduler) self._start_task(fetcher, self._fetch_exit) def _fetch_exit(self, fetcher): opts = self.opts pkg = self.pkg fetch_failed = False if opts.fetchonly: fetch_failed = self._final_exit(fetcher) != os.EX_OK else: fetch_failed = self._default_exit(fetcher) != os.EX_OK if fetch_failed and fetcher.logfile is not None and \ os.path.exists(fetcher.logfile): self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile if fetch_failed or opts.fetchonly: self.wait() return logger = self.logger opts = self.opts pkg_count = self.pkg_count scheduler = self.scheduler settings = self.settings features = settings.features ebuild_path = self._ebuild_path system_set = pkg.root_config.sets["system"] self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings) self._build_dir.lock() # Cleaning is triggered before the setup # phase, in portage.doebuild(). msg = " === (%s of %s) Cleaning (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Clean" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) #buildsyspkg: Check if we need to _force_ binary package creation self._issyspkg = "buildsyspkg" in features and \ system_set.findAtomForPackage(pkg) and \ not opts.buildpkg if opts.buildpkg or self._issyspkg: self._buildpkg = True msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) else: msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Compile" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) build = EbuildExecuter(background=self.background, pkg=pkg, scheduler=scheduler, settings=settings) self._start_task(build, self._build_exit) def _unlock_builddir(self): portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def _build_exit(self, build): if self._default_exit(build) != os.EX_OK: self._unlock_builddir() self.wait() return opts = self.opts buildpkg = self._buildpkg if not buildpkg: self._final_exit(build) self.wait() return if self._issyspkg: msg = ">>> This is a system package, " + \ "let's pack a rescue tarball.\n" log_path = self.settings.get("PORTAGE_LOG_FILE") if log_path is not None: log_file = codecs.open(_unicode_encode(log_path, encoding=_encodings['fs'], errors='strict'), mode='a', encoding=_encodings['content'], errors='replace') try: log_file.write(msg) finally: log_file.close() if not self.background: portage.writemsg_stdout(msg, noiselevel=-1) packager = EbuildBinpkg(background=self.background, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(packager, self._buildpkg_exit) def _buildpkg_exit(self, packager): """ Released build dir lock when there is a failure or when in buildpkgonly mode. Otherwise, the lock will be released when merge() is called. """ if self._default_exit(packager) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.buildpkgonly: phase = 'success_hooks' success_hooks = MiscFunctionsProcess( background=self.background, commands=[phase], phase=phase, pkg=self.pkg, scheduler=self.scheduler, settings=self.settings) self._start_task(success_hooks, self._buildpkgonly_success_hook_exit) return # Continue holding the builddir lock until # after the package has been installed. self._current_task = None self.returncode = packager.returncode self.wait() def _buildpkgonly_success_hook_exit(self, success_hooks): self._default_exit(success_hooks) self.returncode = None # Need to call "clean" phase for buildpkgonly mode portage.elog.elog_process(self.pkg.cpv, self.settings) phase = 'clean' clean_phase = EbuildPhase(background=self.background, pkg=self.pkg, phase=phase, scheduler=self.scheduler, settings=self.settings, tree=self._tree) self._start_task(clean_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._final_exit(clean_phase) != os.EX_OK or \ self.opts.buildpkgonly: self._unlock_builddir() self.wait() def install(self): """ Install the package and then clean up and release locks. Only call this after the build has completed successfully and neither fetchonly nor buildpkgonly mode are enabled. """ find_blockers = self.find_blockers ldpath_mtimes = self.ldpath_mtimes logger = self.logger pkg = self.pkg pkg_count = self.pkg_count settings = self.settings world_atom = self.world_atom ebuild_path = self._ebuild_path tree = self._tree merge = EbuildMerge(find_blockers=self.find_blockers, ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, pkg_count=pkg_count, pkg_path=ebuild_path, scheduler=self.scheduler, settings=settings, tree=tree, world_atom=world_atom) msg = " === (%s of %s) Merging (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) short_msg = "emerge: (%s of %s) %s Merge" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) try: rval = merge.execute() finally: self._unlock_builddir() return rval
class AbstractEbuildProcess(SpawnProcess): __slots__ = ( "phase", "settings", ) + ( "_build_dir", "_build_dir_unlock", "_ipc_daemon", "_exit_command", "_exit_timeout_id", "_start_future", ) _phases_without_builddir = ( "clean", "cleanrm", "depend", "help", ) _phases_interactive_whitelist = ("config", ) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10 # seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = "other" self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and not os.path.isdir( self.settings["PORTAGE_BUILDDIR"]): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % ( self.phase, self.settings["PORTAGE_BUILDDIR"]) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == "Linux" and "cgroup" in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = "/sys/fs/cgroup" cgroup_portage = os.path.join(cgroup_root, "portage") try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ "mount", "-t", "tmpfs", "-o", "rw,nosuid,nodev,noexec,mode=0755", "tmpfs", cgroup_root, ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ "mount", "-t", "cgroup", "-o", "rw,nosuid,nodev,noexec,none,name=portage", "tmpfs", cgroup_portage, ]) with open(os.path.join(cgroup_portage, "release_agent"), "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) with open( os.path.join(cgroup_portage, "notify_on_release"), "w") as f: f.write("1") else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join(cgroup_portage, "release_agent") try: with open(release_agent) as f: release_agent_path = f.readline().rstrip("\n") except EnvironmentError: release_agent_path = None if release_agent_path is None or not os.path.exists( release_agent_path): with open(release_agent, "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix="%s:%s." % (self.settings["CATEGORY"], self.settings["PF"]), ) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings["NOCOLOR"] = "true" start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial( self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon, )) return else: self.settings.pop("PORTAGE_IPC_DAEMON", None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop("PORTAGE_IPC_DAEMON", None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".exit_status") self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon) def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False): if lock_future is not None: if lock_future is not self._start_future: raise AssertionError("lock_future is not self._start_future") self._start_future = None if lock_future.cancelled(): self._build_dir = None self.cancelled = True self._was_cancelled() self._async_wait() return lock_future.result() if start_ipc_daemon: self.settings["PORTAGE_IPC_DAEMON"] = "1" self._start_ipc_daemon() if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if (0 not in self.fd_pipes and self.phase not in self._phases_interactive_whitelist and "interactive" not in self.settings.get("PROPERTIES", "").split()): null_fd = os.open("/dev/null", os.O_RDONLY) self.fd_pipes[0] = null_fd self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in") output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_out") for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions( p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st, ) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { "available_eclasses": query_command, "best_version": query_command, "eclass_path": query_command, "exit": self._exit_command, "has_version": query_command, "license_path": query_command, "master_repositories": query_command, "repository_path": query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon( commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler, ) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = self.scheduler.call_later( self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = self.scheduler.call_later( self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None def _cancel_timeout_cb(self): self._exit_timeout_id = None self._async_waitpid() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = _create_pty_or_pipe( copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ("sesandbox" in self.settings.features and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % ( self.phase, signum, ) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = (_( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase) self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog("eerror", lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = self.settings.get( "NOCOLOR", "false").lower() in ("no", "false") for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _async_waitpid_cb(self, *args, **kwargs): """ Override _async_waitpid_cb to perform cleanup that is not necessarily idempotent. """ SpawnProcess._async_waitpid_cb(self, *args, **kwargs) if self._exit_timeout_id is not None: self._exit_timeout_id.cancel() self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() elif not self.cancelled: exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE") if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir when necessary. """ if self._build_dir is None: SpawnProcess._async_wait(self) elif self._build_dir_unlock is None: if self.returncode is None: raise asyncio.InvalidStateError("Result is not ready for %s" % (self, )) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self._build_dir_unlock is not None: raise AssertionError("unlock already in progress") if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._build_dir_unlock = self._build_dir.async_unlock() # Unlock only once. self._build_dir = None self._build_dir_unlock.add_done_callback( functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_future, returncode=None): # Normally, async_unlock should not raise an exception here. unlock_future.cancelled() or unlock_future.result() if returncode is not None: if unlock_future.cancelled(): self.cancelled = True self._was_cancelled() else: self.returncode = returncode SpawnProcess._async_wait(self)
class Binpkg(CompositeTask): __slots__ = ("find_blockers", "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", "prefetcher", "settings", "world_atom") + \ ("_bintree", "_build_dir", "_build_prefix", "_ebuild_path", "_fetched_pkg", "_image_dir", "_infloc", "_pkg_path", "_tree", "_verify") def _writemsg_level(self, msg, level=0, noiselevel=0): self.scheduler.output(msg, level=level, noiselevel=noiselevel, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher) def _prefetch_exit(self, prefetcher): if self._was_cancelled(): self.wait() return pkg = self.pkg pkg_count = self.pkg_count if not (self.opts.pretend or self.opts.fetchonly): self._build_dir.lock() # Initialize PORTAGE_LOG_FILE (clean_log won't work without it). portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) # If necessary, discard old log so that we don't # append to it. self._build_dir.clean_log() fetcher = BinpkgFetcher(background=self.background, logfile=self.settings.get("PORTAGE_LOG_FILE"), pkg=self.pkg, pretend=self.opts.pretend, scheduler=self.scheduler) if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv): msg = " --- (%s of %s) Fetching Binary (%s::%s)" %\ (pkg_count.curval, pkg_count.maxval, pkg.cpv, fetcher.pkg_path) short_msg = "emerge: (%s of %s) %s Fetch" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) self.logger.log(msg, short_msg=short_msg) # Allow the Scheduler's fetch queue to control the # number of concurrent fetchers. fetcher.addExitListener(self._fetcher_exit) self._task_queued(fetcher) self.scheduler.fetch.schedule(fetcher) return self._fetcher_exit(fetcher) def _fetcher_exit(self, fetcher): # The fetcher only has a returncode when # --getbinpkg is enabled. if fetcher.returncode is not None: self._fetched_pkg = fetcher.pkg_path if self._default_exit(fetcher) != os.EX_OK: self._unlock_builddir() self.wait() return if self.opts.pretend: self._current_task = None self.returncode = os.EX_OK self.wait() return verifier = None if self._verify: if self._fetched_pkg: path = self._fetched_pkg else: path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) logfile = self.settings.get("PORTAGE_LOG_FILE") verifier = BinpkgVerifier(background=self.background, logfile=logfile, pkg=self.pkg, scheduler=self.scheduler, _pkg_path=path) self._start_task(verifier, self._verifier_exit) return self._verifier_exit(verifier) def _verifier_exit(self, verifier): if verifier is not None and \ self._default_exit(verifier) != os.EX_OK: self._unlock_builddir() self.wait() return logger = self.logger pkg = self.pkg pkg_count = self.pkg_count if self._fetched_pkg: pkg_path = self._bintree.getname(self._bintree.inject( pkg.cpv, filename=self._fetched_pkg), allocate_new=False) else: pkg_path = self.pkg.root_config.trees["bintree"].getname( self.pkg.cpv) # This gives bashrc users an opportunity to do various things # such as remove binary packages after they're installed. self.settings["PORTAGE_BINPKG_FILE"] = pkg_path self._pkg_path = pkg_path logfile = self.settings.get("PORTAGE_LOG_FILE") if logfile is not None and os.path.isfile(logfile): # Remove fetch log after successful fetch. try: os.unlink(logfile) except OSError: pass if self.opts.fetchonly: self._current_task = None self.returncode = os.EX_OK self.wait() return msg = " === (%s of %s) Merging Binary (%s::%s)" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv, pkg_path) short_msg = "emerge: (%s of %s) %s Merge Binary" % \ (pkg_count.curval, pkg_count.maxval, pkg.cpv) logger.log(msg, short_msg=short_msg) phase = "clean" settings = self.settings ebuild_phase = EbuildPhase(background=self.background, phase=phase, scheduler=self.scheduler, settings=settings) self._start_task(ebuild_phase, self._clean_exit) def _clean_exit(self, clean_phase): if self._default_exit(clean_phase) != os.EX_OK: self._unlock_builddir() self.wait() return dir_path = self.settings['PORTAGE_BUILDDIR'] infloc = self._infloc pkg = self.pkg pkg_path = self._pkg_path dir_mode = 0o755 for mydir in (dir_path, self._image_dir, infloc): portage.util.ensure_dirs(mydir, uid=portage.data.portage_uid, gid=portage.data.portage_gid, mode=dir_mode) # This initializes PORTAGE_LOG_FILE. portage.prepare_build_dirs(self.settings["ROOT"], self.settings, 1) self._writemsg_level(">>> Extracting info\n") pkg_xpak = portage.xpak.tbz2(self._pkg_path) check_missing_metadata = ("CATEGORY", "PF") missing_metadata = set() for k in check_missing_metadata: v = pkg_xpak.getfile( _unicode_encode(k, encoding=_encodings['repo.content'])) if not v: missing_metadata.add(k) pkg_xpak.unpackinfo(infloc) for k in missing_metadata: if k == "CATEGORY": v = pkg.category elif k == "PF": v = pkg.pf else: continue f = io.open(_unicode_encode(os.path.join(infloc, k), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='backslashreplace') try: f.write(_unicode_decode(v + "\n")) finally: f.close() # Store the md5sum in the vdb. f = io.open(_unicode_encode(os.path.join(infloc, 'BINPKGMD5'), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['content'], errors='strict') try: f.write( _unicode_decode( str(portage.checksum.perform_md5(pkg_path)) + "\n")) finally: f.close() env_extractor = BinpkgEnvExtractor(background=self.background, scheduler=self.scheduler, settings=self.settings) self._start_task(env_extractor, self._env_extractor_exit) def _env_extractor_exit(self, env_extractor): if self._default_exit(env_extractor) != os.EX_OK: self._unlock_builddir() self.wait() return setup_phase = EbuildPhase(background=self.background, phase="setup", scheduler=self.scheduler, settings=self.settings) setup_phase.addExitListener(self._setup_exit) self._task_queued(setup_phase) self.scheduler.scheduleSetup(setup_phase) def _setup_exit(self, setup_phase): if self._default_exit(setup_phase) != os.EX_OK: self._unlock_builddir() self.wait() return extractor = BinpkgExtractorAsync( background=self.background, env=self.settings.environ(), features=self.settings.features, image_dir=self._image_dir, pkg=self.pkg, pkg_path=self._pkg_path, logfile=self.settings.get("PORTAGE_LOG_FILE"), scheduler=self.scheduler) self._writemsg_level(">>> Extracting %s\n" % self.pkg.cpv) self._start_task(extractor, self._extractor_exit) def _extractor_exit(self, extractor): if self._default_exit(extractor) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Extracting '%s'\n" % \ self._pkg_path, noiselevel=-1, level=logging.ERROR) self.wait() return try: with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['repo.content'], errors='replace') as f: self._build_prefix = f.read().rstrip('\n') except IOError: self._build_prefix = "" if self._build_prefix == self.settings["EPREFIX"]: ensure_dirs(self.settings["ED"]) self._current_task = None self.returncode = os.EX_OK self.wait() return env = self.settings.environ() env["PYTHONPATH"] = self.settings["PORTAGE_PYTHONPATH"] chpathtool = SpawnProcess( args=[ portage._python_interpreter, os.path.join(self.settings["PORTAGE_BIN_PATH"], "chpathtool.py"), self.settings["D"], self._build_prefix, self.settings["EPREFIX"] ], background=self.background, env=env, scheduler=self.scheduler, logfile=self.settings.get('PORTAGE_LOG_FILE')) self._writemsg_level(">>> Adjusting Prefix to %s\n" % self.settings["EPREFIX"]) self._start_task(chpathtool, self._chpathtool_exit) def _chpathtool_exit(self, chpathtool): if self._final_exit(chpathtool) != os.EX_OK: self._unlock_builddir() self._writemsg_level("!!! Error Adjusting Prefix to %s\n" % (self.settings["EPREFIX"], ), noiselevel=-1, level=logging.ERROR) self.wait() return # We want to install in "our" prefix, not the binary one with io.open(_unicode_encode(os.path.join(self._infloc, "EPREFIX"), encoding=_encodings['fs'], errors='strict'), mode='w', encoding=_encodings['repo.content'], errors='strict') as f: f.write(self.settings["EPREFIX"] + "\n") # Move the files to the correct location for merge. image_tmp_dir = os.path.join(self.settings["PORTAGE_BUILDDIR"], "image_tmp") build_d = os.path.join(self.settings["D"], self._build_prefix.lstrip(os.sep)) if not os.path.isdir(build_d): # Assume this is a virtual package or something. shutil.rmtree(self._image_dir) ensure_dirs(self.settings["ED"]) else: os.rename(build_d, image_tmp_dir) shutil.rmtree(self._image_dir) ensure_dirs(os.path.dirname(self.settings["ED"].rstrip(os.sep))) os.rename(image_tmp_dir, self.settings["ED"]) self.wait() def _unlock_builddir(self): if self.opts.pretend or self.opts.fetchonly: return portage.elog.elog_process(self.pkg.cpv, self.settings) self._build_dir.unlock() def create_install_task(self): task = EbuildMerge(exit_hook=self._install_exit, find_blockers=self.find_blockers, ldpath_mtimes=self.ldpath_mtimes, logger=self.logger, pkg=self.pkg, pkg_count=self.pkg_count, pkg_path=self._pkg_path, scheduler=self.scheduler, settings=self.settings, tree=self._tree, world_atom=self.world_atom) return task def _install_exit(self, task): self.settings.pop("PORTAGE_BINPKG_FILE", None) self._unlock_builddir() if task.returncode == os.EX_OK and \ 'binpkg-logs' not in self.settings.features and \ self.settings.get("PORTAGE_LOG_FILE"): try: os.unlink(self.settings["PORTAGE_LOG_FILE"]) except OSError: pass
def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call(['mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call(['mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage]) with open(os.path.join( cgroup_portage, 'release_agent'), 'w') as f: f.write(os.path.join(self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) with open(os.path.join( cgroup_portage, 'notify_on_release'), 'w') as f: f.write('1') else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join( cgroup_portage, 'release_agent') try: with open(release_agent) as f: release_agent_path = f.readline().rstrip('\n') except EnvironmentError: release_agent_path = None if (release_agent_path is None or not os.path.exists(release_agent_path)): with open(release_agent, 'w') as f: f.write(os.path.join( self.settings['PORTAGE_BIN_PATH'], 'cgroup-release-agent')) cgroup_path = tempfile.mkdtemp(dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial(self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon)) return else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir( scheduler=event_loop, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess( args=[BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 #600.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_ms) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)
def testIpcDaemon(self): tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] task_scheduler = TaskScheduler(max_jobs=2) build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.clear() task_scheduler.wait() exit_command.reply_hook = exit_command_callback start_time = time.time() task_scheduler.add(daemon) task_scheduler.add(proc) task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT) task_scheduler.clear() task_scheduler.wait() hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for QueueScheduler.run() sleep_time_s = 10 # 10.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.clear() task_scheduler.wait() exit_command.reply_hook = exit_command_callback start_time = time.time() task_scheduler.add(daemon) task_scheduler.add(proc) task_scheduler.run(timeout=short_timeout_ms) task_scheduler.clear() task_scheduler.wait() hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)