def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge, )
def _start(self): pkg = self.pkg settings = self.settings if not self.opts.fetchonly: rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f %s` in another terminal." \ % (_emerge.emergelog._emerge_log_dir) msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) if aux_get_task.cancelled: self._default_final_exit(aux_get_task) return pkg = self.pkg settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._builddir_lock.lock() portage.prepare_build_dirs( settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,)) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit)
def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = self._get_ebuild_path() try: uri_map = self._get_uri_map() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) self._eerror(msg_lines) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return if not uri_map: # Nothing to fetch. self._set_returncode((self.pid, os.EX_OK << 8)) self._async_wait() return settings = self.config_pool.allocate() settings.setcpv(self.pkg) portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) if self.prefetch and \ self._prefetch_size_ok(uri_map, settings, ebuild_path): self.config_pool.deallocate(settings) self._set_returncode((self.pid, os.EX_OK << 8)) self._async_wait() return nocolor = settings.get("NOCOLOR") if self.prefetch: settings["PORTAGE_PARALLEL_FETCHONLY"] = "1" if self.background: nocolor = "true" if nocolor is not None: settings["NOCOLOR"] = nocolor self._settings = settings ForkProcess._start(self) # Free settings now since it's no longer needed in # this process (the subprocess has a private copy). self.config_pool.deallocate(settings) settings = None self._settings = None
def _start_with_metadata(self, aux_get_task): self._assert_current(aux_get_task) if aux_get_task.cancelled: self._default_final_exit(aux_get_task) return pkg = self.pkg settings = self.settings root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result() settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching files in the background.', 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend dir_path = os.path.join(settings["PORTAGE_TMPDIR"], "portage", pkg.category, pkg.pf) self._build_dir = EbuildBuildDir(dir_path=dir_path, pkg=pkg, settings=settings) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path debug = settings.get("PORTAGE_DEBUG") == "1" portage.doebuild_environment(self._ebuild_path, "setup", settings["ROOT"], settings, debug, 1, self._bintree.dbapi) settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif not prefetcher.isAlive(): prefetcher.cancel() elif prefetcher.poll() is None: waiting_msg = ("Fetching '%s' " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal.") % prefetcher.pkg_path msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = self._get_ebuild_path() try: uri_map = self._get_uri_map() except portage.exception.InvalidDependString as e: msg_lines = [] msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ (self.pkg.cpv, e) msg_lines.append(msg) self._eerror(msg_lines) self._set_returncode((self.pid, 1 << 8)) self.wait() return if not uri_map: # Nothing to fetch. self._set_returncode((self.pid, os.EX_OK << 8)) self.wait() return settings = self.config_pool.allocate() settings.setcpv(self.pkg) portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) if self.prefetch and \ self._prefetch_size_ok(uri_map, settings, ebuild_path): self.config_pool.deallocate(settings) self._set_returncode((self.pid, os.EX_OK << 8)) self.wait() return nocolor = settings.get("NOCOLOR") if self.prefetch: settings["PORTAGE_PARALLEL_FETCHONLY"] = "1" if self.background: nocolor = "true" if nocolor is not None: settings["NOCOLOR"] = nocolor self._settings = settings ForkProcess._start(self) # Free settings now since it's no longer needed in # this process (the subprocess has a private copy). self.config_pool.deallocate(settings) settings = None self._settings = None
def _start(self): root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = self._get_ebuild_path() # This is initialized by an earlier _async_uri_map call. uri_map = self._uri_map if not uri_map: # Nothing to fetch. self.returncode = os.EX_OK self._async_wait() return settings = self.config_pool.allocate() settings.setcpv(self.pkg) settings.configdict["pkg"]["SRC_URI"] = self.src_uri portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) if self.prefetch and \ self._prefetch_size_ok(uri_map, settings, ebuild_path): self.config_pool.deallocate(settings) self.returncode = os.EX_OK self._async_wait() return nocolor = settings.get("NOCOLOR") if self.prefetch: settings["PORTAGE_PARALLEL_FETCHONLY"] = "1" if self.background: nocolor = "true" if nocolor is not None: settings["NOCOLOR"] = nocolor self._settings = settings ForkProcess._start(self) # Free settings now since it's no longer needed in # this process (the subprocess has a private copy). self.config_pool.deallocate(settings) settings = None self._settings = None
def lock(self): """ This raises an AlreadyLocked exception if lock() is called while a lock is already held. In order to avoid this, call unlock() or check whether the "locked" attribute is True or False before calling lock(). """ if self._lock_obj is not None: raise self.AlreadyLocked((self._lock_obj, )) dir_path = self.dir_path if dir_path is None: root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv) settings = self.settings settings.setcpv(self.pkg) debug = settings.get("PORTAGE_DEBUG") == "1" use_cache = 1 # always true portage.doebuild_environment(ebuild_path, "setup", root_config.root, self.settings, debug, use_cache, portdb) dir_path = self.settings["PORTAGE_BUILDDIR"] catdir = os.path.dirname(dir_path) self._catdir = catdir portage.util.ensure_dirs(os.path.dirname(catdir), gid=portage.portage_gid, mode=0o70, mask=0) catdir_lock = None try: catdir_lock = portage.locks.lockdir(catdir) portage.util.ensure_dirs(catdir, gid=portage.portage_gid, mode=0o70, mask=0) self._lock_obj = portage.locks.lockdir(dir_path) finally: self.locked = self._lock_obj is not None if catdir_lock is not None: portage.locks.unlockdir(catdir_lock)
def lock(self): """ This raises an AlreadyLocked exception if lock() is called while a lock is already held. In order to avoid this, call unlock() or check whether the "locked" attribute is True or False before calling lock(). """ if self._lock_obj is not None: raise self.AlreadyLocked((self._lock_obj,)) dir_path = self.dir_path if dir_path is None: root_config = self.pkg.root_config portdb = root_config.trees["porttree"].dbapi ebuild_path = portdb.findname(self.pkg.cpv) if ebuild_path is None: raise AssertionError( "ebuild not found for '%s'" % self.pkg.cpv) settings = self.settings settings.setcpv(self.pkg) debug = settings.get("PORTAGE_DEBUG") == "1" use_cache = 1 # always true portage.doebuild_environment(ebuild_path, "setup", root_config.root, self.settings, debug, use_cache, portdb) dir_path = self.settings["PORTAGE_BUILDDIR"] catdir = os.path.dirname(dir_path) self._catdir = catdir portage.util.ensure_dirs(os.path.dirname(catdir), gid=portage.portage_gid, mode=0o70, mask=0) catdir_lock = None try: catdir_lock = portage.locks.lockdir(catdir) portage.util.ensure_dirs(catdir, gid=portage.portage_gid, mode=0o70, mask=0) self._lock_obj = portage.locks.lockdir(dir_path) finally: self.locked = self._lock_obj is not None if catdir_lock is not None: portage.locks.unlockdir(catdir_lock)
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _start(self): pkg = self.pkg settings = self.settings settings.setcpv(pkg) self._tree = "bintree" self._bintree = self.pkg.root_config.trees[self._tree] self._verify = not self.opts.pretend # Use realpath like doebuild_environment() does, since we assert # that this path is literally identical to PORTAGE_BUILDDIR. dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]), "portage", pkg.category, pkg.pf) self._image_dir = os.path.join(dir_path, "image") self._infloc = os.path.join(dir_path, "build-info") self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild") settings["EBUILD"] = self._ebuild_path portage.doebuild_environment(self._ebuild_path, 'setup', settings=self.settings, db=self._bintree.dbapi) if dir_path != self.settings['PORTAGE_BUILDDIR']: raise AssertionError("'%s' != '%s'" % \ (dir_path, self.settings['PORTAGE_BUILDDIR'])) self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=settings) settings.configdict["pkg"]["EMERGE_FROM"] = "binary" settings.configdict["pkg"]["MERGE_TYPE"] = "binary" if eapi_exports_replace_vars(settings["EAPI"]): vardb = self.pkg.root_config.trees["vartree"].dbapi settings["REPLACING_VERSIONS"] = " ".join( set(portage.versions.cpv_getversion(x) \ for x in vardb.match(self.pkg.slot_atom) + \ vardb.match('='+self.pkg.cpv))) # The prefetcher has already completed or it # could be running now. If it's running now, # wait for it to complete since it holds # a lock on the file being fetched. The # portage.locks functions are only designed # to work between separate processes. Since # the lock is held by the current process, # use the scheduler and fetcher methods to # synchronize with the fetcher. prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: if not self.background: fetch_log = os.path.join( _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') msg = ( 'Fetching in the background:', prefetcher.pkg_path, 'To view fetch progress, run in another terminal:', 'tail -f %s' % fetch_log, ) out = portage.output.EOutput() for l in msg: out.einfo(l) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def testEbuildFetch(self): user_config = { "make.conf": ( 'GENTOO_MIRRORS="{scheme}://{host}:{port}"', ), } distfiles = { 'bar': b'bar\n', 'foo': b'foo\n', } ebuilds = { 'dev-libs/A-1': { 'EAPI': '7', 'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar {scheme}://{host}:{port}/distfiles/foo.txt -> foo''', }, } loop = SchedulerInterface(global_event_loop()) def run_async(func, *args, **kwargs): with ForkExecutor(loop=loop) as executor: return loop.run_until_complete(loop.run_in_executor(executor, functools.partial(func, *args, **kwargs))) scheme = 'http' host = '127.0.0.1' content = {} content['/distfiles/layout.conf'] = b'[structure]\n0=flat\n' for k, v in distfiles.items(): # mirror path content['/distfiles/{}'.format(k)] = v # upstream path content['/distfiles/{}.txt'.format(k)] = v with AsyncHTTPServer(host, content, loop) as server: ebuilds_subst = {} for cpv, metadata in ebuilds.items(): metadata = metadata.copy() metadata['SRC_URI'] = metadata['SRC_URI'].format( scheme=scheme, host=host, port=server.server_port) ebuilds_subst[cpv] = metadata user_config_subst = user_config.copy() for configname, configdata in user_config.items(): configdata_sub = [] for line in configdata: configdata_sub.append(line.format( scheme=scheme, host=host, port=server.server_port)) user_config_subst[configname] = tuple(configdata_sub) playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles, user_config=user_config_subst) ro_distdir = tempfile.mkdtemp() eubin = os.path.join(playground.eprefix, "usr", "bin") try: fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND']) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND'])) os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin))) resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND']) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND'])) if resume_bin != fetch_bin: os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin))) root_config = playground.trees[playground.eroot]['root_config'] portdb = root_config.trees["porttree"].dbapi settings = config(clone=playground.settings) # Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given. foo_uri = {'foo': ('{scheme}://{host}:{port}/distfiles/foo'.format(scheme=scheme, host=host, port=server.server_port),)} foo_path = os.path.join(settings['DISTDIR'], 'foo') foo_stale_content = b'stale content\n' with open(foo_path, 'wb') as f: f.write(b'stale content\n') self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False))) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), foo_stale_content) with open(foo_path, 'rb') as f: self.assertNotEqual(f.read(), distfiles['foo']) # Use force=True to update the stale file. self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), distfiles['foo']) # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR. # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that # FETCHCOMMAND must perform atomic rename itself due to read-only # DISTDIR. with open(foo_path, 'wb') as f: f.write(b'stale content\n') orig_fetchcommand = settings['FETCHCOMMAND'] orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode temp_fetchcommand = os.path.join(eubin, 'fetchcommand') with open(temp_fetchcommand, 'w') as f: f.write(""" set -e URI=$1 DISTDIR=$2 FILE=$3 trap 'chmod a-w "${DISTDIR}"' EXIT chmod ug+w "${DISTDIR}" %s mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}" """ % orig_fetchcommand.replace('${FILE}', '${FILE}.__download__')) settings['FETCHCOMMAND'] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (BASH_BINARY, temp_fetchcommand) settings.features.add('skiprocheck') settings.features.remove('distlocks') os.chmod(settings['DISTDIR'], 0o555) try: self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) finally: settings['FETCHCOMMAND'] = orig_fetchcommand os.chmod(settings['DISTDIR'], orig_distdir_mode) settings.features.remove('skiprocheck') settings.features.add('distlocks') os.unlink(temp_fetchcommand) with open(foo_path, 'rb') as f: self.assertEqual(f.read(), distfiles['foo']) # Test emirrordist invocation. emirrordist_cmd = (portage._python_interpreter, '-b', '-Wd', os.path.join(self.bindir, 'emirrordist'), '--distfiles', settings['DISTDIR'], '--config-root', settings['EPREFIX'], '--repositories-configuration', settings.repositories.config_string(), '--repo', 'test_repo', '--mirror') env = settings.environ() env['PYTHONPATH'] = ':'.join( filter(None, [PORTAGE_PYM_PATH] + os.environ.get('PYTHONPATH', '').split(':'))) for k in distfiles: os.unlink(os.path.join(settings['DISTDIR'], k)) proc = loop.run_until_complete(asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)) self.assertEqual(loop.run_until_complete(proc.wait()), 0) for k in distfiles: with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Tests only work with one ebuild at a time, so the config # pool only needs a single config instance. class config_pool: @staticmethod def allocate(): return settings @staticmethod def deallocate(settings): pass def async_fetch(pkg, ebuild_path): fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path, fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop) fetcher.start() return fetcher.async_wait() for cpv in ebuilds: metadata = dict(zip(Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys))) pkg = Package(built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name='ebuild') settings.setcpv(pkg) ebuild_path = portdb.findname(pkg.cpv) portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) # Test good files in DISTDIR for k in settings['AA'].split(): os.stat(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test digestgen with fetch os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest')) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) with ForkExecutor(loop=loop) as executor: self.assertTrue(bool(loop.run_until_complete( loop.run_in_executor(executor, functools.partial( digestgen, mysettings=settings, myportdb=portdb))))) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test missing files in DISTDIR for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test empty files in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: pass self.assertEqual(os.stat(file_path).st_size, 0) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test non-empty files containing null bytes in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: f.write(len(distfiles[k]) * b'\0') self.assertEqual(os.stat(file_path).st_size, len(distfiles[k])) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test PORTAGE_RO_DISTDIRS settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir) orig_fetchcommand = settings['FETCHCOMMAND'] orig_resumecommand = settings['RESUMECOMMAND'] try: settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.rename(file_path, os.path.join(ro_distdir, k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) self.assertTrue(os.path.islink(file_path)) with open(file_path, 'rb') as f: self.assertEqual(f.read(), distfiles[k]) os.unlink(file_path) finally: settings.pop('PORTAGE_RO_DISTDIRS') settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test local filesystem in GENTOO_MIRRORS orig_mirrors = settings['GENTOO_MIRRORS'] orig_fetchcommand = settings['FETCHCOMMAND'] try: settings['GENTOO_MIRRORS'] = ro_distdir settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['GENTOO_MIRRORS'] = orig_mirrors settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test readonly DISTDIR orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode try: os.chmod(settings['DISTDIR'], 0o555) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: os.chmod(settings['DISTDIR'], orig_distdir_mode) # Test parallel-fetch mode settings['PORTAGE_PARALLEL_FETCHONLY'] = '1' try: self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings.pop('PORTAGE_PARALLEL_FETCHONLY') # Test RESUMECOMMAND orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] try: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.unlink(file_path) with open(file_path + _download_suffix, 'wb') as f: f.write(distfiles[k][:2]) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size # Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR orig_fetchcommand = settings['FETCHCOMMAND'] orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) try: os.chmod(settings['DISTDIR'], 0o555) settings['FETCHCOMMAND'] = '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"')) settings.features.add('skiprocheck') settings.features.remove('distlocks') self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) finally: settings['FETCHCOMMAND'] = orig_fetchcommand os.chmod(settings['DISTDIR'], orig_distdir_mode) settings.features.remove('skiprocheck') settings.features.add('distlocks') finally: shutil.rmtree(ro_distdir) playground.cleanup()
def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._builddir_lock.lock() portage.prepare_build_dirs(settings=self.settings, cleanup=True) # Output only gets logged if it comes after prepare_build_dirs() # which initializes PORTAGE_LOG_FILE. retval, pkgmap = _unmerge_display(self.pkg.root_config, self.opts, "unmerge", [self.pkg.cpv], clean_delay=0, writemsg_level=self._writemsg_level) if retval != os.EX_OK: self._builddir_lock.unlock() self.returncode = retval self._async_wait() return self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ), noiselevel=-1) self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, )) unmerge_task = MergeProcess( mycat=cat, mypkg=pf, settings=self.settings, treetype="vartree", vartree=self.pkg.root_config.trees["vartree"], scheduler=self.scheduler, background=self.background, mydbapi=self.pkg.root_config.trees["vartree"].dbapi, prev_mtimes=self.ldpath_mtimes, logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True) self._start_task(unmerge_task, self._unmerge_exit)
def _start(self): pkg = self.pkg settings = self.settings if not self.opts.fetchonly: rval = _check_temp_dir(settings) if rval != os.EX_OK: self.returncode = rval self._current_task = None self._async_wait() return root_config = pkg.root_config tree = "porttree" self._tree = tree portdb = root_config.trees[tree].dbapi settings.setcpv(pkg) settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" if self.opts.buildpkgonly: settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" else: settings.configdict["pkg"]["MERGE_TYPE"] = "source" ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) if ebuild_path is None: raise AssertionError("ebuild not found for '%s'" % pkg.cpv) self._ebuild_path = ebuild_path portage.doebuild_environment(ebuild_path, 'setup', settings=self.settings, db=portdb) # Check the manifest here since with --keep-going mode it's # currently possible to get this far with a broken manifest. if not self._check_manifest(): self.returncode = 1 self._current_task = None self._async_wait() return prefetcher = self.prefetcher if prefetcher is None: pass elif prefetcher.isAlive() and \ prefetcher.poll() is None: waiting_msg = "Fetching files " + \ "in the background. " + \ "To view fetch progress, run `tail -f " + \ "/var/log/emerge-fetch.log` in another " + \ "terminal." msg_prefix = colorize("GOOD", " * ") from textwrap import wrap waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \ for line in wrap(waiting_msg, 65)) if not self.background: writemsg(waiting_msg, noiselevel=-1) self._current_task = prefetcher prefetcher.addExitListener(self._prefetch_exit) return self._prefetch_exit(prefetcher)
def _testEbuildFetch( self, loop, scheme, host, orig_distfiles, ebuilds, content, server, playground, ro_distdir, ): mirror_layouts = ( ( "[structure]", "0=filename-hash BLAKE2B 8", "1=flat", ), ( "[structure]", "1=filename-hash BLAKE2B 8", "0=flat", ), ( "[structure]", "0=content-hash SHA512 8:8:8", "1=flat", ), ) fetchcommand = portage.util.shlex_split( playground.settings["FETCHCOMMAND"]) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest("FETCHCOMMAND not found: {}".format( playground.settings["FETCHCOMMAND"])) eubin = os.path.join(playground.eprefix, "usr", "bin") os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin))) resumecommand = portage.util.shlex_split( playground.settings["RESUMECOMMAND"]) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest("RESUMECOMMAND not found: {}".format( playground.settings["RESUMECOMMAND"])) if resume_bin != fetch_bin: os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin))) root_config = playground.trees[playground.eroot]["root_config"] portdb = root_config.trees["porttree"].dbapi def run_async(func, *args, **kwargs): with ForkExecutor(loop=loop) as executor: return loop.run_until_complete( loop.run_in_executor( executor, functools.partial(func, *args, **kwargs))) for layout_lines in mirror_layouts: settings = config(clone=playground.settings) layout_data = "".join("{}\n".format(line) for line in layout_lines) mirror_conf = MirrorLayoutConfig() mirror_conf.read_from_file(io.StringIO(layout_data)) layouts = mirror_conf.get_all_layouts() content["/distfiles/layout.conf"] = layout_data.encode("utf8") distfiles = {} for k, v in orig_distfiles.items(): filename = DistfileName( k, digests=dict((algo, checksum_str(v, hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS), ) distfiles[filename] = v # mirror path for layout in layouts: content["/distfiles/" + layout.get_path(filename)] = v # upstream path content["/distfiles/{}.txt".format(k)] = v shutil.rmtree(settings["DISTDIR"]) os.makedirs(settings["DISTDIR"]) with open(os.path.join(settings["DISTDIR"], "layout.conf"), "wt") as f: f.write(layout_data) if any( isinstance(layout, ContentHashLayout) for layout in layouts): content_db = os.path.join(playground.eprefix, "var/db/emirrordist/content.db") os.makedirs(os.path.dirname(content_db), exist_ok=True) try: os.unlink(content_db) except OSError: pass else: content_db = None # Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given. foo_uri = { "foo": ("{scheme}://{host}:{port}/distfiles/foo".format( scheme=scheme, host=host, port=server.server_port), ) } foo_path = os.path.join(settings["DISTDIR"], "foo") foo_stale_content = b"stale content\n" with open(foo_path, "wb") as f: f.write(b"stale content\n") self.assertTrue( bool(run_async(fetch, foo_uri, settings, try_mirrors=False))) with open(foo_path, "rb") as f: self.assertEqual(f.read(), foo_stale_content) with open(foo_path, "rb") as f: self.assertNotEqual(f.read(), distfiles["foo"]) # Use force=True to update the stale file. self.assertTrue( bool( run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) with open(foo_path, "rb") as f: self.assertEqual(f.read(), distfiles["foo"]) # Test force=True with FEATURES=skiprocheck, using read-only DISTDIR. # FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that # FETCHCOMMAND must perform atomic rename itself due to read-only # DISTDIR. with open(foo_path, "wb") as f: f.write(b"stale content\n") orig_fetchcommand = settings["FETCHCOMMAND"] orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode temp_fetchcommand = os.path.join(eubin, "fetchcommand") with open(temp_fetchcommand, "w") as f: f.write(""" set -e URI=$1 DISTDIR=$2 FILE=$3 trap 'chmod a-w "${DISTDIR}"' EXIT chmod ug+w "${DISTDIR}" %s mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}" """ % orig_fetchcommand.replace("${FILE}", "${FILE}.__download__")) settings[ "FETCHCOMMAND"] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % ( BASH_BINARY, temp_fetchcommand, ) settings.features.add("skiprocheck") settings.features.remove("distlocks") os.chmod(settings["DISTDIR"], 0o555) try: self.assertTrue( bool( run_async(fetch, foo_uri, settings, try_mirrors=False, force=True))) finally: settings["FETCHCOMMAND"] = orig_fetchcommand os.chmod(settings["DISTDIR"], orig_distdir_mode) settings.features.remove("skiprocheck") settings.features.add("distlocks") os.unlink(temp_fetchcommand) with open(foo_path, "rb") as f: self.assertEqual(f.read(), distfiles["foo"]) # Test emirrordist invocation. emirrordist_cmd = ( portage._python_interpreter, "-b", "-Wd", os.path.join(self.bindir, "emirrordist"), "--distfiles", settings["DISTDIR"], "--config-root", settings["EPREFIX"], "--delete", "--repositories-configuration", settings.repositories.config_string(), "--repo", "test_repo", "--mirror", ) if content_db is not None: emirrordist_cmd = emirrordist_cmd + ( "--content-db", content_db, ) env = settings.environ() env["PYTHONPATH"] = ":".join( filter( None, [PORTAGE_PYM_PATH] + os.environ.get("PYTHONPATH", "").split(":"), )) for k in distfiles: try: os.unlink(os.path.join(settings["DISTDIR"], k)) except OSError: pass proc = loop.run_until_complete( asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)) self.assertEqual(loop.run_until_complete(proc.wait()), 0) for k in distfiles: with open( os.path.join(settings["DISTDIR"], layouts[0].get_path(k)), "rb") as f: self.assertEqual(f.read(), distfiles[k]) if content_db is not None: loop.run_until_complete( self._test_content_db( emirrordist_cmd, env, layouts, content_db, distfiles, settings, portdb, )) # Tests only work with one ebuild at a time, so the config # pool only needs a single config instance. class config_pool: @staticmethod def allocate(): return settings @staticmethod def deallocate(settings): pass def async_fetch(pkg, ebuild_path): fetcher = EbuildFetcher( config_pool=config_pool, ebuild_path=ebuild_path, fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop, ) fetcher.start() return fetcher.async_wait() for cpv in ebuilds: metadata = dict( zip( Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys), )) pkg = Package( built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name="ebuild", ) settings.setcpv(pkg) ebuild_path = portdb.findname(pkg.cpv) portage.doebuild_environment(ebuild_path, "fetch", settings=settings, db=portdb) # Test good files in DISTDIR for k in settings["AA"].split(): os.stat(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test digestgen with fetch os.unlink( os.path.join(os.path.dirname(ebuild_path), "Manifest")) for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) with ForkExecutor(loop=loop) as executor: self.assertTrue( bool( loop.run_until_complete( loop.run_in_executor( executor, functools.partial(digestgen, mysettings=settings, myportdb=portdb), )))) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test missing files in DISTDIR for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test empty files in DISTDIR for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) with open(file_path, "wb") as f: pass self.assertEqual(os.stat(file_path).st_size, 0) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test non-empty files containing null bytes in DISTDIR for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) with open(file_path, "wb") as f: f.write(len(distfiles[k]) * b"\0") self.assertEqual( os.stat(file_path).st_size, len(distfiles[k])) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) # Test PORTAGE_RO_DISTDIRS settings["PORTAGE_RO_DISTDIRS"] = '"{}"'.format(ro_distdir) orig_fetchcommand = settings["FETCHCOMMAND"] orig_resumecommand = settings["RESUMECOMMAND"] try: settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = "" for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) os.rename(file_path, os.path.join(ro_distdir, k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) self.assertTrue(os.path.islink(file_path)) with open(file_path, "rb") as f: self.assertEqual(f.read(), distfiles[k]) os.unlink(file_path) finally: settings.pop("PORTAGE_RO_DISTDIRS") settings["FETCHCOMMAND"] = orig_fetchcommand settings["RESUMECOMMAND"] = orig_resumecommand # Test local filesystem in GENTOO_MIRRORS orig_mirrors = settings["GENTOO_MIRRORS"] orig_fetchcommand = settings["FETCHCOMMAND"] try: settings["GENTOO_MIRRORS"] = ro_distdir settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = "" self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings["GENTOO_MIRRORS"] = orig_mirrors settings["FETCHCOMMAND"] = orig_fetchcommand settings["RESUMECOMMAND"] = orig_resumecommand # Test readonly DISTDIR orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode try: os.chmod(settings["DISTDIR"], 0o555) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: os.chmod(settings["DISTDIR"], orig_distdir_mode) # Test parallel-fetch mode settings["PORTAGE_PARALLEL_FETCHONLY"] = "1" try: self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings.pop("PORTAGE_PARALLEL_FETCHONLY") # Test RESUMECOMMAND orig_resume_min_size = settings[ "PORTAGE_FETCH_RESUME_MIN_SIZE"] try: settings["PORTAGE_FETCH_RESUME_MIN_SIZE"] = "2" for k in settings["AA"].split(): file_path = os.path.join(settings["DISTDIR"], k) os.unlink(file_path) with open(file_path + _download_suffix, "wb") as f: f.write(distfiles[k][:2]) self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings["AA"].split(): with open(os.path.join(settings["DISTDIR"], k), "rb") as f: self.assertEqual(f.read(), distfiles[k]) finally: settings[ "PORTAGE_FETCH_RESUME_MIN_SIZE"] = orig_resume_min_size # Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR orig_fetchcommand = settings["FETCHCOMMAND"] orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode for k in settings["AA"].split(): os.unlink(os.path.join(settings["DISTDIR"], k)) try: os.chmod(settings["DISTDIR"], 0o555) settings["FETCHCOMMAND"] = ( '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"'))) settings.features.add("skiprocheck") settings.features.remove("distlocks") self.assertEqual( loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) finally: settings["FETCHCOMMAND"] = orig_fetchcommand os.chmod(settings["DISTDIR"], orig_distdir_mode) settings.features.remove("skiprocheck") settings.features.add("distlocks")
def testEbuildFetch(self): distfiles = { 'bar': b'bar\n', 'foo': b'foo\n', } ebuilds = { 'dev-libs/A-1': { 'EAPI': '7', 'RESTRICT': 'primaryuri', 'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar {scheme}://{host}:{port}/distfiles/foo.txt -> foo''', }, } loop = SchedulerInterface(global_event_loop()) scheme = 'http' host = '127.0.0.1' content = {} for k, v in distfiles.items(): content['/distfiles/{}.txt'.format(k)] = v with AsyncHTTPServer(host, content, loop) as server: ebuilds_subst = {} for cpv, metadata in ebuilds.items(): metadata = metadata.copy() metadata['SRC_URI'] = metadata['SRC_URI'].format( scheme=scheme, host=host, port=server.server_port) ebuilds_subst[cpv] = metadata playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles) ro_distdir = tempfile.mkdtemp() try: fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND']) fetch_bin = portage.process.find_binary(fetchcommand[0]) if fetch_bin is None: self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND'])) resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND']) resume_bin = portage.process.find_binary(resumecommand[0]) if resume_bin is None: self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND'])) root_config = playground.trees[playground.eroot]['root_config'] portdb = root_config.trees["porttree"].dbapi settings = config(clone=playground.settings) # Tests only work with one ebuild at a time, so the config # pool only needs a single config instance. class config_pool: @staticmethod def allocate(): return settings @staticmethod def deallocate(settings): pass def async_fetch(pkg, ebuild_path): fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path, fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop) fetcher.start() return fetcher.async_wait() for cpv in ebuilds: metadata = dict(zip(Package.metadata_keys, portdb.aux_get(cpv, Package.metadata_keys))) pkg = Package(built=False, cpv=cpv, installed=False, metadata=metadata, root_config=root_config, type_name='ebuild') settings.setcpv(pkg) ebuild_path = portdb.findname(pkg.cpv) portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb) # Test good files in DISTDIR for k in settings['AA'].split(): os.stat(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test digestgen with fetch os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest')) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) with ForkExecutor(loop=loop) as executor: self.assertTrue(bool(loop.run_until_complete( loop.run_in_executor(executor, functools.partial( digestgen, mysettings=settings, myportdb=portdb))))) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test missing files in DISTDIR for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test empty files in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: pass self.assertEqual(os.stat(file_path).st_size, 0) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test non-empty files containing null bytes in DISTDIR for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) with open(file_path, 'wb') as f: f.write(len(distfiles[k]) * b'\0') self.assertEqual(os.stat(file_path).st_size, len(distfiles[k])) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) # Test PORTAGE_RO_DISTDIRS settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir) orig_fetchcommand = settings['FETCHCOMMAND'] orig_resumecommand = settings['RESUMECOMMAND'] try: settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.rename(file_path, os.path.join(ro_distdir, k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) self.assertTrue(os.path.islink(file_path)) with open(file_path, 'rb') as f: self.assertEqual(f.read(), distfiles[k]) os.unlink(file_path) finally: settings.pop('PORTAGE_RO_DISTDIRS') settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test local filesystem in GENTOO_MIRRORS orig_mirrors = settings['GENTOO_MIRRORS'] orig_fetchcommand = settings['FETCHCOMMAND'] try: settings['GENTOO_MIRRORS'] = ro_distdir settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = '' self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['GENTOO_MIRRORS'] = orig_mirrors settings['FETCHCOMMAND'] = orig_fetchcommand settings['RESUMECOMMAND'] = orig_resumecommand # Test readonly DISTDIR orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode try: os.chmod(settings['DISTDIR'], 0o555) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: os.chmod(settings['DISTDIR'], orig_distdir_mode) # Test parallel-fetch mode settings['PORTAGE_PARALLEL_FETCHONLY'] = '1' try: self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) for k in settings['AA'].split(): os.unlink(os.path.join(settings['DISTDIR'], k)) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings.pop('PORTAGE_PARALLEL_FETCHONLY') # Test RESUMECOMMAND orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] try: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2' for k in settings['AA'].split(): file_path = os.path.join(settings['DISTDIR'], k) os.unlink(file_path) with open(file_path + _download_suffix, 'wb') as f: f.write(distfiles[k][:2]) self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0) for k in settings['AA'].split(): with open(os.path.join(settings['DISTDIR'], k), 'rb') as f: self.assertEqual(f.read(), distfiles[k]) finally: settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size finally: shutil.rmtree(ro_distdir) playground.cleanup()