def _start(self): vardb = self.pkg.root_config.trees["vartree"].dbapi dbdir = vardb.getpath(self.pkg.cpv) if not os.path.exists(dbdir): # Apparently the package got uninstalled # already, so we can safely return early. self.returncode = os.EX_OK self._async_wait() return self.settings.setcpv(self.pkg) cat, pf = portage.catsplit(self.pkg.cpv) myebuildpath = os.path.join(dbdir, pf + ".ebuild") try: portage.doebuild_environment(myebuildpath, "prerm", settings=self.settings, db=vardb) except UnsupportedAPIException: # This is safe to ignore since this function is # guaranteed to set PORTAGE_BUILDDIR even though # it raises UnsupportedAPIException. The error # will be logged when it prevents the pkg_prerm # and pkg_postrm phases from executing. pass self._builddir_lock = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._start_task( AsyncTaskFuture(future=self._builddir_lock.async_lock()), self._start_unmerge)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_task(
AsyncTaskFuture(future=self._builddir_lock.async_lock()),
self._start_unmerge,
)
def _start(self):
pkg = self.pkg
settings = self.settings
if not self.opts.fetchonly:
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path, 'setup',
settings=self.settings, db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f %s` in another terminal." \
% (_emerge.emergelog._emerge_log_dir)
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start_with_metadata(self, aux_get_task):
self._assert_current(aux_get_task)
if aux_get_task.cancelled:
self._default_final_exit(aux_get_task)
return
pkg = self.pkg
settings = self.settings
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result()
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path,
'setup',
settings=self.settings,
db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching files in the background.',
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath, "prerm",
settings=self.settings, db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(
settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts, "unmerge", [self.pkg.cpv], clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv,),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv,))
unmerge_task = MergeProcess(
mycat=cat, mypkg=pf, settings=self.settings,
treetype="vartree", vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler, background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"), unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = self._get_ebuild_path()
try:
uri_map = self._get_uri_map()
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
self._eerror(msg_lines)
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
if not uri_map:
# Nothing to fetch.
self._set_returncode((self.pid, os.EX_OK << 8))
self._async_wait()
return
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
portage.doebuild_environment(ebuild_path,
'fetch',
settings=settings,
db=portdb)
if self.prefetch and \
self._prefetch_size_ok(uri_map, settings, ebuild_path):
self.config_pool.deallocate(settings)
self._set_returncode((self.pid, os.EX_OK << 8))
self._async_wait()
return
nocolor = settings.get("NOCOLOR")
if self.prefetch:
settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
if self.background:
nocolor = "true"
if nocolor is not None:
settings["NOCOLOR"] = nocolor
self._settings = settings
ForkProcess._start(self)
# Free settings now since it's no longer needed in
# this process (the subprocess has a private copy).
self.config_pool.deallocate(settings)
settings = None
self._settings = None
def _start_with_metadata(self, aux_get_task):
self._assert_current(aux_get_task)
if aux_get_task.cancelled:
self._default_final_exit(aux_get_task)
return
pkg = self.pkg
settings = self.settings
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["SRC_URI"], = aux_get_task.future.result()
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path, 'setup',
settings=self.settings, db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log')
msg = (
'Fetching files in the background.',
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = self._get_ebuild_path()
try:
uri_map = self._get_uri_map()
except portage.exception.InvalidDependString as e:
msg_lines = []
msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \
(self.pkg.cpv, e)
msg_lines.append(msg)
self._eerror(msg_lines)
self._set_returncode((self.pid, 1 << 8))
self.wait()
return
if not uri_map:
# Nothing to fetch.
self._set_returncode((self.pid, os.EX_OK << 8))
self.wait()
return
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
portage.doebuild_environment(ebuild_path, 'fetch',
settings=settings, db=portdb)
if self.prefetch and \
self._prefetch_size_ok(uri_map, settings, ebuild_path):
self.config_pool.deallocate(settings)
self._set_returncode((self.pid, os.EX_OK << 8))
self.wait()
return
nocolor = settings.get("NOCOLOR")
if self.prefetch:
settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
if self.background:
nocolor = "true"
if nocolor is not None:
settings["NOCOLOR"] = nocolor
self._settings = settings
ForkProcess._start(self)
# Free settings now since it's no longer needed in
# this process (the subprocess has a private copy).
self.config_pool.deallocate(settings)
settings = None
self._settings = None
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = self._get_ebuild_path()
# This is initialized by an earlier _async_uri_map call.
uri_map = self._uri_map
if not uri_map:
# Nothing to fetch.
self.returncode = os.EX_OK
self._async_wait()
return
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
settings.configdict["pkg"]["SRC_URI"] = self.src_uri
portage.doebuild_environment(ebuild_path,
'fetch',
settings=settings,
db=portdb)
if self.prefetch and \
self._prefetch_size_ok(uri_map, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self._async_wait()
return
nocolor = settings.get("NOCOLOR")
if self.prefetch:
settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
if self.background:
nocolor = "true"
if nocolor is not None:
settings["NOCOLOR"] = nocolor
self._settings = settings
ForkProcess._start(self)
# Free settings now since it's no longer needed in
# this process (the subprocess has a private copy).
self.config_pool.deallocate(settings)
settings = None
self._settings = None
def lock(self):
"""
This raises an AlreadyLocked exception if lock() is called
while a lock is already held. In order to avoid this, call
unlock() or check whether the "locked" attribute is True
or False before calling lock().
"""
if self._lock_obj is not None:
raise self.AlreadyLocked((self._lock_obj, ))
dir_path = self.dir_path
if dir_path is None:
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" %
self.pkg.cpv)
settings = self.settings
settings.setcpv(self.pkg)
debug = settings.get("PORTAGE_DEBUG") == "1"
use_cache = 1 # always true
portage.doebuild_environment(ebuild_path, "setup",
root_config.root, self.settings,
debug, use_cache, portdb)
dir_path = self.settings["PORTAGE_BUILDDIR"]
catdir = os.path.dirname(dir_path)
self._catdir = catdir
portage.util.ensure_dirs(os.path.dirname(catdir),
gid=portage.portage_gid,
mode=0o70,
mask=0)
catdir_lock = None
try:
catdir_lock = portage.locks.lockdir(catdir)
portage.util.ensure_dirs(catdir,
gid=portage.portage_gid,
mode=0o70,
mask=0)
self._lock_obj = portage.locks.lockdir(dir_path)
finally:
self.locked = self._lock_obj is not None
if catdir_lock is not None:
portage.locks.unlockdir(catdir_lock)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = self._get_ebuild_path()
# This is initialized by an earlier _async_uri_map call.
uri_map = self._uri_map
if not uri_map:
# Nothing to fetch.
self.returncode = os.EX_OK
self._async_wait()
return
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
settings.configdict["pkg"]["SRC_URI"] = self.src_uri
portage.doebuild_environment(ebuild_path, 'fetch',
settings=settings, db=portdb)
if self.prefetch and \
self._prefetch_size_ok(uri_map, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self._async_wait()
return
nocolor = settings.get("NOCOLOR")
if self.prefetch:
settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
if self.background:
nocolor = "true"
if nocolor is not None:
settings["NOCOLOR"] = nocolor
self._settings = settings
ForkProcess._start(self)
# Free settings now since it's no longer needed in
# this process (the subprocess has a private copy).
self.config_pool.deallocate(settings)
settings = None
self._settings = None
def lock(self):
"""
This raises an AlreadyLocked exception if lock() is called
while a lock is already held. In order to avoid this, call
unlock() or check whether the "locked" attribute is True
or False before calling lock().
"""
if self._lock_obj is not None:
raise self.AlreadyLocked((self._lock_obj,))
dir_path = self.dir_path
if dir_path is None:
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError(
"ebuild not found for '%s'" % self.pkg.cpv)
settings = self.settings
settings.setcpv(self.pkg)
debug = settings.get("PORTAGE_DEBUG") == "1"
use_cache = 1 # always true
portage.doebuild_environment(ebuild_path, "setup", root_config.root,
self.settings, debug, use_cache, portdb)
dir_path = self.settings["PORTAGE_BUILDDIR"]
catdir = os.path.dirname(dir_path)
self._catdir = catdir
portage.util.ensure_dirs(os.path.dirname(catdir),
gid=portage.portage_gid,
mode=0o70, mask=0)
catdir_lock = None
try:
catdir_lock = portage.locks.lockdir(catdir)
portage.util.ensure_dirs(catdir,
gid=portage.portage_gid,
mode=0o70, mask=0)
self._lock_obj = portage.locks.lockdir(dir_path)
finally:
self.locked = self._lock_obj is not None
if catdir_lock is not None:
portage.locks.unlockdir(catdir_lock)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path,
'setup',
settings=self.settings,
db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path, 'setup',
settings=self.settings, db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(
_emerge.emergelog._emerge_log_dir, 'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def testEbuildFetch(self):
user_config = {
"make.conf":
(
'GENTOO_MIRRORS="{scheme}://{host}:{port}"',
),
}
distfiles = {
'bar': b'bar\n',
'foo': b'foo\n',
}
ebuilds = {
'dev-libs/A-1': {
'EAPI': '7',
'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar
{scheme}://{host}:{port}/distfiles/foo.txt -> foo''',
},
}
loop = SchedulerInterface(global_event_loop())
def run_async(func, *args, **kwargs):
with ForkExecutor(loop=loop) as executor:
return loop.run_until_complete(loop.run_in_executor(executor,
functools.partial(func, *args, **kwargs)))
scheme = 'http'
host = '127.0.0.1'
content = {}
content['/distfiles/layout.conf'] = b'[structure]\n0=flat\n'
for k, v in distfiles.items():
# mirror path
content['/distfiles/{}'.format(k)] = v
# upstream path
content['/distfiles/{}.txt'.format(k)] = v
with AsyncHTTPServer(host, content, loop) as server:
ebuilds_subst = {}
for cpv, metadata in ebuilds.items():
metadata = metadata.copy()
metadata['SRC_URI'] = metadata['SRC_URI'].format(
scheme=scheme, host=host, port=server.server_port)
ebuilds_subst[cpv] = metadata
user_config_subst = user_config.copy()
for configname, configdata in user_config.items():
configdata_sub = []
for line in configdata:
configdata_sub.append(line.format(
scheme=scheme, host=host, port=server.server_port))
user_config_subst[configname] = tuple(configdata_sub)
playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles, user_config=user_config_subst)
ro_distdir = tempfile.mkdtemp()
eubin = os.path.join(playground.eprefix, "usr", "bin")
try:
fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND'])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is None:
self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND']))
os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin)))
resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND'])
resume_bin = portage.process.find_binary(resumecommand[0])
if resume_bin is None:
self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND']))
if resume_bin != fetch_bin:
os.symlink(resume_bin, os.path.join(eubin, os.path.basename(resume_bin)))
root_config = playground.trees[playground.eroot]['root_config']
portdb = root_config.trees["porttree"].dbapi
settings = config(clone=playground.settings)
# Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given.
foo_uri = {'foo': ('{scheme}://{host}:{port}/distfiles/foo'.format(scheme=scheme, host=host, port=server.server_port),)}
foo_path = os.path.join(settings['DISTDIR'], 'foo')
foo_stale_content = b'stale content\n'
with open(foo_path, 'wb') as f:
f.write(b'stale content\n')
self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False)))
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), foo_stale_content)
with open(foo_path, 'rb') as f:
self.assertNotEqual(f.read(), distfiles['foo'])
# Use force=True to update the stale file.
self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True)))
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), distfiles['foo'])
# Test force=True with FEATURES=skiprocheck, using read-only DISTDIR.
# FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that
# FETCHCOMMAND must perform atomic rename itself due to read-only
# DISTDIR.
with open(foo_path, 'wb') as f:
f.write(b'stale content\n')
orig_fetchcommand = settings['FETCHCOMMAND']
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
temp_fetchcommand = os.path.join(eubin, 'fetchcommand')
with open(temp_fetchcommand, 'w') as f:
f.write("""
set -e
URI=$1
DISTDIR=$2
FILE=$3
trap 'chmod a-w "${DISTDIR}"' EXIT
chmod ug+w "${DISTDIR}"
%s
mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}"
""" % orig_fetchcommand.replace('${FILE}', '${FILE}.__download__'))
settings['FETCHCOMMAND'] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (BASH_BINARY, temp_fetchcommand)
settings.features.add('skiprocheck')
settings.features.remove('distlocks')
os.chmod(settings['DISTDIR'], 0o555)
try:
self.assertTrue(bool(run_async(fetch, foo_uri, settings, try_mirrors=False, force=True)))
finally:
settings['FETCHCOMMAND'] = orig_fetchcommand
os.chmod(settings['DISTDIR'], orig_distdir_mode)
settings.features.remove('skiprocheck')
settings.features.add('distlocks')
os.unlink(temp_fetchcommand)
with open(foo_path, 'rb') as f:
self.assertEqual(f.read(), distfiles['foo'])
# Test emirrordist invocation.
emirrordist_cmd = (portage._python_interpreter, '-b', '-Wd',
os.path.join(self.bindir, 'emirrordist'),
'--distfiles', settings['DISTDIR'],
'--config-root', settings['EPREFIX'],
'--repositories-configuration', settings.repositories.config_string(),
'--repo', 'test_repo', '--mirror')
env = settings.environ()
env['PYTHONPATH'] = ':'.join(
filter(None, [PORTAGE_PYM_PATH] + os.environ.get('PYTHONPATH', '').split(':')))
for k in distfiles:
os.unlink(os.path.join(settings['DISTDIR'], k))
proc = loop.run_until_complete(asyncio.create_subprocess_exec(*emirrordist_cmd, env=env))
self.assertEqual(loop.run_until_complete(proc.wait()), 0)
for k in distfiles:
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool:
@staticmethod
def allocate():
return settings
@staticmethod
def deallocate(settings):
pass
def async_fetch(pkg, ebuild_path):
fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path,
fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop)
fetcher.start()
return fetcher.async_wait()
for cpv in ebuilds:
metadata = dict(zip(Package.metadata_keys,
portdb.aux_get(cpv, Package.metadata_keys)))
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
ebuild_path = portdb.findname(pkg.cpv)
portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb)
# Test good files in DISTDIR
for k in settings['AA'].split():
os.stat(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test digestgen with fetch
os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest'))
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
with ForkExecutor(loop=loop) as executor:
self.assertTrue(bool(loop.run_until_complete(
loop.run_in_executor(executor, functools.partial(
digestgen, mysettings=settings, myportdb=portdb)))))
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test missing files in DISTDIR
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test empty files in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
pass
self.assertEqual(os.stat(file_path).st_size, 0)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test non-empty files containing null bytes in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
f.write(len(distfiles[k]) * b'\0')
self.assertEqual(os.stat(file_path).st_size, len(distfiles[k]))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test PORTAGE_RO_DISTDIRS
settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir)
orig_fetchcommand = settings['FETCHCOMMAND']
orig_resumecommand = settings['RESUMECOMMAND']
try:
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.rename(file_path, os.path.join(ro_distdir, k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
self.assertTrue(os.path.islink(file_path))
with open(file_path, 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
os.unlink(file_path)
finally:
settings.pop('PORTAGE_RO_DISTDIRS')
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test local filesystem in GENTOO_MIRRORS
orig_mirrors = settings['GENTOO_MIRRORS']
orig_fetchcommand = settings['FETCHCOMMAND']
try:
settings['GENTOO_MIRRORS'] = ro_distdir
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['GENTOO_MIRRORS'] = orig_mirrors
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test readonly DISTDIR
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
try:
os.chmod(settings['DISTDIR'], 0o555)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
os.chmod(settings['DISTDIR'], orig_distdir_mode)
# Test parallel-fetch mode
settings['PORTAGE_PARALLEL_FETCHONLY'] = '1'
try:
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings.pop('PORTAGE_PARALLEL_FETCHONLY')
# Test RESUMECOMMAND
orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE']
try:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2'
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.unlink(file_path)
with open(file_path + _download_suffix, 'wb') as f:
f.write(distfiles[k][:2])
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size
# Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR
orig_fetchcommand = settings['FETCHCOMMAND']
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
try:
os.chmod(settings['DISTDIR'], 0o555)
settings['FETCHCOMMAND'] = '"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"' % (BASH_BINARY, orig_fetchcommand.replace('"', '\\"'))
settings.features.add('skiprocheck')
settings.features.remove('distlocks')
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
finally:
settings['FETCHCOMMAND'] = orig_fetchcommand
os.chmod(settings['DISTDIR'], orig_distdir_mode)
settings.features.remove('skiprocheck')
settings.features.add('distlocks')
finally:
shutil.rmtree(ro_distdir)
playground.cleanup()
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts,
"unmerge", [self.pkg.cpv],
clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, ))
unmerge_task = MergeProcess(
mycat=cat,
mypkg=pf,
settings=self.settings,
treetype="vartree",
vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler,
background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
def _start(self):
pkg = self.pkg
settings = self.settings
if not self.opts.fetchonly:
rval = _check_temp_dir(settings)
if rval != os.EX_OK:
self.returncode = rval
self._current_task = None
self._async_wait()
return
root_config = pkg.root_config
tree = "porttree"
self._tree = tree
portdb = root_config.trees[tree].dbapi
settings.setcpv(pkg)
settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild"
if self.opts.buildpkgonly:
settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly"
else:
settings.configdict["pkg"]["MERGE_TYPE"] = "source"
ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % pkg.cpv)
self._ebuild_path = ebuild_path
portage.doebuild_environment(ebuild_path,
'setup',
settings=self.settings,
db=portdb)
# Check the manifest here since with --keep-going mode it's
# currently possible to get this far with a broken manifest.
if not self._check_manifest():
self.returncode = 1
self._current_task = None
self._async_wait()
return
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
waiting_msg = "Fetching files " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal."
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _testEbuildFetch(
self,
loop,
scheme,
host,
orig_distfiles,
ebuilds,
content,
server,
playground,
ro_distdir,
):
mirror_layouts = (
(
"[structure]",
"0=filename-hash BLAKE2B 8",
"1=flat",
),
(
"[structure]",
"1=filename-hash BLAKE2B 8",
"0=flat",
),
(
"[structure]",
"0=content-hash SHA512 8:8:8",
"1=flat",
),
)
fetchcommand = portage.util.shlex_split(
playground.settings["FETCHCOMMAND"])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is None:
self.skipTest("FETCHCOMMAND not found: {}".format(
playground.settings["FETCHCOMMAND"]))
eubin = os.path.join(playground.eprefix, "usr", "bin")
os.symlink(fetch_bin, os.path.join(eubin, os.path.basename(fetch_bin)))
resumecommand = portage.util.shlex_split(
playground.settings["RESUMECOMMAND"])
resume_bin = portage.process.find_binary(resumecommand[0])
if resume_bin is None:
self.skipTest("RESUMECOMMAND not found: {}".format(
playground.settings["RESUMECOMMAND"]))
if resume_bin != fetch_bin:
os.symlink(resume_bin,
os.path.join(eubin, os.path.basename(resume_bin)))
root_config = playground.trees[playground.eroot]["root_config"]
portdb = root_config.trees["porttree"].dbapi
def run_async(func, *args, **kwargs):
with ForkExecutor(loop=loop) as executor:
return loop.run_until_complete(
loop.run_in_executor(
executor, functools.partial(func, *args, **kwargs)))
for layout_lines in mirror_layouts:
settings = config(clone=playground.settings)
layout_data = "".join("{}\n".format(line) for line in layout_lines)
mirror_conf = MirrorLayoutConfig()
mirror_conf.read_from_file(io.StringIO(layout_data))
layouts = mirror_conf.get_all_layouts()
content["/distfiles/layout.conf"] = layout_data.encode("utf8")
distfiles = {}
for k, v in orig_distfiles.items():
filename = DistfileName(
k,
digests=dict((algo, checksum_str(v, hashname=algo))
for algo in MANIFEST2_HASH_DEFAULTS),
)
distfiles[filename] = v
# mirror path
for layout in layouts:
content["/distfiles/" + layout.get_path(filename)] = v
# upstream path
content["/distfiles/{}.txt".format(k)] = v
shutil.rmtree(settings["DISTDIR"])
os.makedirs(settings["DISTDIR"])
with open(os.path.join(settings["DISTDIR"], "layout.conf"),
"wt") as f:
f.write(layout_data)
if any(
isinstance(layout, ContentHashLayout)
for layout in layouts):
content_db = os.path.join(playground.eprefix,
"var/db/emirrordist/content.db")
os.makedirs(os.path.dirname(content_db), exist_ok=True)
try:
os.unlink(content_db)
except OSError:
pass
else:
content_db = None
# Demonstrate that fetch preserves a stale file in DISTDIR when no digests are given.
foo_uri = {
"foo": ("{scheme}://{host}:{port}/distfiles/foo".format(
scheme=scheme, host=host, port=server.server_port), )
}
foo_path = os.path.join(settings["DISTDIR"], "foo")
foo_stale_content = b"stale content\n"
with open(foo_path, "wb") as f:
f.write(b"stale content\n")
self.assertTrue(
bool(run_async(fetch, foo_uri, settings, try_mirrors=False)))
with open(foo_path, "rb") as f:
self.assertEqual(f.read(), foo_stale_content)
with open(foo_path, "rb") as f:
self.assertNotEqual(f.read(), distfiles["foo"])
# Use force=True to update the stale file.
self.assertTrue(
bool(
run_async(fetch,
foo_uri,
settings,
try_mirrors=False,
force=True)))
with open(foo_path, "rb") as f:
self.assertEqual(f.read(), distfiles["foo"])
# Test force=True with FEATURES=skiprocheck, using read-only DISTDIR.
# FETCHCOMMAND is set to temporarily chmod +w DISTDIR. Note that
# FETCHCOMMAND must perform atomic rename itself due to read-only
# DISTDIR.
with open(foo_path, "wb") as f:
f.write(b"stale content\n")
orig_fetchcommand = settings["FETCHCOMMAND"]
orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
temp_fetchcommand = os.path.join(eubin, "fetchcommand")
with open(temp_fetchcommand, "w") as f:
f.write("""
set -e
URI=$1
DISTDIR=$2
FILE=$3
trap 'chmod a-w "${DISTDIR}"' EXIT
chmod ug+w "${DISTDIR}"
%s
mv -f "${DISTDIR}/${FILE}.__download__" "${DISTDIR}/${FILE}"
""" % orig_fetchcommand.replace("${FILE}", "${FILE}.__download__"))
settings[
"FETCHCOMMAND"] = '"%s" "%s" "${URI}" "${DISTDIR}" "${FILE}"' % (
BASH_BINARY,
temp_fetchcommand,
)
settings.features.add("skiprocheck")
settings.features.remove("distlocks")
os.chmod(settings["DISTDIR"], 0o555)
try:
self.assertTrue(
bool(
run_async(fetch,
foo_uri,
settings,
try_mirrors=False,
force=True)))
finally:
settings["FETCHCOMMAND"] = orig_fetchcommand
os.chmod(settings["DISTDIR"], orig_distdir_mode)
settings.features.remove("skiprocheck")
settings.features.add("distlocks")
os.unlink(temp_fetchcommand)
with open(foo_path, "rb") as f:
self.assertEqual(f.read(), distfiles["foo"])
# Test emirrordist invocation.
emirrordist_cmd = (
portage._python_interpreter,
"-b",
"-Wd",
os.path.join(self.bindir, "emirrordist"),
"--distfiles",
settings["DISTDIR"],
"--config-root",
settings["EPREFIX"],
"--delete",
"--repositories-configuration",
settings.repositories.config_string(),
"--repo",
"test_repo",
"--mirror",
)
if content_db is not None:
emirrordist_cmd = emirrordist_cmd + (
"--content-db",
content_db,
)
env = settings.environ()
env["PYTHONPATH"] = ":".join(
filter(
None,
[PORTAGE_PYM_PATH] +
os.environ.get("PYTHONPATH", "").split(":"),
))
for k in distfiles:
try:
os.unlink(os.path.join(settings["DISTDIR"], k))
except OSError:
pass
proc = loop.run_until_complete(
asyncio.create_subprocess_exec(*emirrordist_cmd, env=env))
self.assertEqual(loop.run_until_complete(proc.wait()), 0)
for k in distfiles:
with open(
os.path.join(settings["DISTDIR"],
layouts[0].get_path(k)), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
if content_db is not None:
loop.run_until_complete(
self._test_content_db(
emirrordist_cmd,
env,
layouts,
content_db,
distfiles,
settings,
portdb,
))
# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool:
@staticmethod
def allocate():
return settings
@staticmethod
def deallocate(settings):
pass
def async_fetch(pkg, ebuild_path):
fetcher = EbuildFetcher(
config_pool=config_pool,
ebuild_path=ebuild_path,
fetchonly=False,
fetchall=True,
pkg=pkg,
scheduler=loop,
)
fetcher.start()
return fetcher.async_wait()
for cpv in ebuilds:
metadata = dict(
zip(
Package.metadata_keys,
portdb.aux_get(cpv, Package.metadata_keys),
))
pkg = Package(
built=False,
cpv=cpv,
installed=False,
metadata=metadata,
root_config=root_config,
type_name="ebuild",
)
settings.setcpv(pkg)
ebuild_path = portdb.findname(pkg.cpv)
portage.doebuild_environment(ebuild_path,
"fetch",
settings=settings,
db=portdb)
# Test good files in DISTDIR
for k in settings["AA"].split():
os.stat(os.path.join(settings["DISTDIR"], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
# Test digestgen with fetch
os.unlink(
os.path.join(os.path.dirname(ebuild_path), "Manifest"))
for k in settings["AA"].split():
os.unlink(os.path.join(settings["DISTDIR"], k))
with ForkExecutor(loop=loop) as executor:
self.assertTrue(
bool(
loop.run_until_complete(
loop.run_in_executor(
executor,
functools.partial(digestgen,
mysettings=settings,
myportdb=portdb),
))))
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
# Test missing files in DISTDIR
for k in settings["AA"].split():
os.unlink(os.path.join(settings["DISTDIR"], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
# Test empty files in DISTDIR
for k in settings["AA"].split():
file_path = os.path.join(settings["DISTDIR"], k)
with open(file_path, "wb") as f:
pass
self.assertEqual(os.stat(file_path).st_size, 0)
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
# Test non-empty files containing null bytes in DISTDIR
for k in settings["AA"].split():
file_path = os.path.join(settings["DISTDIR"], k)
with open(file_path, "wb") as f:
f.write(len(distfiles[k]) * b"\0")
self.assertEqual(
os.stat(file_path).st_size, len(distfiles[k]))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k), "rb") as f:
self.assertEqual(f.read(), distfiles[k])
# Test PORTAGE_RO_DISTDIRS
settings["PORTAGE_RO_DISTDIRS"] = '"{}"'.format(ro_distdir)
orig_fetchcommand = settings["FETCHCOMMAND"]
orig_resumecommand = settings["RESUMECOMMAND"]
try:
settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = ""
for k in settings["AA"].split():
file_path = os.path.join(settings["DISTDIR"], k)
os.rename(file_path, os.path.join(ro_distdir, k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
file_path = os.path.join(settings["DISTDIR"], k)
self.assertTrue(os.path.islink(file_path))
with open(file_path, "rb") as f:
self.assertEqual(f.read(), distfiles[k])
os.unlink(file_path)
finally:
settings.pop("PORTAGE_RO_DISTDIRS")
settings["FETCHCOMMAND"] = orig_fetchcommand
settings["RESUMECOMMAND"] = orig_resumecommand
# Test local filesystem in GENTOO_MIRRORS
orig_mirrors = settings["GENTOO_MIRRORS"]
orig_fetchcommand = settings["FETCHCOMMAND"]
try:
settings["GENTOO_MIRRORS"] = ro_distdir
settings["FETCHCOMMAND"] = settings["RESUMECOMMAND"] = ""
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings["GENTOO_MIRRORS"] = orig_mirrors
settings["FETCHCOMMAND"] = orig_fetchcommand
settings["RESUMECOMMAND"] = orig_resumecommand
# Test readonly DISTDIR
orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
try:
os.chmod(settings["DISTDIR"], 0o555)
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
finally:
os.chmod(settings["DISTDIR"], orig_distdir_mode)
# Test parallel-fetch mode
settings["PORTAGE_PARALLEL_FETCHONLY"] = "1"
try:
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
for k in settings["AA"].split():
os.unlink(os.path.join(settings["DISTDIR"], k))
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings.pop("PORTAGE_PARALLEL_FETCHONLY")
# Test RESUMECOMMAND
orig_resume_min_size = settings[
"PORTAGE_FETCH_RESUME_MIN_SIZE"]
try:
settings["PORTAGE_FETCH_RESUME_MIN_SIZE"] = "2"
for k in settings["AA"].split():
file_path = os.path.join(settings["DISTDIR"], k)
os.unlink(file_path)
with open(file_path + _download_suffix, "wb") as f:
f.write(distfiles[k][:2])
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
for k in settings["AA"].split():
with open(os.path.join(settings["DISTDIR"], k),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings[
"PORTAGE_FETCH_RESUME_MIN_SIZE"] = orig_resume_min_size
# Test readonly DISTDIR + skiprocheck, with FETCHCOMMAND set to temporarily chmod DISTDIR
orig_fetchcommand = settings["FETCHCOMMAND"]
orig_distdir_mode = os.stat(settings["DISTDIR"]).st_mode
for k in settings["AA"].split():
os.unlink(os.path.join(settings["DISTDIR"], k))
try:
os.chmod(settings["DISTDIR"], 0o555)
settings["FETCHCOMMAND"] = (
'"%s" -c "chmod ug+w \\"${DISTDIR}\\"; %s; status=\\$?; chmod a-w \\"${DISTDIR}\\"; exit \\$status"'
% (BASH_BINARY, orig_fetchcommand.replace('"', '\\"')))
settings.features.add("skiprocheck")
settings.features.remove("distlocks")
self.assertEqual(
loop.run_until_complete(async_fetch(pkg, ebuild_path)),
0)
finally:
settings["FETCHCOMMAND"] = orig_fetchcommand
os.chmod(settings["DISTDIR"], orig_distdir_mode)
settings.features.remove("skiprocheck")
settings.features.add("distlocks")
def testEbuildFetch(self):
distfiles = {
'bar': b'bar\n',
'foo': b'foo\n',
}
ebuilds = {
'dev-libs/A-1': {
'EAPI': '7',
'RESTRICT': 'primaryuri',
'SRC_URI': '''{scheme}://{host}:{port}/distfiles/bar.txt -> bar
{scheme}://{host}:{port}/distfiles/foo.txt -> foo''',
},
}
loop = SchedulerInterface(global_event_loop())
scheme = 'http'
host = '127.0.0.1'
content = {}
for k, v in distfiles.items():
content['/distfiles/{}.txt'.format(k)] = v
with AsyncHTTPServer(host, content, loop) as server:
ebuilds_subst = {}
for cpv, metadata in ebuilds.items():
metadata = metadata.copy()
metadata['SRC_URI'] = metadata['SRC_URI'].format(
scheme=scheme, host=host, port=server.server_port)
ebuilds_subst[cpv] = metadata
playground = ResolverPlayground(ebuilds=ebuilds_subst, distfiles=distfiles)
ro_distdir = tempfile.mkdtemp()
try:
fetchcommand = portage.util.shlex_split(playground.settings['FETCHCOMMAND'])
fetch_bin = portage.process.find_binary(fetchcommand[0])
if fetch_bin is None:
self.skipTest('FETCHCOMMAND not found: {}'.format(playground.settings['FETCHCOMMAND']))
resumecommand = portage.util.shlex_split(playground.settings['RESUMECOMMAND'])
resume_bin = portage.process.find_binary(resumecommand[0])
if resume_bin is None:
self.skipTest('RESUMECOMMAND not found: {}'.format(playground.settings['RESUMECOMMAND']))
root_config = playground.trees[playground.eroot]['root_config']
portdb = root_config.trees["porttree"].dbapi
settings = config(clone=playground.settings)
# Tests only work with one ebuild at a time, so the config
# pool only needs a single config instance.
class config_pool:
@staticmethod
def allocate():
return settings
@staticmethod
def deallocate(settings):
pass
def async_fetch(pkg, ebuild_path):
fetcher = EbuildFetcher(config_pool=config_pool, ebuild_path=ebuild_path,
fetchonly=False, fetchall=True, pkg=pkg, scheduler=loop)
fetcher.start()
return fetcher.async_wait()
for cpv in ebuilds:
metadata = dict(zip(Package.metadata_keys,
portdb.aux_get(cpv, Package.metadata_keys)))
pkg = Package(built=False, cpv=cpv, installed=False,
metadata=metadata, root_config=root_config,
type_name='ebuild')
settings.setcpv(pkg)
ebuild_path = portdb.findname(pkg.cpv)
portage.doebuild_environment(ebuild_path, 'fetch', settings=settings, db=portdb)
# Test good files in DISTDIR
for k in settings['AA'].split():
os.stat(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test digestgen with fetch
os.unlink(os.path.join(os.path.dirname(ebuild_path), 'Manifest'))
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
with ForkExecutor(loop=loop) as executor:
self.assertTrue(bool(loop.run_until_complete(
loop.run_in_executor(executor, functools.partial(
digestgen, mysettings=settings, myportdb=portdb)))))
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test missing files in DISTDIR
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test empty files in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
pass
self.assertEqual(os.stat(file_path).st_size, 0)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test non-empty files containing null bytes in DISTDIR
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
with open(file_path, 'wb') as f:
f.write(len(distfiles[k]) * b'\0')
self.assertEqual(os.stat(file_path).st_size, len(distfiles[k]))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
# Test PORTAGE_RO_DISTDIRS
settings['PORTAGE_RO_DISTDIRS'] = '"{}"'.format(ro_distdir)
orig_fetchcommand = settings['FETCHCOMMAND']
orig_resumecommand = settings['RESUMECOMMAND']
try:
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.rename(file_path, os.path.join(ro_distdir, k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
self.assertTrue(os.path.islink(file_path))
with open(file_path, 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
os.unlink(file_path)
finally:
settings.pop('PORTAGE_RO_DISTDIRS')
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test local filesystem in GENTOO_MIRRORS
orig_mirrors = settings['GENTOO_MIRRORS']
orig_fetchcommand = settings['FETCHCOMMAND']
try:
settings['GENTOO_MIRRORS'] = ro_distdir
settings['FETCHCOMMAND'] = settings['RESUMECOMMAND'] = ''
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['GENTOO_MIRRORS'] = orig_mirrors
settings['FETCHCOMMAND'] = orig_fetchcommand
settings['RESUMECOMMAND'] = orig_resumecommand
# Test readonly DISTDIR
orig_distdir_mode = os.stat(settings['DISTDIR']).st_mode
try:
os.chmod(settings['DISTDIR'], 0o555)
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
os.chmod(settings['DISTDIR'], orig_distdir_mode)
# Test parallel-fetch mode
settings['PORTAGE_PARALLEL_FETCHONLY'] = '1'
try:
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
for k in settings['AA'].split():
os.unlink(os.path.join(settings['DISTDIR'], k))
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings.pop('PORTAGE_PARALLEL_FETCHONLY')
# Test RESUMECOMMAND
orig_resume_min_size = settings['PORTAGE_FETCH_RESUME_MIN_SIZE']
try:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = '2'
for k in settings['AA'].split():
file_path = os.path.join(settings['DISTDIR'], k)
os.unlink(file_path)
with open(file_path + _download_suffix, 'wb') as f:
f.write(distfiles[k][:2])
self.assertEqual(loop.run_until_complete(async_fetch(pkg, ebuild_path)), 0)
for k in settings['AA'].split():
with open(os.path.join(settings['DISTDIR'], k), 'rb') as f:
self.assertEqual(f.read(), distfiles[k])
finally:
settings['PORTAGE_FETCH_RESUME_MIN_SIZE'] = orig_resume_min_size
finally:
shutil.rmtree(ro_distdir)
playground.cleanup()
