def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_task(
AsyncTaskFuture(future=self._builddir_lock.async_lock()),
self._start_unmerge,
)
def _prefetch_exit(self, prefetcher):
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning needs to happen before fetch, since the build dir
# is used for log handling.
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(self.pkg_count.curval, self.pkg_count.maxval,
self.pkg.cpv, self._ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv)
self.logger.log(msg, short_msg=short_msg)
pre_clean_phase = EbuildPhase(background=self.background,
phase='clean',
scheduler=self.scheduler,
settings=self.settings)
self._start_task(pre_clean_phase, self._pre_clean_exit)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
dir_path = os.path.join(settings["PORTAGE_TMPDIR"],
"portage", pkg.category, pkg.pf)
self._build_dir = EbuildBuildDir(dir_path=dir_path,
pkg=pkg, settings=settings)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
debug = settings.get("PORTAGE_DEBUG") == "1"
portage.doebuild_environment(self._ebuild_path, "setup",
settings["ROOT"], settings, debug, 1, self._bintree.dbapi)
settings.configdict["pkg"]["EMERGE_FROM"] = pkg.type_name
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif not prefetcher.isAlive():
prefetcher.cancel()
elif prefetcher.poll() is None:
waiting_msg = ("Fetching '%s' " + \
"in the background. " + \
"To view fetch progress, run `tail -f " + \
"/var/log/emerge-fetch.log` in another " + \
"terminal.") % prefetcher.pkg_path
msg_prefix = colorize("GOOD", " * ")
from textwrap import wrap
waiting_msg = "".join("%s%s\n" % (msg_prefix, line) \
for line in wrap(waiting_msg, 65))
if not self.background:
writemsg(waiting_msg, noiselevel=-1)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
if retval == os.EX_OK:
self._current_task = None
self.returncode = os.EX_OK
self._async_wait()
else:
# For pretend mode, the convention it to execute
# pkg_nofetch and return a successful exitcode.
self._start_task(
SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=self.pkg.root_config.trees[
self._tree].dbapi,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings), self._default_final_exit)
return
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
self.returncode = retval
self.wait()
return
else:
fetcher = EbuildFetcher(config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=False,
logfile=None,
pkg=self.pkg,
scheduler=self.scheduler)
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _start(self):
pkg = self.pkg
settings = self.settings
settings.setcpv(pkg)
self._tree = "bintree"
self._bintree = self.pkg.root_config.trees[self._tree]
self._verify = not self.opts.pretend
# Use realpath like doebuild_environment() does, since we assert
# that this path is literally identical to PORTAGE_BUILDDIR.
dir_path = os.path.join(os.path.realpath(settings["PORTAGE_TMPDIR"]),
"portage", pkg.category, pkg.pf)
self._image_dir = os.path.join(dir_path, "image")
self._infloc = os.path.join(dir_path, "build-info")
self._ebuild_path = os.path.join(self._infloc, pkg.pf + ".ebuild")
settings["EBUILD"] = self._ebuild_path
portage.doebuild_environment(self._ebuild_path,
'setup',
settings=self.settings,
db=self._bintree.dbapi)
if dir_path != self.settings['PORTAGE_BUILDDIR']:
raise AssertionError("'%s' != '%s'" % \
(dir_path, self.settings['PORTAGE_BUILDDIR']))
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
settings.configdict["pkg"]["EMERGE_FROM"] = "binary"
settings.configdict["pkg"]["MERGE_TYPE"] = "binary"
if eapi_exports_replace_vars(settings["EAPI"]):
vardb = self.pkg.root_config.trees["vartree"].dbapi
settings["REPLACING_VERSIONS"] = " ".join(
set(portage.versions.cpv_getversion(x) \
for x in vardb.match(self.pkg.slot_atom) + \
vardb.match('='+self.pkg.cpv)))
# The prefetcher has already completed or it
# could be running now. If it's running now,
# wait for it to complete since it holds
# a lock on the file being fetched. The
# portage.locks functions are only designed
# to work between separate processes. Since
# the lock is held by the current process,
# use the scheduler and fetcher methods to
# synchronize with the fetcher.
prefetcher = self.prefetcher
if prefetcher is None:
pass
elif prefetcher.isAlive() and \
prefetcher.poll() is None:
if not self.background:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
'emerge-fetch.log')
msg = (
'Fetching in the background:',
prefetcher.pkg_path,
'To view fetch progress, run in another terminal:',
'tail -f %s' % fetch_log,
)
out = portage.output.EOutput()
for l in msg:
out.einfo(l)
self._current_task = prefetcher
prefetcher.addExitListener(self._prefetch_exit)
return
self._prefetch_exit(prefetcher)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not os.path.isdir(
self.settings["PORTAGE_BUILDDIR"]):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % (
self.phase, self.settings["PORTAGE_BUILDDIR"])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == "Linux"
and "cgroup" in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = "/sys/fs/cgroup"
cgroup_portage = os.path.join(cgroup_root, "portage")
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
"mount",
"-t",
"tmpfs",
"-o",
"rw,nosuid,nodev,noexec,mode=0755",
"tmpfs",
cgroup_root,
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
"mount",
"-t",
"cgroup",
"-o",
"rw,nosuid,nodev,noexec,none,name=portage",
"tmpfs",
cgroup_portage,
])
with open(os.path.join(cgroup_portage, "release_agent"),
"w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
with open(
os.path.join(cgroup_portage, "notify_on_release"),
"w") as f:
f.write("1")
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(cgroup_portage,
"release_agent")
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip("\n")
except EnvironmentError:
release_agent_path = None
if release_agent_path is None or not os.path.exists(
release_agent_path):
with open(release_agent, "w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix="%s:%s." %
(self.settings["CATEGORY"], self.settings["PF"]),
)
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings["NOCOLOR"] = "true"
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(
self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon,
))
return
else:
self.settings.pop("PORTAGE_IPC_DAEMON", None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop("PORTAGE_IPC_DAEMON", None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".exit_status")
self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
'mount', '-t', 'tmpfs', '-o',
'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
cgroup_root
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
'mount', '-t', 'cgroup', '-o',
'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
cgroup_portage
])
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix='%s:%s.' %
(self.settings["CATEGORY"], self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def testIpcDaemon(self):
event_loop = global_event_loop()
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
env['PYTHONDONTWRITEBYTECODE'] = os.environ.get(
'PYTHONDONTWRITEBYTECODE', '')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
build_dir = EbuildBuildDir(scheduler=event_loop, settings=env)
event_loop.run_until_complete(build_dir.async_lock())
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for EventLoop/AsyncScheduler.
# Use a ridiculously long sleep_time_s in case the user's
# system is heavily loaded (see bug #436334).
sleep_time_s = 600 # seconds
short_timeout_s = 0.010 # seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo)
proc = SleepProcess(seconds=sleep_time_s)
task_scheduler = TaskScheduler(iter([daemon, proc]),
max_jobs=2,
event_loop=event_loop)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.cancel()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
self._run(event_loop, task_scheduler, short_timeout_s)
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
event_loop.run_until_complete(build_dir.async_unlock())
shutil.rmtree(tmpdir)
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self.wait()
return
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _prefetch_exit(self, prefetcher):
if self._was_cancelled():
self.wait()
return
opts = self.opts
pkg = self.pkg
settings = self.settings
if opts.fetchonly:
if opts.pretend:
fetcher = EbuildFetchonly(fetch_all=opts.fetch_all_uri,
pkg=pkg,
pretend=opts.pretend,
settings=settings)
retval = fetcher.execute()
if retval == os.EX_OK:
self._current_task = None
self.returncode = os.EX_OK
self._async_wait()
else:
# For pretend mode, the convention it to execute
# pkg_nofetch and return a successful exitcode.
self._start_task(
SpawnNofetchWithoutBuilddir(
background=self.background,
portdb=self.pkg.root_config.trees[
self._tree].dbapi,
ebuild_path=self._ebuild_path,
scheduler=self.scheduler,
settings=self.settings), self._default_final_exit)
return
quiet_setting = settings.get("PORTAGE_QUIET", False)
fetch_log = None
logwrite_access = False
if quiet_setting:
fetch_log = os.path.join(_emerge.emergelog._emerge_log_dir,
"emerge-fetch.log")
logwrite_access = os.access(first_existing(fetch_log), os.W_OK)
fetcher = EbuildFetcher(
config_pool=self.config_pool,
ebuild_path=self._ebuild_path,
fetchall=self.opts.fetch_all_uri,
fetchonly=self.opts.fetchonly,
background=quiet_setting if logwrite_access else False,
logfile=fetch_log if logwrite_access else None,
pkg=self.pkg,
scheduler=self.scheduler,
)
if fetch_log and logwrite_access:
fetcher.addExitListener(self._fetchonly_exit)
self._task_queued(fetcher)
self.scheduler.fetch.schedule(fetcher, force_queue=True)
else:
self._start_task(fetcher, self._fetchonly_exit)
return
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=settings)
self._start_task(AsyncTaskFuture(future=self._build_dir.async_lock()),
self._start_pre_clean)
def _start(self):
vardb = self.pkg.root_config.trees["vartree"].dbapi
dbdir = vardb.getpath(self.pkg.cpv)
if not os.path.exists(dbdir):
# Apparently the package got uninstalled
# already, so we can safely return early.
self.returncode = os.EX_OK
self._async_wait()
return
self.settings.setcpv(self.pkg)
cat, pf = portage.catsplit(self.pkg.cpv)
myebuildpath = os.path.join(dbdir, pf + ".ebuild")
try:
portage.doebuild_environment(myebuildpath,
"prerm",
settings=self.settings,
db=vardb)
except UnsupportedAPIException:
# This is safe to ignore since this function is
# guaranteed to set PORTAGE_BUILDDIR even though
# it raises UnsupportedAPIException. The error
# will be logged when it prevents the pkg_prerm
# and pkg_postrm phases from executing.
pass
self._builddir_lock = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._builddir_lock.lock()
portage.prepare_build_dirs(settings=self.settings, cleanup=True)
# Output only gets logged if it comes after prepare_build_dirs()
# which initializes PORTAGE_LOG_FILE.
retval, pkgmap = _unmerge_display(self.pkg.root_config,
self.opts,
"unmerge", [self.pkg.cpv],
clean_delay=0,
writemsg_level=self._writemsg_level)
if retval != os.EX_OK:
self._builddir_lock.unlock()
self.returncode = retval
self._async_wait()
return
self._writemsg_level(">>> Unmerging %s...\n" % (self.pkg.cpv, ),
noiselevel=-1)
self._emergelog("=== Unmerging... (%s)" % (self.pkg.cpv, ))
unmerge_task = MergeProcess(
mycat=cat,
mypkg=pf,
settings=self.settings,
treetype="vartree",
vartree=self.pkg.root_config.trees["vartree"],
scheduler=self.scheduler,
background=self.background,
mydbapi=self.pkg.root_config.trees["vartree"].dbapi,
prev_mtimes=self.ldpath_mtimes,
logfile=self.settings.get("PORTAGE_LOG_FILE"),
unmerge=True)
self._start_task(unmerge_task, self._unmerge_exit)
def _fetch_exit(self, fetcher):
opts = self.opts
pkg = self.pkg
fetch_failed = False
if opts.fetchonly:
fetch_failed = self._final_exit(fetcher) != os.EX_OK
else:
fetch_failed = self._default_exit(fetcher) != os.EX_OK
if fetch_failed and fetcher.logfile is not None and \
os.path.exists(fetcher.logfile):
self.settings["PORTAGE_LOG_FILE"] = fetcher.logfile
if fetch_failed or opts.fetchonly:
self.wait()
return
logger = self.logger
opts = self.opts
pkg_count = self.pkg_count
scheduler = self.scheduler
settings = self.settings
features = settings.features
ebuild_path = self._ebuild_path
system_set = pkg.root_config.sets["system"]
self._build_dir = EbuildBuildDir(pkg=pkg, settings=settings)
self._build_dir.lock()
# Cleaning is triggered before the setup
# phase, in portage.doebuild().
msg = " === (%s of %s) Cleaning (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Clean" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
#buildsyspkg: Check if we need to _force_ binary package creation
self._issyspkg = "buildsyspkg" in features and \
system_set.findAtomForPackage(pkg) and \
not opts.buildpkg
if opts.buildpkg or self._issyspkg:
self._buildpkg = True
msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
else:
msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path)
short_msg = "emerge: (%s of %s) %s Compile" % \
(pkg_count.curval, pkg_count.maxval, pkg.cpv)
logger.log(msg, short_msg=short_msg)
build = EbuildExecuter(background=self.background,
pkg=pkg,
scheduler=scheduler,
settings=settings)
self._start_task(build, self._build_exit)
def _start(self):
root_config = self.pkg.root_config
portdb = root_config.trees["porttree"].dbapi
ebuild_path = portdb.findname(self.pkg.cpv)
if ebuild_path is None:
raise AssertionError("ebuild not found for '%s'" % self.pkg.cpv)
settings = self.config_pool.allocate()
settings.setcpv(self.pkg)
if self.prefetch and \
self._prefetch_size_ok(portdb, settings, ebuild_path):
self.config_pool.deallocate(settings)
self.returncode = os.EX_OK
self.wait()
return
# In prefetch mode, logging goes to emerge-fetch.log and the builddir
# should not be touched since otherwise it could interfere with
# another instance of the same cpv concurrently being built for a
# different $ROOT (currently, builds only cooperate with prefetchers
# that are spawned for the same $ROOT).
if not self.prefetch:
self._build_dir = EbuildBuildDir(pkg=self.pkg, settings=settings)
self._build_dir.lock()
self._build_dir.clean_log()
portage.prepare_build_dirs(self.pkg.root, self._build_dir.settings, 0)
if self.logfile is None:
self.logfile = settings.get("PORTAGE_LOG_FILE")
phase = "fetch"
if self.fetchall:
phase = "fetchall"
# If any incremental variables have been overridden
# via the environment, those values need to be passed
# along here so that they are correctly considered by
# the config instance in the subproccess.
fetch_env = os.environ.copy()
fetch_env['PORTAGE_CONFIGROOT'] = settings['PORTAGE_CONFIGROOT']
nocolor = settings.get("NOCOLOR")
if nocolor is not None:
fetch_env["NOCOLOR"] = nocolor
fetch_env["PORTAGE_NICENESS"] = "0"
if self.prefetch:
fetch_env["PORTAGE_PARALLEL_FETCHONLY"] = "1"
ebuild_binary = os.path.join(
settings["PORTAGE_BIN_PATH"], "ebuild")
fetch_args = [ebuild_binary, ebuild_path, phase]
debug = settings.get("PORTAGE_DEBUG") == "1"
if debug:
fetch_args.append("--debug")
if not self.background and nocolor not in ('yes', 'true'):
# Force consistent color output, in case we are capturing fetch
# output through a normal pipe due to unavailability of ptys.
fetch_args.append('--color=y')
self.args = fetch_args
self.env = fetch_env
if self._build_dir is None:
# Free settings now since we only have a local reference.
self.config_pool.deallocate(settings)
SpawnProcess._start(self)
def testIpcDaemon(self):
tmpdir = tempfile.mkdtemp()
build_dir = None
try:
env = {}
# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
# need to be inherited by ebuild subprocesses.
if 'PORTAGE_USERNAME' in os.environ:
env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
if 'PORTAGE_GRPNAME' in os.environ:
env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']
env['PORTAGE_PYTHON'] = _python_interpreter
env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]
task_scheduler = TaskScheduler(max_jobs=2)
build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface,
settings=env)
build_dir.lock()
ensure_dirs(env['PORTAGE_BUILDDIR'])
input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
os.mkfifo(input_fifo)
os.mkfifo(output_fifo)
for exitcode in (0, 1, 2):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(args=[
BASH_BINARY, "-c",
'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, True,
"command not received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(exit_command.exitcode, exitcode)
# Intentionally short timeout test for QueueScheduler.run()
sleep_time_s = 10 # 10.000 seconds
short_timeout_ms = 10 # 0.010 seconds
for i in range(3):
exit_command = ExitCommand()
commands = {'exit': exit_command}
daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=task_scheduler.sched_iface)
proc = SpawnProcess(
args=[BASH_BINARY, "-c",
'exec sleep %d' % sleep_time_s],
env=env,
scheduler=task_scheduler.sched_iface)
self.received_command = False
def exit_command_callback():
self.received_command = True
task_scheduler.clear()
task_scheduler.wait()
exit_command.reply_hook = exit_command_callback
start_time = time.time()
task_scheduler.add(daemon)
task_scheduler.add(proc)
task_scheduler.run(timeout=short_timeout_ms)
task_scheduler.clear()
task_scheduler.wait()
hardlock_cleanup(env['PORTAGE_BUILDDIR'],
remove_all_locks=True)
self.assertEqual(self.received_command, False,
"command received after %d seconds" % \
(time.time() - start_time,))
self.assertEqual(proc.isAlive(), False)
self.assertEqual(daemon.isAlive(), False)
self.assertEqual(proc.returncode == os.EX_OK, False)
finally:
if build_dir is not None:
build_dir.unlock()
shutil.rmtree(tmpdir)
