class AbstractEbuildProcess(SpawnProcess):
__slots__ = (
"phase",
"settings",
) + (
"_build_dir",
"_build_dir_unlock",
"_ipc_daemon",
"_exit_command",
"_exit_timeout_id",
"_start_future",
)
_phases_without_builddir = (
"clean",
"cleanrm",
"depend",
"help",
)
_phases_interactive_whitelist = ("config", )
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10 # seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = "other"
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and not os.path.isdir(
self.settings["PORTAGE_BUILDDIR"]):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % (
self.phase, self.settings["PORTAGE_BUILDDIR"])
self._eerror(textwrap.wrap(msg, 72))
self.returncode = 1
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == "Linux"
and "cgroup" in self.settings.features
and self.phase not in _global_pid_phases):
cgroup_root = "/sys/fs/cgroup"
cgroup_portage = os.path.join(cgroup_root, "portage")
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
"mount",
"-t",
"tmpfs",
"-o",
"rw,nosuid,nodev,noexec,mode=0755",
"tmpfs",
cgroup_root,
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
"mount",
"-t",
"cgroup",
"-o",
"rw,nosuid,nodev,noexec,none,name=portage",
"tmpfs",
cgroup_portage,
])
with open(os.path.join(cgroup_portage, "release_agent"),
"w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
with open(
os.path.join(cgroup_portage, "notify_on_release"),
"w") as f:
f.write("1")
else:
# Update release_agent if it no longer exists, because
# it refers to a temporary path when portage is updating
# itself.
release_agent = os.path.join(cgroup_portage,
"release_agent")
try:
with open(release_agent) as f:
release_agent_path = f.readline().rstrip("\n")
except EnvironmentError:
release_agent_path = None
if release_agent_path is None or not os.path.exists(
release_agent_path):
with open(release_agent, "w") as f:
f.write(
os.path.join(
self.settings["PORTAGE_BIN_PATH"],
"cgroup-release-agent",
))
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix="%s:%s." %
(self.settings["CATEGORY"], self.settings["PF"]),
)
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings["NOCOLOR"] = "true"
start_ipc_daemon = False
if self._enable_ipc_daemon:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
if self.phase not in self._phases_without_builddir:
start_ipc_daemon = True
if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._start_future = self._build_dir.async_lock()
self._start_future.add_done_callback(
functools.partial(
self._start_post_builddir_lock,
start_ipc_daemon=start_ipc_daemon,
))
return
else:
self.settings.pop("PORTAGE_IPC_DAEMON", None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop("PORTAGE_IPC_DAEMON", None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".exit_status")
self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)
def _start_post_builddir_lock(self,
lock_future=None,
start_ipc_daemon=False):
if lock_future is not None:
if lock_future is not self._start_future:
raise AssertionError("lock_future is not self._start_future")
self._start_future = None
if lock_future.cancelled():
self._build_dir = None
self.cancelled = True
self._was_cancelled()
self._async_wait()
return
lock_future.result()
if start_ipc_daemon:
self.settings["PORTAGE_IPC_DAEMON"] = "1"
self._start_ipc_daemon()
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if (0 not in self.fd_pipes
and self.phase not in self._phases_interactive_whitelist
and "interactive" not in self.settings.get("PROPERTIES",
"").split()):
null_fd = os.open("/dev/null", os.O_RDONLY)
self.fd_pipes[0] = null_fd
self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
".ipc_out")
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(
p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st,
)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
"available_eclasses": query_command,
"best_version": query_command,
"eclass_path": query_command,
"exit": self._exit_command,
"has_version": query_command,
"license_path": query_command,
"master_repositories": query_command,
"repository_path": query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(
commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler,
)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = self.scheduler.call_later(
self._exit_timeout, self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = self.scheduler.call_later(
self._cancel_timeout, self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self._async_waitpid()
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = _create_pty_or_pipe(
copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ("sesandbox" in self.settings.features
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (
self.phase,
signum,
)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = (_(
"The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase)
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog("eerror", lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = self.settings.get(
"NOCOLOR", "false").lower() in ("no", "false")
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _async_waitpid_cb(self, *args, **kwargs):
"""
Override _async_waitpid_cb to perform cleanup that is
not necessarily idempotent.
"""
SpawnProcess._async_waitpid_cb(self, *args, **kwargs)
if self._exit_timeout_id is not None:
self._exit_timeout_id.cancel()
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
elif not self.cancelled:
exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE")
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
def _async_wait(self):
"""
Override _async_wait to asynchronously unlock self._build_dir
when necessary.
"""
if self._build_dir is None:
SpawnProcess._async_wait(self)
elif self._build_dir_unlock is None:
if self.returncode is None:
raise asyncio.InvalidStateError("Result is not ready for %s" %
(self, ))
self._async_unlock_builddir(returncode=self.returncode)
def _async_unlock_builddir(self, returncode=None):
"""
Release the lock asynchronously, and if a returncode parameter
is given then set self.returncode and notify exit listeners.
"""
if self._build_dir_unlock is not None:
raise AssertionError("unlock already in progress")
if returncode is not None:
# The returncode will be set after unlock is complete.
self.returncode = None
self._build_dir_unlock = self._build_dir.async_unlock()
# Unlock only once.
self._build_dir = None
self._build_dir_unlock.add_done_callback(
functools.partial(self._unlock_builddir_exit,
returncode=returncode))
def _unlock_builddir_exit(self, unlock_future, returncode=None):
# Normally, async_unlock should not raise an exception here.
unlock_future.cancelled() or unlock_future.result()
if returncode is not None:
if unlock_future.cancelled():
self.cancelled = True
self._was_cancelled()
else:
self.returncode = returncode
SpawnProcess._async_wait(self)
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')
_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
_phases_interactive_whitelist = ('config',)
_phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config')
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call(['mount', '-t', 'tmpfs',
'-o', 'rw,nosuid,nodev,noexec,mode=0755',
'tmpfs', cgroup_root])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call(['mount', '-t', 'cgroup',
'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
'tmpfs', cgroup_portage])
cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
prefix='%s:%s.' % (self.settings["CATEGORY"],
self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(
scheduler=self.scheduler, settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770, stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
'available_eclasses' : query_command,
'best_version' : query_command,
'eclass_path' : query_command,
'exit' : self._exit_command,
'has_version' : query_command,
'license_path' : query_command,
'master_repositories' : query_command,
'repository_path' : query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = \
self.scheduler.timeout_add(self._exit_timeout,
self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = \
self.scheduler.timeout_add(self._cancel_timeout,
self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
return False # only run once
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self.wait()
return False # only run once
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (self.phase, signum)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = _("The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog('eerror', lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _log_poll_exception(self, event):
self._elog("eerror",
["%s received strange poll event: %s\n" % \
(self.__class__.__name__, event,)])
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.cgroup is not None:
try:
shutil.rmtree(self.cgroup)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if self._exit_timeout_id is not None:
self.scheduler.source_remove(self._exit_timeout_id)
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
if self._build_dir is not None:
self._build_dir.unlock()
self._build_dir = None
elif not self.cancelled:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_ipc_daemon', '_exit_command',)
_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(
self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
SpawnProcess._start(self)
def _init_ipc_fifos(self):
input_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(
self.settings['PORTAGE_BUILDDIR'], '.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770, stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings)
commands = {
'best_version' : query_command,
'exit' : self._exit_command,
'has_version' : query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout)
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _unexpected_exit(self):
phase = self.phase
msg = _("The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
out = StringIO()
phase = self.phase
for line in lines:
eerror(line, phase=phase, key=self.settings.mycpv, out=out)
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'], errors='replace')
if msg:
self.scheduler.output(msg,
log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
self.returncode = 1
self._unexpected_exit()
else:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
self.returncode = 1
self._unexpected_exit()
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')
_phases_without_builddir = (
'clean',
'cleanrm',
'depend',
'help',
)
_phases_interactive_whitelist = ('config', )
_phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm',
'config')
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
need_builddir = self.phase not in self._phases_without_builddir
# This can happen if the pre-clean phase triggers
# die_hooks for some reason, and PORTAGE_BUILDDIR
# doesn't exist yet.
if need_builddir and \
not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
msg = _("The ebuild phase '%s' has been aborted "
"since PORTAGE_BUILDDIR does not exist: '%s'") % \
(self.phase, self.settings['PORTAGE_BUILDDIR'])
self._eerror(textwrap.wrap(msg, 72))
self._set_returncode((self.pid, 1 << 8))
self._async_wait()
return
# Check if the cgroup hierarchy is in place. If it's not, mount it.
if (os.geteuid() == 0 and platform.system() == 'Linux'
and 'cgroup' in self.settings.features
and self.phase not in self._phases_without_cgroup):
cgroup_root = '/sys/fs/cgroup'
cgroup_portage = os.path.join(cgroup_root, 'portage')
try:
# cgroup tmpfs
if not os.path.ismount(cgroup_root):
# we expect /sys/fs to be there already
if not os.path.isdir(cgroup_root):
os.mkdir(cgroup_root, 0o755)
subprocess.check_call([
'mount', '-t', 'tmpfs', '-o',
'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
cgroup_root
])
# portage subsystem
if not os.path.ismount(cgroup_portage):
if not os.path.isdir(cgroup_portage):
os.mkdir(cgroup_portage, 0o755)
subprocess.check_call([
'mount', '-t', 'cgroup', '-o',
'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
cgroup_portage
])
cgroup_path = tempfile.mkdtemp(
dir=cgroup_portage,
prefix='%s:%s.' %
(self.settings["CATEGORY"], self.settings["PF"]))
except (subprocess.CalledProcessError, OSError):
pass
else:
self.cgroup = cgroup_path
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
settings=self.settings)
self._build_dir.lock()
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.fd_pipes is None:
self.fd_pipes = {}
null_fd = None
if 0 not in self.fd_pipes and \
self.phase not in self._phases_interactive_whitelist and \
"interactive" not in self.settings.get("PROPERTIES", "").split():
null_fd = os.open('/dev/null', os.O_RDONLY)
self.fd_pipes[0] = null_fd
try:
SpawnProcess._start(self)
finally:
if null_fd is not None:
os.close(null_fd)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings, self.phase)
commands = {
'available_eclasses': query_command,
'best_version': query_command,
'eclass_path': query_command,
'exit': self._exit_command,
'has_version': query_command,
'license_path': query_command,
'master_repositories': query_command,
'repository_path': query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self._exit_timeout_id = \
self.scheduler.timeout_add(self._exit_timeout,
self._exit_command_timeout_cb)
def _exit_command_timeout_cb(self):
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
self._exit_timeout_id = \
self.scheduler.timeout_add(self._cancel_timeout,
self._cancel_timeout_cb)
else:
self._exit_timeout_id = None
return False # only run once
def _cancel_timeout_cb(self):
self._exit_timeout_id = None
self.wait()
return False # only run once
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = None
if not self.background:
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _killed_by_signal(self, signum):
msg = _("The ebuild phase '%s' has been "
"killed by signal %s.") % (self.phase, signum)
self._eerror(textwrap.wrap(msg, 72))
def _unexpected_exit(self):
phase = self.phase
msg = _(
"The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
self._elog('eerror', lines)
def _elog(self, elog_funcname, lines):
out = io.StringIO()
phase = self.phase
elog_func = getattr(elog_messages, elog_funcname)
global_havecolor = portage.output.havecolor
try:
portage.output.havecolor = \
self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
for line in lines:
elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
finally:
portage.output.havecolor = global_havecolor
msg = out.getvalue()
if msg:
log_path = None
if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
log_path = self.settings.get("PORTAGE_LOG_FILE")
self.scheduler.output(msg, log_path=log_path)
def _log_poll_exception(self, event):
self._elog("eerror",
["%s received strange poll event: %s\n" % \
(self.__class__.__name__, event,)])
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self.cgroup is not None:
try:
shutil.rmtree(self.cgroup)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
if self._exit_timeout_id is not None:
self.scheduler.source_remove(self._exit_timeout_id)
self._exit_timeout_id = None
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
if self._build_dir is not None:
self._build_dir.unlock()
self._build_dir = None
elif not self.cancelled:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
if self.returncode < 0:
if not self.cancelled:
self._killed_by_signal(-self.returncode)
else:
self.returncode = 1
if not self.cancelled:
self._unexpected_exit()
class AbstractEbuildProcess(SpawnProcess):
__slots__ = ('phase', 'settings',) + \
('_ipc_daemon', '_exit_command',)
_phases_without_builddir = (
'clean',
'cleanrm',
'depend',
'help',
)
# Number of milliseconds to allow natural exit of the ebuild
# process after it has called the exit command via IPC. It
# doesn't hurt to be generous here since the scheduler
# continues to process events during this period, and it can
# return long before the timeout expires.
_exit_timeout = 10000 # 10 seconds
# The EbuildIpcDaemon support is well tested, but this variable
# is left so we can temporarily disable it if any issues arise.
_enable_ipc_daemon = True
def __init__(self, **kwargs):
SpawnProcess.__init__(self, **kwargs)
if self.phase is None:
phase = self.settings.get("EBUILD_PHASE")
if not phase:
phase = 'other'
self.phase = phase
def _start(self):
if self.background:
# Automatically prevent color codes from showing up in logs,
# since we're not displaying to a terminal anyway.
self.settings['NOCOLOR'] = 'true'
if self._enable_ipc_daemon:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
if self.phase not in self._phases_without_builddir:
self.settings['PORTAGE_IPC_DAEMON'] = "1"
self._start_ipc_daemon()
else:
self.settings.pop('PORTAGE_IPC_DAEMON', None)
else:
# Since the IPC daemon is disabled, use a simple tempfile based
# approach to detect unexpected exit like in bug #190128.
self.settings.pop('PORTAGE_IPC_DAEMON', None)
if self.phase not in self._phases_without_builddir:
exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.exit_status')
self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
try:
os.unlink(exit_file)
except OSError:
if os.path.exists(exit_file):
# make sure it doesn't exist
raise
else:
self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
SpawnProcess._start(self)
def _init_ipc_fifos(self):
input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'],
'.ipc_out')
for p in (input_fifo, output_fifo):
st = None
try:
st = os.lstat(p)
except OSError:
os.mkfifo(p)
else:
if not stat.S_ISFIFO(st.st_mode):
st = None
try:
os.unlink(p)
except OSError:
pass
os.mkfifo(p)
apply_secpass_permissions(p,
uid=os.getuid(),
gid=portage.data.portage_gid,
mode=0o770,
stat_cached=st)
return (input_fifo, output_fifo)
def _start_ipc_daemon(self):
self._exit_command = ExitCommand()
self._exit_command.reply_hook = self._exit_command_callback
query_command = QueryCommand(self.settings)
commands = {
'best_version': query_command,
'exit': self._exit_command,
'has_version': query_command,
}
input_fifo, output_fifo = self._init_ipc_fifos()
self._ipc_daemon = EbuildIpcDaemon(commands=commands,
input_fifo=input_fifo,
output_fifo=output_fifo,
scheduler=self.scheduler)
self._ipc_daemon.start()
def _exit_command_callback(self):
if self._registered:
# Let the process exit naturally, if possible.
self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout)
if self._registered:
# If it doesn't exit naturally in a reasonable amount
# of time, kill it (solves bug #278895). We try to avoid
# this when possible since it makes sandbox complain about
# being killed by a signal.
self.cancel()
def _orphan_process_warn(self):
phase = self.phase
msg = _("The ebuild phase '%s' with pid %s appears "
"to have left an orphan process running in the "
"background.") % (phase, self.pid)
self._eerror(textwrap.wrap(msg, 72))
def _pipe(self, fd_pipes):
stdout_pipe = fd_pipes.get(1)
got_pty, master_fd, slave_fd = \
_create_pty_or_pipe(copy_term_size=stdout_pipe)
return (master_fd, slave_fd)
def _can_log(self, slave_fd):
# With sesandbox, logging works through a pty but not through a
# normal pipe. So, disable logging if ptys are broken.
# See Bug #162404.
# TODO: Add support for logging via named pipe (fifo) with
# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
# to be compatible with sesandbox.
return not ('sesandbox' in self.settings.features \
and self.settings.selinux_enabled()) or os.isatty(slave_fd)
def _unexpected_exit(self):
phase = self.phase
msg = _(
"The ebuild phase '%s' has exited "
"unexpectedly. This type of behavior "
"is known to be triggered "
"by things such as failed variable "
"assignments (bug #190128) or bad substitution "
"errors (bug #200313). Normally, before exiting, bash should "
"have displayed an error message above. If bash did not "
"produce an error message above, it's possible "
"that the ebuild has called `exit` when it "
"should have called `die` instead. This behavior may also "
"be triggered by a corrupt bash binary or a hardware "
"problem such as memory or cpu malfunction. If the problem is not "
"reproducible or it appears to occur randomly, then it is likely "
"to be triggered by a hardware problem. "
"If you suspect a hardware problem then you should "
"try some basic hardware diagnostics such as memtest. "
"Please do not report this as a bug unless it is consistently "
"reproducible and you are sure that your bash binary and hardware "
"are functioning properly.") % phase
self._eerror(textwrap.wrap(msg, 72))
def _eerror(self, lines):
out = StringIO()
phase = self.phase
for line in lines:
eerror(line, phase=phase, key=self.settings.mycpv, out=out)
msg = _unicode_decode(out.getvalue(),
encoding=_encodings['content'],
errors='replace')
if msg:
self.scheduler.output(
msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))
def _set_returncode(self, wait_retval):
SpawnProcess._set_returncode(self, wait_retval)
if self._ipc_daemon is not None:
self._ipc_daemon.cancel()
if self._exit_command.exitcode is not None:
self.returncode = self._exit_command.exitcode
else:
self.returncode = 1
self._unexpected_exit()
else:
exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
if exit_file and not os.path.exists(exit_file):
self.returncode = 1
self._unexpected_exit()