def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'best_version' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start()
def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses': query_command, 'best_version': query_command, 'eclass_path': query_command, 'exit': self._exit_command, 'has_version': query_command, 'license_path': query_command, 'master_repositories': query_command, 'repository_path': query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start()
def testIpcDaemon(self): tmpdir = tempfile.mkdtemp() try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = tmpdir input_fifo = os.path.join(tmpdir, '.ipc_in') output_fifo = os.path.join(tmpdir, '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): task_scheduler = TaskScheduler(max_jobs=2) exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env, scheduler=task_scheduler.sched_iface) def exit_command_callback(): proc.cancel() daemon.cancel() exit_command.reply_hook = exit_command_callback task_scheduler.add(daemon) task_scheduler.add(proc) task_scheduler.run() self.assertEqual(exit_command.exitcode, exitcode) finally: shutil.rmtree(tmpdir)
def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses' : query_command, 'best_version' : query_command, 'eclass_path' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, 'license_path' : query_command, 'master_repositories' : query_command, 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start()
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir( scheduler=event_loop, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess( args=[BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 #600.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_ms) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)
class AbstractEbuildProcess(SpawnProcess): __slots__ = ( "phase", "settings", ) + ( "_build_dir", "_build_dir_unlock", "_ipc_daemon", "_exit_command", "_exit_timeout_id", "_start_future", ) _phases_without_builddir = ( "clean", "cleanrm", "depend", "help", ) _phases_interactive_whitelist = ("config", ) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10 # seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = "other" self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and not os.path.isdir( self.settings["PORTAGE_BUILDDIR"]): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % ( self.phase, self.settings["PORTAGE_BUILDDIR"]) self._eerror(textwrap.wrap(msg, 72)) self.returncode = 1 self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == "Linux" and "cgroup" in self.settings.features and self.phase not in _global_pid_phases): cgroup_root = "/sys/fs/cgroup" cgroup_portage = os.path.join(cgroup_root, "portage") try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ "mount", "-t", "tmpfs", "-o", "rw,nosuid,nodev,noexec,mode=0755", "tmpfs", cgroup_root, ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ "mount", "-t", "cgroup", "-o", "rw,nosuid,nodev,noexec,none,name=portage", "tmpfs", cgroup_portage, ]) with open(os.path.join(cgroup_portage, "release_agent"), "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) with open( os.path.join(cgroup_portage, "notify_on_release"), "w") as f: f.write("1") else: # Update release_agent if it no longer exists, because # it refers to a temporary path when portage is updating # itself. release_agent = os.path.join(cgroup_portage, "release_agent") try: with open(release_agent) as f: release_agent_path = f.readline().rstrip("\n") except EnvironmentError: release_agent_path = None if release_agent_path is None or not os.path.exists( release_agent_path): with open(release_agent, "w") as f: f.write( os.path.join( self.settings["PORTAGE_BIN_PATH"], "cgroup-release-agent", )) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix="%s:%s." % (self.settings["CATEGORY"], self.settings["PF"]), ) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings["NOCOLOR"] = "true" start_ipc_daemon = False if self._enable_ipc_daemon: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) if self.phase not in self._phases_without_builddir: start_ipc_daemon = True if "PORTAGE_BUILDDIR_LOCKED" not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._start_future = self._build_dir.async_lock() self._start_future.add_done_callback( functools.partial( self._start_post_builddir_lock, start_ipc_daemon=start_ipc_daemon, )) return else: self.settings.pop("PORTAGE_IPC_DAEMON", None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop("PORTAGE_IPC_DAEMON", None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".exit_status") self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None) self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon) def _start_post_builddir_lock(self, lock_future=None, start_ipc_daemon=False): if lock_future is not None: if lock_future is not self._start_future: raise AssertionError("lock_future is not self._start_future") self._start_future = None if lock_future.cancelled(): self._build_dir = None self.cancelled = True self._was_cancelled() self._async_wait() return lock_future.result() if start_ipc_daemon: self.settings["PORTAGE_IPC_DAEMON"] = "1" self._start_ipc_daemon() if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if (0 not in self.fd_pipes and self.phase not in self._phases_interactive_whitelist and "interactive" not in self.settings.get("PROPERTIES", "").split()): null_fd = os.open("/dev/null", os.O_RDONLY) self.fd_pipes[0] = null_fd self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD") try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in") output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_out") for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions( p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st, ) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { "available_eclasses": query_command, "best_version": query_command, "eclass_path": query_command, "exit": self._exit_command, "has_version": query_command, "license_path": query_command, "master_repositories": query_command, "repository_path": query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon( commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler, ) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = self.scheduler.call_later( self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = self.scheduler.call_later( self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None def _cancel_timeout_cb(self): self._exit_timeout_id = None self._async_waitpid() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = _create_pty_or_pipe( copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ("sesandbox" in self.settings.features and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % ( self.phase, signum, ) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = (_( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase) self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog("eerror", lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = self.settings.get( "NOCOLOR", "false").lower() in ("no", "false") for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _async_waitpid_cb(self, *args, **kwargs): """ Override _async_waitpid_cb to perform cleanup that is not necessarily idempotent. """ SpawnProcess._async_waitpid_cb(self, *args, **kwargs) if self._exit_timeout_id is not None: self._exit_timeout_id.cancel() self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() elif not self.cancelled: exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE") if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() def _async_wait(self): """ Override _async_wait to asynchronously unlock self._build_dir when necessary. """ if self._build_dir is None: SpawnProcess._async_wait(self) elif self._build_dir_unlock is None: if self.returncode is None: raise asyncio.InvalidStateError("Result is not ready for %s" % (self, )) self._async_unlock_builddir(returncode=self.returncode) def _async_unlock_builddir(self, returncode=None): """ Release the lock asynchronously, and if a returncode parameter is given then set self.returncode and notify exit listeners. """ if self._build_dir_unlock is not None: raise AssertionError("unlock already in progress") if returncode is not None: # The returncode will be set after unlock is complete. self.returncode = None self._build_dir_unlock = self._build_dir.async_unlock() # Unlock only once. self._build_dir = None self._build_dir_unlock.add_done_callback( functools.partial(self._unlock_builddir_exit, returncode=returncode)) def _unlock_builddir_exit(self, unlock_future, returncode=None): # Normally, async_unlock should not raise an exception here. unlock_future.cancelled() or unlock_future.result() if returncode is not None: if unlock_future.cancelled(): self.cancelled = True self._was_cancelled() else: self.returncode = returncode SpawnProcess._async_wait(self)
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) _phases_interactive_whitelist = ('config',) _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call(['mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call(['mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage]) cgroup_path = tempfile.mkdtemp(dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir( scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses' : query_command, 'best_version' : query_command, 'eclass_path' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, 'license_path' : query_command, 'master_repositories' : query_command, 'repository_path' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = \ self.scheduler.timeout_add(self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = \ self.scheduler.timeout_add(self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None return False # only run once def _cancel_timeout_cb(self): self._exit_timeout_id = None self.wait() return False # only run once def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = _("The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog('eerror', lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = \ self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false') for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _log_poll_exception(self, event): self._elog("eerror", ["%s received strange poll event: %s\n" % \ (self.__class__.__name__, event,)]) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self.cgroup is not None: try: shutil.rmtree(self.cgroup) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if self._exit_timeout_id is not None: self.scheduler.source_remove(self._exit_timeout_id) self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() if self._build_dir is not None: self._build_dir.unlock() self._build_dir = None elif not self.cancelled: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit()
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id') _phases_without_builddir = ( 'clean', 'cleanrm', 'depend', 'help', ) _phases_interactive_whitelist = ('config', ) _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config') # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): need_builddir = self.phase not in self._phases_without_builddir # This can happen if the pre-clean phase triggers # die_hooks for some reason, and PORTAGE_BUILDDIR # doesn't exist yet. if need_builddir and \ not os.path.isdir(self.settings['PORTAGE_BUILDDIR']): msg = _("The ebuild phase '%s' has been aborted " "since PORTAGE_BUILDDIR does not exist: '%s'") % \ (self.phase, self.settings['PORTAGE_BUILDDIR']) self._eerror(textwrap.wrap(msg, 72)) self._set_returncode((self.pid, 1 << 8)) self._async_wait() return # Check if the cgroup hierarchy is in place. If it's not, mount it. if (os.geteuid() == 0 and platform.system() == 'Linux' and 'cgroup' in self.settings.features and self.phase not in self._phases_without_cgroup): cgroup_root = '/sys/fs/cgroup' cgroup_portage = os.path.join(cgroup_root, 'portage') try: # cgroup tmpfs if not os.path.ismount(cgroup_root): # we expect /sys/fs to be there already if not os.path.isdir(cgroup_root): os.mkdir(cgroup_root, 0o755) subprocess.check_call([ 'mount', '-t', 'tmpfs', '-o', 'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs', cgroup_root ]) # portage subsystem if not os.path.ismount(cgroup_portage): if not os.path.isdir(cgroup_portage): os.mkdir(cgroup_portage, 0o755) subprocess.check_call([ 'mount', '-t', 'cgroup', '-o', 'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs', cgroup_portage ]) cgroup_path = tempfile.mkdtemp( dir=cgroup_portage, prefix='%s:%s.' % (self.settings["CATEGORY"], self.settings["PF"])) except (subprocess.CalledProcessError, OSError): pass else: self.cgroup = cgroup_path if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings: self._build_dir = EbuildBuildDir(scheduler=self.scheduler, settings=self.settings) self._build_dir.lock() self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.fd_pipes is None: self.fd_pipes = {} null_fd = None if 0 not in self.fd_pipes and \ self.phase not in self._phases_interactive_whitelist and \ "interactive" not in self.settings.get("PROPERTIES", "").split(): null_fd = os.open('/dev/null', os.O_RDONLY) self.fd_pipes[0] = null_fd try: SpawnProcess._start(self) finally: if null_fd is not None: os.close(null_fd) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings, self.phase) commands = { 'available_eclasses': query_command, 'best_version': query_command, 'eclass_path': query_command, 'exit': self._exit_command, 'has_version': query_command, 'license_path': query_command, 'master_repositories': query_command, 'repository_path': query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self._exit_timeout_id = \ self.scheduler.timeout_add(self._exit_timeout, self._exit_command_timeout_cb) def _exit_command_timeout_cb(self): if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() self._exit_timeout_id = \ self.scheduler.timeout_add(self._cancel_timeout, self._cancel_timeout_cb) else: self._exit_timeout_id = None return False # only run once def _cancel_timeout_cb(self): self._exit_timeout_id = None self.wait() return False # only run once def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = None if not self.background: stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _killed_by_signal(self, signum): msg = _("The ebuild phase '%s' has been " "killed by signal %s.") % (self.phase, signum) self._eerror(textwrap.wrap(msg, 72)) def _unexpected_exit(self): phase = self.phase msg = _( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): self._elog('eerror', lines) def _elog(self, elog_funcname, lines): out = io.StringIO() phase = self.phase elog_func = getattr(elog_messages, elog_funcname) global_havecolor = portage.output.havecolor try: portage.output.havecolor = \ self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false') for line in lines: elog_func(line, phase=phase, key=self.settings.mycpv, out=out) finally: portage.output.havecolor = global_havecolor msg = out.getvalue() if msg: log_path = None if self.settings.get("PORTAGE_BACKGROUND") != "subprocess": log_path = self.settings.get("PORTAGE_LOG_FILE") self.scheduler.output(msg, log_path=log_path) def _log_poll_exception(self, event): self._elog("eerror", ["%s received strange poll event: %s\n" % \ (self.__class__.__name__, event,)]) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self.cgroup is not None: try: shutil.rmtree(self.cgroup) except EnvironmentError as e: if e.errno != errno.ENOENT: raise if self._exit_timeout_id is not None: self.scheduler.source_remove(self._exit_timeout_id) self._exit_timeout_id = None if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit() if self._build_dir is not None: self._build_dir.unlock() self._build_dir = None elif not self.cancelled: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): if self.returncode < 0: if not self.cancelled: self._killed_by_signal(-self.returncode) else: self.returncode = 1 if not self.cancelled: self._unexpected_exit()
def testIpcDaemon(self): event_loop = global_event_loop() tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') env['PYTHONDONTWRITEBYTECODE'] = os.environ.get( 'PYTHONDONTWRITEBYTECODE', '') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] build_dir = EbuildBuildDir(scheduler=event_loop, settings=env) event_loop.run_until_complete(build_dir.async_lock()) ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for EventLoop/AsyncScheduler. # Use a ridiculously long sleep_time_s in case the user's # system is heavily loaded (see bug #436334). sleep_time_s = 600 # seconds short_timeout_s = 0.010 # seconds for i in range(3): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo) proc = SleepProcess(seconds=sleep_time_s) task_scheduler = TaskScheduler(iter([daemon, proc]), max_jobs=2, event_loop=event_loop) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.cancel() exit_command.reply_hook = exit_command_callback start_time = time.time() self._run(event_loop, task_scheduler, short_timeout_s) hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: event_loop.run_until_complete(build_dir.async_unlock()) shutil.rmtree(tmpdir)
def testIpcDaemon(self): tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') task_scheduler = TaskScheduler(max_jobs=2) build_dir = EbuildBuildDir( scheduler=task_scheduler.sched_iface, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True proc.cancel() daemon.cancel() exit_command.reply_hook = exit_command_callback task_scheduler.add(daemon) task_scheduler.add(proc) start_time = time.time() task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT) task_scheduler.clear() self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for QueueScheduler.run() sleep_time_s = 10 # 10.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit' : exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True proc.cancel() daemon.cancel() exit_command.reply_hook = exit_command_callback task_scheduler.add(daemon) task_scheduler.add(proc) start_time = time.time() task_scheduler.run(timeout=short_timeout_ms) task_scheduler.clear() self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_ipc_daemon', '_exit_command',) _phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) SpawnProcess._start(self) def _init_ipc_fifos(self): input_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join( self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings) commands = { 'best_version' : query_command, 'exit' : self._exit_command, 'has_version' : query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout) if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _unexpected_exit(self): phase = self.phase msg = _("The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): out = StringIO() phase = self.phase for line in lines: eerror(line, phase=phase, key=self.settings.mycpv, out=out) msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: self.scheduler.output(msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: self.returncode = 1 self._unexpected_exit() else: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): self.returncode = 1 self._unexpected_exit()
class AbstractEbuildProcess(SpawnProcess): __slots__ = ('phase', 'settings',) + \ ('_ipc_daemon', '_exit_command',) _phases_without_builddir = ( 'clean', 'cleanrm', 'depend', 'help', ) # Number of milliseconds to allow natural exit of the ebuild # process after it has called the exit command via IPC. It # doesn't hurt to be generous here since the scheduler # continues to process events during this period, and it can # return long before the timeout expires. _exit_timeout = 10000 # 10 seconds # The EbuildIpcDaemon support is well tested, but this variable # is left so we can temporarily disable it if any issues arise. _enable_ipc_daemon = True def __init__(self, **kwargs): SpawnProcess.__init__(self, **kwargs) if self.phase is None: phase = self.settings.get("EBUILD_PHASE") if not phase: phase = 'other' self.phase = phase def _start(self): if self.background: # Automatically prevent color codes from showing up in logs, # since we're not displaying to a terminal anyway. self.settings['NOCOLOR'] = 'true' if self._enable_ipc_daemon: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) if self.phase not in self._phases_without_builddir: self.settings['PORTAGE_IPC_DAEMON'] = "1" self._start_ipc_daemon() else: self.settings.pop('PORTAGE_IPC_DAEMON', None) else: # Since the IPC daemon is disabled, use a simple tempfile based # approach to detect unexpected exit like in bug #190128. self.settings.pop('PORTAGE_IPC_DAEMON', None) if self.phase not in self._phases_without_builddir: exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.exit_status') self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file try: os.unlink(exit_file) except OSError: if os.path.exists(exit_file): # make sure it doesn't exist raise else: self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None) SpawnProcess._start(self) def _init_ipc_fifos(self): input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_out') for p in (input_fifo, output_fifo): st = None try: st = os.lstat(p) except OSError: os.mkfifo(p) else: if not stat.S_ISFIFO(st.st_mode): st = None try: os.unlink(p) except OSError: pass os.mkfifo(p) apply_secpass_permissions(p, uid=os.getuid(), gid=portage.data.portage_gid, mode=0o770, stat_cached=st) return (input_fifo, output_fifo) def _start_ipc_daemon(self): self._exit_command = ExitCommand() self._exit_command.reply_hook = self._exit_command_callback query_command = QueryCommand(self.settings) commands = { 'best_version': query_command, 'exit': self._exit_command, 'has_version': query_command, } input_fifo, output_fifo = self._init_ipc_fifos() self._ipc_daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=self.scheduler) self._ipc_daemon.start() def _exit_command_callback(self): if self._registered: # Let the process exit naturally, if possible. self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout) if self._registered: # If it doesn't exit naturally in a reasonable amount # of time, kill it (solves bug #278895). We try to avoid # this when possible since it makes sandbox complain about # being killed by a signal. self.cancel() def _orphan_process_warn(self): phase = self.phase msg = _("The ebuild phase '%s' with pid %s appears " "to have left an orphan process running in the " "background.") % (phase, self.pid) self._eerror(textwrap.wrap(msg, 72)) def _pipe(self, fd_pipes): stdout_pipe = fd_pipes.get(1) got_pty, master_fd, slave_fd = \ _create_pty_or_pipe(copy_term_size=stdout_pipe) return (master_fd, slave_fd) def _can_log(self, slave_fd): # With sesandbox, logging works through a pty but not through a # normal pipe. So, disable logging if ptys are broken. # See Bug #162404. # TODO: Add support for logging via named pipe (fifo) with # sesandbox, since EbuildIpcDaemon uses a fifo and it's known # to be compatible with sesandbox. return not ('sesandbox' in self.settings.features \ and self.settings.selinux_enabled()) or os.isatty(slave_fd) def _unexpected_exit(self): phase = self.phase msg = _( "The ebuild phase '%s' has exited " "unexpectedly. This type of behavior " "is known to be triggered " "by things such as failed variable " "assignments (bug #190128) or bad substitution " "errors (bug #200313). Normally, before exiting, bash should " "have displayed an error message above. If bash did not " "produce an error message above, it's possible " "that the ebuild has called `exit` when it " "should have called `die` instead. This behavior may also " "be triggered by a corrupt bash binary or a hardware " "problem such as memory or cpu malfunction. If the problem is not " "reproducible or it appears to occur randomly, then it is likely " "to be triggered by a hardware problem. " "If you suspect a hardware problem then you should " "try some basic hardware diagnostics such as memtest. " "Please do not report this as a bug unless it is consistently " "reproducible and you are sure that your bash binary and hardware " "are functioning properly.") % phase self._eerror(textwrap.wrap(msg, 72)) def _eerror(self, lines): out = StringIO() phase = self.phase for line in lines: eerror(line, phase=phase, key=self.settings.mycpv, out=out) msg = _unicode_decode(out.getvalue(), encoding=_encodings['content'], errors='replace') if msg: self.scheduler.output( msg, log_path=self.settings.get("PORTAGE_LOG_FILE")) def _set_returncode(self, wait_retval): SpawnProcess._set_returncode(self, wait_retval) if self._ipc_daemon is not None: self._ipc_daemon.cancel() if self._exit_command.exitcode is not None: self.returncode = self._exit_command.exitcode else: self.returncode = 1 self._unexpected_exit() else: exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE') if exit_file and not os.path.exists(exit_file): self.returncode = 1 self._unexpected_exit()
def testIpcDaemon(self): tmpdir = tempfile.mkdtemp() build_dir = None try: env = {} # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they # need to be inherited by ebuild subprocesses. if 'PORTAGE_USERNAME' in os.environ: env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME'] if 'PORTAGE_GRPNAME' in os.environ: env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME'] env['PORTAGE_PYTHON'] = _python_interpreter env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1') if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ: env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \ os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"] task_scheduler = TaskScheduler(max_jobs=2) build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface, settings=env) build_dir.lock() ensure_dirs(env['PORTAGE_BUILDDIR']) input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in') output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out') os.mkfifo(input_fifo) os.mkfifo(output_fifo) for exitcode in (0, 1, 2): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess(args=[ BASH_BINARY, "-c", '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode ], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.clear() task_scheduler.wait() exit_command.reply_hook = exit_command_callback start_time = time.time() task_scheduler.add(daemon) task_scheduler.add(proc) task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT) task_scheduler.clear() task_scheduler.wait() hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, True, "command not received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(exit_command.exitcode, exitcode) # Intentionally short timeout test for QueueScheduler.run() sleep_time_s = 10 # 10.000 seconds short_timeout_ms = 10 # 0.010 seconds for i in range(3): exit_command = ExitCommand() commands = {'exit': exit_command} daemon = EbuildIpcDaemon(commands=commands, input_fifo=input_fifo, output_fifo=output_fifo, scheduler=task_scheduler.sched_iface) proc = SpawnProcess( args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s], env=env, scheduler=task_scheduler.sched_iface) self.received_command = False def exit_command_callback(): self.received_command = True task_scheduler.clear() task_scheduler.wait() exit_command.reply_hook = exit_command_callback start_time = time.time() task_scheduler.add(daemon) task_scheduler.add(proc) task_scheduler.run(timeout=short_timeout_ms) task_scheduler.clear() task_scheduler.wait() hardlock_cleanup(env['PORTAGE_BUILDDIR'], remove_all_locks=True) self.assertEqual(self.received_command, False, "command received after %d seconds" % \ (time.time() - start_time,)) self.assertEqual(proc.isAlive(), False) self.assertEqual(daemon.isAlive(), False) self.assertEqual(proc.returncode == os.EX_OK, False) finally: if build_dir is not None: build_dir.unlock() shutil.rmtree(tmpdir)