Пример #1
0
	def _start_ipc_daemon(self):
		self._exit_command = ExitCommand()
		self._exit_command.reply_hook = self._exit_command_callback
		query_command = QueryCommand(self.settings, self.phase)
		commands = {
			'best_version' : query_command,
			'exit'         : self._exit_command,
			'has_version'  : query_command,
		}
		input_fifo, output_fifo = self._init_ipc_fifos()
		self._ipc_daemon = EbuildIpcDaemon(commands=commands,
			input_fifo=input_fifo,
			output_fifo=output_fifo,
			scheduler=self.scheduler)
		self._ipc_daemon.start()
Пример #2
0
 def _start_ipc_daemon(self):
     self._exit_command = ExitCommand()
     self._exit_command.reply_hook = self._exit_command_callback
     query_command = QueryCommand(self.settings, self.phase)
     commands = {
         'available_eclasses': query_command,
         'best_version': query_command,
         'eclass_path': query_command,
         'exit': self._exit_command,
         'has_version': query_command,
         'license_path': query_command,
         'master_repositories': query_command,
         'repository_path': query_command,
     }
     input_fifo, output_fifo = self._init_ipc_fifos()
     self._ipc_daemon = EbuildIpcDaemon(commands=commands,
                                        input_fifo=input_fifo,
                                        output_fifo=output_fifo,
                                        scheduler=self.scheduler)
     self._ipc_daemon.start()
Пример #3
0
	def _start_ipc_daemon(self):
		self._exit_command = ExitCommand()
		self._exit_command.reply_hook = self._exit_command_callback
		query_command = QueryCommand(self.settings, self.phase)
		commands = {
			'best_version'        : query_command,
			'exit'                : self._exit_command,
			'has_version'         : query_command,
		}
		input_fifo, output_fifo = self._init_ipc_fifos()
		self._ipc_daemon = EbuildIpcDaemon(commands=commands,
			input_fifo=input_fifo,
			output_fifo=output_fifo,
			scheduler=self.scheduler)
		self._ipc_daemon.start()
Пример #4
0
    def testIpcDaemon(self):
        tmpdir = tempfile.mkdtemp()
        try:
            env = {}

            # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
            # need to be inherited by ebuild subprocesses.
            if 'PORTAGE_USERNAME' in os.environ:
                env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
            if 'PORTAGE_GRPNAME' in os.environ:
                env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']

            env['PORTAGE_PYTHON'] = _python_interpreter
            env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
            env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
            env['PORTAGE_BUILDDIR'] = tmpdir

            input_fifo = os.path.join(tmpdir, '.ipc_in')
            output_fifo = os.path.join(tmpdir, '.ipc_out')
            os.mkfifo(input_fifo)
            os.mkfifo(output_fifo)
            for exitcode in (0, 1, 2):
                task_scheduler = TaskScheduler(max_jobs=2)
                exit_command = ExitCommand()
                commands = {'exit': exit_command}
                daemon = EbuildIpcDaemon(commands=commands,
                                         input_fifo=input_fifo,
                                         output_fifo=output_fifo,
                                         scheduler=task_scheduler.sched_iface)
                proc = SpawnProcess(args=[
                    BASH_BINARY, "-c",
                    '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
                ],
                                    env=env,
                                    scheduler=task_scheduler.sched_iface)

                def exit_command_callback():
                    proc.cancel()
                    daemon.cancel()

                exit_command.reply_hook = exit_command_callback
                task_scheduler.add(daemon)
                task_scheduler.add(proc)
                task_scheduler.run()
                self.assertEqual(exit_command.exitcode, exitcode)
        finally:
            shutil.rmtree(tmpdir)
Пример #5
0
	def _start_ipc_daemon(self):
		self._exit_command = ExitCommand()
		self._exit_command.reply_hook = self._exit_command_callback
		query_command = QueryCommand(self.settings, self.phase)
		commands = {
			'available_eclasses'  : query_command,
			'best_version'        : query_command,
			'eclass_path'         : query_command,
			'exit'                : self._exit_command,
			'has_version'         : query_command,
			'license_path'        : query_command,
			'master_repositories' : query_command,
			'repository_path'     : query_command,
		}
		input_fifo, output_fifo = self._init_ipc_fifos()
		self._ipc_daemon = EbuildIpcDaemon(commands=commands,
			input_fifo=input_fifo,
			output_fifo=output_fifo,
			scheduler=self.scheduler)
		self._ipc_daemon.start()
Пример #6
0
	def testIpcDaemon(self):
		event_loop = global_event_loop()
		tmpdir = tempfile.mkdtemp()
		build_dir = None
		try:
			env = {}

			# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
			# need to be inherited by ebuild subprocesses.
			if 'PORTAGE_USERNAME' in os.environ:
				env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
			if 'PORTAGE_GRPNAME' in os.environ:
				env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']

			env['PORTAGE_PYTHON'] = _python_interpreter
			env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
			env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
			env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')

			if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
				env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
					os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]

			build_dir = EbuildBuildDir(
				scheduler=event_loop,
				settings=env)
			build_dir.lock()
			ensure_dirs(env['PORTAGE_BUILDDIR'])

			input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
			output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
			os.mkfifo(input_fifo)
			os.mkfifo(output_fifo)

			for exitcode in (0, 1, 2):
				exit_command = ExitCommand()
				commands = {'exit' : exit_command}
				daemon = EbuildIpcDaemon(commands=commands,
					input_fifo=input_fifo,
					output_fifo=output_fifo)
				proc = SpawnProcess(
					args=[BASH_BINARY, "-c",
					'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode],
					env=env)
				task_scheduler = TaskScheduler(iter([daemon, proc]),
					max_jobs=2, event_loop=event_loop)

				self.received_command = False
				def exit_command_callback():
					self.received_command = True
					task_scheduler.cancel()

				exit_command.reply_hook = exit_command_callback
				start_time = time.time()
				self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)

				hardlock_cleanup(env['PORTAGE_BUILDDIR'],
					remove_all_locks=True)

				self.assertEqual(self.received_command, True,
					"command not received after %d seconds" % \
					(time.time() - start_time,))
				self.assertEqual(proc.isAlive(), False)
				self.assertEqual(daemon.isAlive(), False)
				self.assertEqual(exit_command.exitcode, exitcode)

			# Intentionally short timeout test for EventLoop/AsyncScheduler.
			# Use a ridiculously long sleep_time_s in case the user's
			# system is heavily loaded (see bug #436334).
			sleep_time_s = 600     #600.000 seconds
			short_timeout_ms = 10  #  0.010 seconds

			for i in range(3):
				exit_command = ExitCommand()
				commands = {'exit' : exit_command}
				daemon = EbuildIpcDaemon(commands=commands,
					input_fifo=input_fifo,
					output_fifo=output_fifo)
				proc = SleepProcess(seconds=sleep_time_s)
				task_scheduler = TaskScheduler(iter([daemon, proc]),
					max_jobs=2, event_loop=event_loop)

				self.received_command = False
				def exit_command_callback():
					self.received_command = True
					task_scheduler.cancel()

				exit_command.reply_hook = exit_command_callback
				start_time = time.time()
				self._run(event_loop, task_scheduler, short_timeout_ms)

				hardlock_cleanup(env['PORTAGE_BUILDDIR'],
					remove_all_locks=True)

				self.assertEqual(self.received_command, False,
					"command received after %d seconds" % \
					(time.time() - start_time,))
				self.assertEqual(proc.isAlive(), False)
				self.assertEqual(daemon.isAlive(), False)
				self.assertEqual(proc.returncode == os.EX_OK, False)

		finally:
			if build_dir is not None:
				build_dir.unlock()
			shutil.rmtree(tmpdir)
Пример #7
0
class AbstractEbuildProcess(SpawnProcess):

    __slots__ = (
        "phase",
        "settings",
    ) + (
        "_build_dir",
        "_build_dir_unlock",
        "_ipc_daemon",
        "_exit_command",
        "_exit_timeout_id",
        "_start_future",
    )

    _phases_without_builddir = (
        "clean",
        "cleanrm",
        "depend",
        "help",
    )
    _phases_interactive_whitelist = ("config", )

    # Number of milliseconds to allow natural exit of the ebuild
    # process after it has called the exit command via IPC. It
    # doesn't hurt to be generous here since the scheduler
    # continues to process events during this period, and it can
    # return long before the timeout expires.
    _exit_timeout = 10  # seconds

    # The EbuildIpcDaemon support is well tested, but this variable
    # is left so we can temporarily disable it if any issues arise.
    _enable_ipc_daemon = True

    def __init__(self, **kwargs):
        SpawnProcess.__init__(self, **kwargs)
        if self.phase is None:
            phase = self.settings.get("EBUILD_PHASE")
            if not phase:
                phase = "other"
            self.phase = phase

    def _start(self):

        need_builddir = self.phase not in self._phases_without_builddir

        # This can happen if the pre-clean phase triggers
        # die_hooks for some reason, and PORTAGE_BUILDDIR
        # doesn't exist yet.
        if need_builddir and not os.path.isdir(
                self.settings["PORTAGE_BUILDDIR"]):
            msg = _("The ebuild phase '%s' has been aborted "
                    "since PORTAGE_BUILDDIR does not exist: '%s'") % (
                        self.phase, self.settings["PORTAGE_BUILDDIR"])
            self._eerror(textwrap.wrap(msg, 72))
            self.returncode = 1
            self._async_wait()
            return

        # Check if the cgroup hierarchy is in place. If it's not, mount it.
        if (os.geteuid() == 0 and platform.system() == "Linux"
                and "cgroup" in self.settings.features
                and self.phase not in _global_pid_phases):
            cgroup_root = "/sys/fs/cgroup"
            cgroup_portage = os.path.join(cgroup_root, "portage")

            try:
                # cgroup tmpfs
                if not os.path.ismount(cgroup_root):
                    # we expect /sys/fs to be there already
                    if not os.path.isdir(cgroup_root):
                        os.mkdir(cgroup_root, 0o755)
                    subprocess.check_call([
                        "mount",
                        "-t",
                        "tmpfs",
                        "-o",
                        "rw,nosuid,nodev,noexec,mode=0755",
                        "tmpfs",
                        cgroup_root,
                    ])

                # portage subsystem
                if not os.path.ismount(cgroup_portage):
                    if not os.path.isdir(cgroup_portage):
                        os.mkdir(cgroup_portage, 0o755)
                    subprocess.check_call([
                        "mount",
                        "-t",
                        "cgroup",
                        "-o",
                        "rw,nosuid,nodev,noexec,none,name=portage",
                        "tmpfs",
                        cgroup_portage,
                    ])
                    with open(os.path.join(cgroup_portage, "release_agent"),
                              "w") as f:
                        f.write(
                            os.path.join(
                                self.settings["PORTAGE_BIN_PATH"],
                                "cgroup-release-agent",
                            ))
                    with open(
                            os.path.join(cgroup_portage, "notify_on_release"),
                            "w") as f:
                        f.write("1")
                else:
                    # Update release_agent if it no longer exists, because
                    # it refers to a temporary path when portage is updating
                    # itself.
                    release_agent = os.path.join(cgroup_portage,
                                                 "release_agent")
                    try:
                        with open(release_agent) as f:
                            release_agent_path = f.readline().rstrip("\n")
                    except EnvironmentError:
                        release_agent_path = None

                    if release_agent_path is None or not os.path.exists(
                            release_agent_path):
                        with open(release_agent, "w") as f:
                            f.write(
                                os.path.join(
                                    self.settings["PORTAGE_BIN_PATH"],
                                    "cgroup-release-agent",
                                ))

                cgroup_path = tempfile.mkdtemp(
                    dir=cgroup_portage,
                    prefix="%s:%s." %
                    (self.settings["CATEGORY"], self.settings["PF"]),
                )
            except (subprocess.CalledProcessError, OSError):
                pass
            else:
                self.cgroup = cgroup_path

        if self.background:
            # Automatically prevent color codes from showing up in logs,
            # since we're not displaying to a terminal anyway.
            self.settings["NOCOLOR"] = "true"

        start_ipc_daemon = False
        if self._enable_ipc_daemon:
            self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)
            if self.phase not in self._phases_without_builddir:
                start_ipc_daemon = True
                if "PORTAGE_BUILDDIR_LOCKED" not in self.settings:
                    self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
                                                     settings=self.settings)
                    self._start_future = self._build_dir.async_lock()
                    self._start_future.add_done_callback(
                        functools.partial(
                            self._start_post_builddir_lock,
                            start_ipc_daemon=start_ipc_daemon,
                        ))
                    return
            else:
                self.settings.pop("PORTAGE_IPC_DAEMON", None)
        else:
            # Since the IPC daemon is disabled, use a simple tempfile based
            # approach to detect unexpected exit like in bug #190128.
            self.settings.pop("PORTAGE_IPC_DAEMON", None)
            if self.phase not in self._phases_without_builddir:
                exit_file = os.path.join(self.settings["PORTAGE_BUILDDIR"],
                                         ".exit_status")
                self.settings["PORTAGE_EBUILD_EXIT_FILE"] = exit_file
                try:
                    os.unlink(exit_file)
                except OSError:
                    if os.path.exists(exit_file):
                        # make sure it doesn't exist
                        raise
            else:
                self.settings.pop("PORTAGE_EBUILD_EXIT_FILE", None)

        self._start_post_builddir_lock(start_ipc_daemon=start_ipc_daemon)

    def _start_post_builddir_lock(self,
                                  lock_future=None,
                                  start_ipc_daemon=False):
        if lock_future is not None:
            if lock_future is not self._start_future:
                raise AssertionError("lock_future is not self._start_future")
            self._start_future = None
            if lock_future.cancelled():
                self._build_dir = None
                self.cancelled = True
                self._was_cancelled()
                self._async_wait()
                return

            lock_future.result()

        if start_ipc_daemon:
            self.settings["PORTAGE_IPC_DAEMON"] = "1"
            self._start_ipc_daemon()

        if self.fd_pipes is None:
            self.fd_pipes = {}
        null_fd = None
        if (0 not in self.fd_pipes
                and self.phase not in self._phases_interactive_whitelist
                and "interactive" not in self.settings.get("PROPERTIES",
                                                           "").split()):
            null_fd = os.open("/dev/null", os.O_RDONLY)
            self.fd_pipes[0] = null_fd

        self.log_filter_file = self.settings.get("PORTAGE_LOG_FILTER_FILE_CMD")
        try:
            SpawnProcess._start(self)
        finally:
            if null_fd is not None:
                os.close(null_fd)

    def _init_ipc_fifos(self):

        input_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"], ".ipc_in")
        output_fifo = os.path.join(self.settings["PORTAGE_BUILDDIR"],
                                   ".ipc_out")

        for p in (input_fifo, output_fifo):

            st = None
            try:
                st = os.lstat(p)
            except OSError:
                os.mkfifo(p)
            else:
                if not stat.S_ISFIFO(st.st_mode):
                    st = None
                    try:
                        os.unlink(p)
                    except OSError:
                        pass
                    os.mkfifo(p)

            apply_secpass_permissions(
                p,
                uid=os.getuid(),
                gid=portage.data.portage_gid,
                mode=0o770,
                stat_cached=st,
            )

        return (input_fifo, output_fifo)

    def _start_ipc_daemon(self):
        self._exit_command = ExitCommand()
        self._exit_command.reply_hook = self._exit_command_callback
        query_command = QueryCommand(self.settings, self.phase)
        commands = {
            "available_eclasses": query_command,
            "best_version": query_command,
            "eclass_path": query_command,
            "exit": self._exit_command,
            "has_version": query_command,
            "license_path": query_command,
            "master_repositories": query_command,
            "repository_path": query_command,
        }
        input_fifo, output_fifo = self._init_ipc_fifos()
        self._ipc_daemon = EbuildIpcDaemon(
            commands=commands,
            input_fifo=input_fifo,
            output_fifo=output_fifo,
            scheduler=self.scheduler,
        )
        self._ipc_daemon.start()

    def _exit_command_callback(self):
        if self._registered:
            # Let the process exit naturally, if possible.
            self._exit_timeout_id = self.scheduler.call_later(
                self._exit_timeout, self._exit_command_timeout_cb)

    def _exit_command_timeout_cb(self):
        if self._registered:
            # If it doesn't exit naturally in a reasonable amount
            # of time, kill it (solves bug #278895). We try to avoid
            # this when possible since it makes sandbox complain about
            # being killed by a signal.
            self.cancel()
            self._exit_timeout_id = self.scheduler.call_later(
                self._cancel_timeout, self._cancel_timeout_cb)
        else:
            self._exit_timeout_id = None

    def _cancel_timeout_cb(self):
        self._exit_timeout_id = None
        self._async_waitpid()

    def _orphan_process_warn(self):
        phase = self.phase

        msg = _("The ebuild phase '%s' with pid %s appears "
                "to have left an orphan process running in the "
                "background.") % (phase, self.pid)

        self._eerror(textwrap.wrap(msg, 72))

    def _pipe(self, fd_pipes):
        stdout_pipe = None
        if not self.background:
            stdout_pipe = fd_pipes.get(1)
        got_pty, master_fd, slave_fd = _create_pty_or_pipe(
            copy_term_size=stdout_pipe)
        return (master_fd, slave_fd)

    def _can_log(self, slave_fd):
        # With sesandbox, logging works through a pty but not through a
        # normal pipe. So, disable logging if ptys are broken.
        # See Bug #162404.
        # TODO: Add support for logging via named pipe (fifo) with
        # sesandbox, since EbuildIpcDaemon uses a fifo and it's known
        # to be compatible with sesandbox.
        return not ("sesandbox" in self.settings.features
                    and self.settings.selinux_enabled()) or os.isatty(slave_fd)

    def _killed_by_signal(self, signum):
        msg = _("The ebuild phase '%s' has been "
                "killed by signal %s.") % (
                    self.phase,
                    signum,
                )
        self._eerror(textwrap.wrap(msg, 72))

    def _unexpected_exit(self):

        phase = self.phase

        msg = (_(
            "The ebuild phase '%s' has exited "
            "unexpectedly. This type of behavior "
            "is known to be triggered "
            "by things such as failed variable "
            "assignments (bug #190128) or bad substitution "
            "errors (bug #200313). Normally, before exiting, bash should "
            "have displayed an error message above. If bash did not "
            "produce an error message above, it's possible "
            "that the ebuild has called `exit` when it "
            "should have called `die` instead. This behavior may also "
            "be triggered by a corrupt bash binary or a hardware "
            "problem such as memory or cpu malfunction. If the problem is not "
            "reproducible or it appears to occur randomly, then it is likely "
            "to be triggered by a hardware problem. "
            "If you suspect a hardware problem then you should "
            "try some basic hardware diagnostics such as memtest. "
            "Please do not report this as a bug unless it is consistently "
            "reproducible and you are sure that your bash binary and hardware "
            "are functioning properly.") % phase)

        self._eerror(textwrap.wrap(msg, 72))

    def _eerror(self, lines):
        self._elog("eerror", lines)

    def _elog(self, elog_funcname, lines):
        out = io.StringIO()
        phase = self.phase
        elog_func = getattr(elog_messages, elog_funcname)
        global_havecolor = portage.output.havecolor
        try:
            portage.output.havecolor = self.settings.get(
                "NOCOLOR", "false").lower() in ("no", "false")
            for line in lines:
                elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
        finally:
            portage.output.havecolor = global_havecolor
        msg = out.getvalue()
        if msg:
            log_path = None
            if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
                log_path = self.settings.get("PORTAGE_LOG_FILE")
            self.scheduler.output(msg, log_path=log_path)

    def _async_waitpid_cb(self, *args, **kwargs):
        """
        Override _async_waitpid_cb to perform cleanup that is
        not necessarily idempotent.
        """
        SpawnProcess._async_waitpid_cb(self, *args, **kwargs)

        if self._exit_timeout_id is not None:
            self._exit_timeout_id.cancel()
            self._exit_timeout_id = None

        if self._ipc_daemon is not None:
            self._ipc_daemon.cancel()
            if self._exit_command.exitcode is not None:
                self.returncode = self._exit_command.exitcode
            else:
                if self.returncode < 0:
                    if not self.cancelled:
                        self._killed_by_signal(-self.returncode)
                else:
                    self.returncode = 1
                    if not self.cancelled:
                        self._unexpected_exit()

        elif not self.cancelled:
            exit_file = self.settings.get("PORTAGE_EBUILD_EXIT_FILE")
            if exit_file and not os.path.exists(exit_file):
                if self.returncode < 0:
                    if not self.cancelled:
                        self._killed_by_signal(-self.returncode)
                else:
                    self.returncode = 1
                    if not self.cancelled:
                        self._unexpected_exit()

    def _async_wait(self):
        """
        Override _async_wait to asynchronously unlock self._build_dir
        when necessary.
        """
        if self._build_dir is None:
            SpawnProcess._async_wait(self)
        elif self._build_dir_unlock is None:
            if self.returncode is None:
                raise asyncio.InvalidStateError("Result is not ready for %s" %
                                                (self, ))
            self._async_unlock_builddir(returncode=self.returncode)

    def _async_unlock_builddir(self, returncode=None):
        """
        Release the lock asynchronously, and if a returncode parameter
        is given then set self.returncode and notify exit listeners.
        """
        if self._build_dir_unlock is not None:
            raise AssertionError("unlock already in progress")
        if returncode is not None:
            # The returncode will be set after unlock is complete.
            self.returncode = None
        self._build_dir_unlock = self._build_dir.async_unlock()
        # Unlock only once.
        self._build_dir = None
        self._build_dir_unlock.add_done_callback(
            functools.partial(self._unlock_builddir_exit,
                              returncode=returncode))

    def _unlock_builddir_exit(self, unlock_future, returncode=None):
        # Normally, async_unlock should not raise an exception here.
        unlock_future.cancelled() or unlock_future.result()
        if returncode is not None:
            if unlock_future.cancelled():
                self.cancelled = True
                self._was_cancelled()
            else:
                self.returncode = returncode
            SpawnProcess._async_wait(self)
Пример #8
0
class AbstractEbuildProcess(SpawnProcess):

	__slots__ = ('phase', 'settings',) + \
		('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')

	_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)
	_phases_interactive_whitelist = ('config',)
	_phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm', 'config')

	# Number of milliseconds to allow natural exit of the ebuild
	# process after it has called the exit command via IPC. It
	# doesn't hurt to be generous here since the scheduler
	# continues to process events during this period, and it can
	# return long before the timeout expires.
	_exit_timeout = 10000 # 10 seconds

	# The EbuildIpcDaemon support is well tested, but this variable
	# is left so we can temporarily disable it if any issues arise.
	_enable_ipc_daemon = True

	def __init__(self, **kwargs):
		SpawnProcess.__init__(self, **kwargs)
		if self.phase is None:
			phase = self.settings.get("EBUILD_PHASE")
			if not phase:
				phase = 'other'
			self.phase = phase

	def _start(self):

		need_builddir = self.phase not in self._phases_without_builddir

		# This can happen if the pre-clean phase triggers
		# die_hooks for some reason, and PORTAGE_BUILDDIR
		# doesn't exist yet.
		if need_builddir and \
			not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
			msg = _("The ebuild phase '%s' has been aborted "
			"since PORTAGE_BUILDDIR does not exist: '%s'") % \
			(self.phase, self.settings['PORTAGE_BUILDDIR'])
			self._eerror(textwrap.wrap(msg, 72))
			self._set_returncode((self.pid, 1 << 8))
			self._async_wait()
			return

		# Check if the cgroup hierarchy is in place. If it's not, mount it.
		if (os.geteuid() == 0 and platform.system() == 'Linux'
				and 'cgroup' in self.settings.features
				and self.phase not in self._phases_without_cgroup):
			cgroup_root = '/sys/fs/cgroup'
			cgroup_portage = os.path.join(cgroup_root, 'portage')

			try:
				# cgroup tmpfs
				if not os.path.ismount(cgroup_root):
					# we expect /sys/fs to be there already
					if not os.path.isdir(cgroup_root):
						os.mkdir(cgroup_root, 0o755)
					subprocess.check_call(['mount', '-t', 'tmpfs',
						'-o', 'rw,nosuid,nodev,noexec,mode=0755',
						'tmpfs', cgroup_root])

				# portage subsystem
				if not os.path.ismount(cgroup_portage):
					if not os.path.isdir(cgroup_portage):
						os.mkdir(cgroup_portage, 0o755)
					subprocess.check_call(['mount', '-t', 'cgroup',
						'-o', 'rw,nosuid,nodev,noexec,none,name=portage',
						'tmpfs', cgroup_portage])

				cgroup_path = tempfile.mkdtemp(dir=cgroup_portage,
					prefix='%s:%s.' % (self.settings["CATEGORY"],
					self.settings["PF"]))
			except (subprocess.CalledProcessError, OSError):
				pass
			else:
				self.cgroup = cgroup_path

		if self.background:
			# Automatically prevent color codes from showing up in logs,
			# since we're not displaying to a terminal anyway.
			self.settings['NOCOLOR'] = 'true'

		if self._enable_ipc_daemon:
			self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
			if self.phase not in self._phases_without_builddir:
				if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
					self._build_dir = EbuildBuildDir(
						scheduler=self.scheduler, settings=self.settings)
					self._build_dir.lock()
				self.settings['PORTAGE_IPC_DAEMON'] = "1"
				self._start_ipc_daemon()
			else:
				self.settings.pop('PORTAGE_IPC_DAEMON', None)
		else:
			# Since the IPC daemon is disabled, use a simple tempfile based
			# approach to detect unexpected exit like in bug #190128.
			self.settings.pop('PORTAGE_IPC_DAEMON', None)
			if self.phase not in self._phases_without_builddir:
				exit_file = os.path.join(
					self.settings['PORTAGE_BUILDDIR'],
					'.exit_status')
				self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
				try:
					os.unlink(exit_file)
				except OSError:
					if os.path.exists(exit_file):
						# make sure it doesn't exist
						raise
			else:
				self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

		if self.fd_pipes is None:
			self.fd_pipes = {}
		null_fd = None
		if 0 not in self.fd_pipes and \
			self.phase not in self._phases_interactive_whitelist and \
			"interactive" not in self.settings.get("PROPERTIES", "").split():
			null_fd = os.open('/dev/null', os.O_RDONLY)
			self.fd_pipes[0] = null_fd

		try:
			SpawnProcess._start(self)
		finally:
			if null_fd is not None:
				os.close(null_fd)

	def _init_ipc_fifos(self):

		input_fifo = os.path.join(
			self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
		output_fifo = os.path.join(
			self.settings['PORTAGE_BUILDDIR'], '.ipc_out')

		for p in (input_fifo, output_fifo):

			st = None
			try:
				st = os.lstat(p)
			except OSError:
				os.mkfifo(p)
			else:
				if not stat.S_ISFIFO(st.st_mode):
					st = None
					try:
						os.unlink(p)
					except OSError:
						pass
					os.mkfifo(p)

			apply_secpass_permissions(p,
				uid=os.getuid(),
				gid=portage.data.portage_gid,
				mode=0o770, stat_cached=st)

		return (input_fifo, output_fifo)

	def _start_ipc_daemon(self):
		self._exit_command = ExitCommand()
		self._exit_command.reply_hook = self._exit_command_callback
		query_command = QueryCommand(self.settings, self.phase)
		commands = {
			'available_eclasses'  : query_command,
			'best_version'        : query_command,
			'eclass_path'         : query_command,
			'exit'                : self._exit_command,
			'has_version'         : query_command,
			'license_path'        : query_command,
			'master_repositories' : query_command,
			'repository_path'     : query_command,
		}
		input_fifo, output_fifo = self._init_ipc_fifos()
		self._ipc_daemon = EbuildIpcDaemon(commands=commands,
			input_fifo=input_fifo,
			output_fifo=output_fifo,
			scheduler=self.scheduler)
		self._ipc_daemon.start()

	def _exit_command_callback(self):
		if self._registered:
			# Let the process exit naturally, if possible.
			self._exit_timeout_id = \
				self.scheduler.timeout_add(self._exit_timeout,
				self._exit_command_timeout_cb)

	def _exit_command_timeout_cb(self):
		if self._registered:
			# If it doesn't exit naturally in a reasonable amount
			# of time, kill it (solves bug #278895). We try to avoid
			# this when possible since it makes sandbox complain about
			# being killed by a signal.
			self.cancel()
			self._exit_timeout_id = \
				self.scheduler.timeout_add(self._cancel_timeout,
					self._cancel_timeout_cb)
		else:
			self._exit_timeout_id = None

		return False # only run once

	def _cancel_timeout_cb(self):
		self._exit_timeout_id = None
		self.wait()
		return False # only run once

	def _orphan_process_warn(self):
		phase = self.phase

		msg = _("The ebuild phase '%s' with pid %s appears "
		"to have left an orphan process running in the "
		"background.") % (phase, self.pid)

		self._eerror(textwrap.wrap(msg, 72))

	def _pipe(self, fd_pipes):
		stdout_pipe = None
		if not self.background:
			stdout_pipe = fd_pipes.get(1)
		got_pty, master_fd, slave_fd = \
			_create_pty_or_pipe(copy_term_size=stdout_pipe)
		return (master_fd, slave_fd)

	def _can_log(self, slave_fd):
		# With sesandbox, logging works through a pty but not through a
		# normal pipe. So, disable logging if ptys are broken.
		# See Bug #162404.
		# TODO: Add support for logging via named pipe (fifo) with
		# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
		# to be compatible with sesandbox.
		return not ('sesandbox' in self.settings.features \
			and self.settings.selinux_enabled()) or os.isatty(slave_fd)

	def _killed_by_signal(self, signum):
		msg = _("The ebuild phase '%s' has been "
		"killed by signal %s.") % (self.phase, signum)
		self._eerror(textwrap.wrap(msg, 72))

	def _unexpected_exit(self):

		phase = self.phase

		msg = _("The ebuild phase '%s' has exited "
		"unexpectedly. This type of behavior "
		"is known to be triggered "
		"by things such as failed variable "
		"assignments (bug #190128) or bad substitution "
		"errors (bug #200313). Normally, before exiting, bash should "
		"have displayed an error message above. If bash did not "
		"produce an error message above, it's possible "
		"that the ebuild has called `exit` when it "
		"should have called `die` instead. This behavior may also "
		"be triggered by a corrupt bash binary or a hardware "
		"problem such as memory or cpu malfunction. If the problem is not "
		"reproducible or it appears to occur randomly, then it is likely "
		"to be triggered by a hardware problem. "
		"If you suspect a hardware problem then you should "
		"try some basic hardware diagnostics such as memtest. "
		"Please do not report this as a bug unless it is consistently "
		"reproducible and you are sure that your bash binary and hardware "
		"are functioning properly.") % phase

		self._eerror(textwrap.wrap(msg, 72))

	def _eerror(self, lines):
		self._elog('eerror', lines)

	def _elog(self, elog_funcname, lines):
		out = io.StringIO()
		phase = self.phase
		elog_func = getattr(elog_messages, elog_funcname)
		global_havecolor = portage.output.havecolor
		try:
			portage.output.havecolor = \
				self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
			for line in lines:
				elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
		finally:
			portage.output.havecolor = global_havecolor
		msg = out.getvalue()
		if msg:
			log_path = None
			if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
				log_path = self.settings.get("PORTAGE_LOG_FILE")
			self.scheduler.output(msg, log_path=log_path)

	def _log_poll_exception(self, event):
		self._elog("eerror",
			["%s received strange poll event: %s\n" % \
			(self.__class__.__name__, event,)])

	def _set_returncode(self, wait_retval):
		SpawnProcess._set_returncode(self, wait_retval)

		if self.cgroup is not None:
			try:
				shutil.rmtree(self.cgroup)
			except EnvironmentError as e:
				if e.errno != errno.ENOENT:
					raise

		if self._exit_timeout_id is not None:
			self.scheduler.source_remove(self._exit_timeout_id)
			self._exit_timeout_id = None

		if self._ipc_daemon is not None:
			self._ipc_daemon.cancel()
			if self._exit_command.exitcode is not None:
				self.returncode = self._exit_command.exitcode
			else:
				if self.returncode < 0:
					if not self.cancelled:
						self._killed_by_signal(-self.returncode)
				else:
					self.returncode = 1
					if not self.cancelled:
						self._unexpected_exit()
			if self._build_dir is not None:
				self._build_dir.unlock()
				self._build_dir = None
		elif not self.cancelled:
			exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
			if exit_file and not os.path.exists(exit_file):
				if self.returncode < 0:
					if not self.cancelled:
						self._killed_by_signal(-self.returncode)
				else:
					self.returncode = 1
					if not self.cancelled:
						self._unexpected_exit()
Пример #9
0
class AbstractEbuildProcess(SpawnProcess):

    __slots__ = ('phase', 'settings',) + \
     ('_build_dir', '_ipc_daemon', '_exit_command', '_exit_timeout_id')

    _phases_without_builddir = (
        'clean',
        'cleanrm',
        'depend',
        'help',
    )
    _phases_interactive_whitelist = ('config', )
    _phases_without_cgroup = ('preinst', 'postinst', 'prerm', 'postrm',
                              'config')

    # Number of milliseconds to allow natural exit of the ebuild
    # process after it has called the exit command via IPC. It
    # doesn't hurt to be generous here since the scheduler
    # continues to process events during this period, and it can
    # return long before the timeout expires.
    _exit_timeout = 10000  # 10 seconds

    # The EbuildIpcDaemon support is well tested, but this variable
    # is left so we can temporarily disable it if any issues arise.
    _enable_ipc_daemon = True

    def __init__(self, **kwargs):
        SpawnProcess.__init__(self, **kwargs)
        if self.phase is None:
            phase = self.settings.get("EBUILD_PHASE")
            if not phase:
                phase = 'other'
            self.phase = phase

    def _start(self):

        need_builddir = self.phase not in self._phases_without_builddir

        # This can happen if the pre-clean phase triggers
        # die_hooks for some reason, and PORTAGE_BUILDDIR
        # doesn't exist yet.
        if need_builddir and \
         not os.path.isdir(self.settings['PORTAGE_BUILDDIR']):
            msg = _("The ebuild phase '%s' has been aborted "
            "since PORTAGE_BUILDDIR does not exist: '%s'") % \
            (self.phase, self.settings['PORTAGE_BUILDDIR'])
            self._eerror(textwrap.wrap(msg, 72))
            self._set_returncode((self.pid, 1 << 8))
            self._async_wait()
            return

        # Check if the cgroup hierarchy is in place. If it's not, mount it.
        if (os.geteuid() == 0 and platform.system() == 'Linux'
                and 'cgroup' in self.settings.features
                and self.phase not in self._phases_without_cgroup):
            cgroup_root = '/sys/fs/cgroup'
            cgroup_portage = os.path.join(cgroup_root, 'portage')

            try:
                # cgroup tmpfs
                if not os.path.ismount(cgroup_root):
                    # we expect /sys/fs to be there already
                    if not os.path.isdir(cgroup_root):
                        os.mkdir(cgroup_root, 0o755)
                    subprocess.check_call([
                        'mount', '-t', 'tmpfs', '-o',
                        'rw,nosuid,nodev,noexec,mode=0755', 'tmpfs',
                        cgroup_root
                    ])

                # portage subsystem
                if not os.path.ismount(cgroup_portage):
                    if not os.path.isdir(cgroup_portage):
                        os.mkdir(cgroup_portage, 0o755)
                    subprocess.check_call([
                        'mount', '-t', 'cgroup', '-o',
                        'rw,nosuid,nodev,noexec,none,name=portage', 'tmpfs',
                        cgroup_portage
                    ])

                cgroup_path = tempfile.mkdtemp(
                    dir=cgroup_portage,
                    prefix='%s:%s.' %
                    (self.settings["CATEGORY"], self.settings["PF"]))
            except (subprocess.CalledProcessError, OSError):
                pass
            else:
                self.cgroup = cgroup_path

        if self.background:
            # Automatically prevent color codes from showing up in logs,
            # since we're not displaying to a terminal anyway.
            self.settings['NOCOLOR'] = 'true'

        if self._enable_ipc_daemon:
            self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
            if self.phase not in self._phases_without_builddir:
                if 'PORTAGE_BUILDDIR_LOCKED' not in self.settings:
                    self._build_dir = EbuildBuildDir(scheduler=self.scheduler,
                                                     settings=self.settings)
                    self._build_dir.lock()
                self.settings['PORTAGE_IPC_DAEMON'] = "1"
                self._start_ipc_daemon()
            else:
                self.settings.pop('PORTAGE_IPC_DAEMON', None)
        else:
            # Since the IPC daemon is disabled, use a simple tempfile based
            # approach to detect unexpected exit like in bug #190128.
            self.settings.pop('PORTAGE_IPC_DAEMON', None)
            if self.phase not in self._phases_without_builddir:
                exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
                                         '.exit_status')
                self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
                try:
                    os.unlink(exit_file)
                except OSError:
                    if os.path.exists(exit_file):
                        # make sure it doesn't exist
                        raise
            else:
                self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

        if self.fd_pipes is None:
            self.fd_pipes = {}
        null_fd = None
        if 0 not in self.fd_pipes and \
         self.phase not in self._phases_interactive_whitelist and \
         "interactive" not in self.settings.get("PROPERTIES", "").split():
            null_fd = os.open('/dev/null', os.O_RDONLY)
            self.fd_pipes[0] = null_fd

        try:
            SpawnProcess._start(self)
        finally:
            if null_fd is not None:
                os.close(null_fd)

    def _init_ipc_fifos(self):

        input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
        output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'],
                                   '.ipc_out')

        for p in (input_fifo, output_fifo):

            st = None
            try:
                st = os.lstat(p)
            except OSError:
                os.mkfifo(p)
            else:
                if not stat.S_ISFIFO(st.st_mode):
                    st = None
                    try:
                        os.unlink(p)
                    except OSError:
                        pass
                    os.mkfifo(p)

            apply_secpass_permissions(p,
                                      uid=os.getuid(),
                                      gid=portage.data.portage_gid,
                                      mode=0o770,
                                      stat_cached=st)

        return (input_fifo, output_fifo)

    def _start_ipc_daemon(self):
        self._exit_command = ExitCommand()
        self._exit_command.reply_hook = self._exit_command_callback
        query_command = QueryCommand(self.settings, self.phase)
        commands = {
            'available_eclasses': query_command,
            'best_version': query_command,
            'eclass_path': query_command,
            'exit': self._exit_command,
            'has_version': query_command,
            'license_path': query_command,
            'master_repositories': query_command,
            'repository_path': query_command,
        }
        input_fifo, output_fifo = self._init_ipc_fifos()
        self._ipc_daemon = EbuildIpcDaemon(commands=commands,
                                           input_fifo=input_fifo,
                                           output_fifo=output_fifo,
                                           scheduler=self.scheduler)
        self._ipc_daemon.start()

    def _exit_command_callback(self):
        if self._registered:
            # Let the process exit naturally, if possible.
            self._exit_timeout_id = \
             self.scheduler.timeout_add(self._exit_timeout,
             self._exit_command_timeout_cb)

    def _exit_command_timeout_cb(self):
        if self._registered:
            # If it doesn't exit naturally in a reasonable amount
            # of time, kill it (solves bug #278895). We try to avoid
            # this when possible since it makes sandbox complain about
            # being killed by a signal.
            self.cancel()
            self._exit_timeout_id = \
             self.scheduler.timeout_add(self._cancel_timeout,
              self._cancel_timeout_cb)
        else:
            self._exit_timeout_id = None

        return False  # only run once

    def _cancel_timeout_cb(self):
        self._exit_timeout_id = None
        self.wait()
        return False  # only run once

    def _orphan_process_warn(self):
        phase = self.phase

        msg = _("The ebuild phase '%s' with pid %s appears "
                "to have left an orphan process running in the "
                "background.") % (phase, self.pid)

        self._eerror(textwrap.wrap(msg, 72))

    def _pipe(self, fd_pipes):
        stdout_pipe = None
        if not self.background:
            stdout_pipe = fd_pipes.get(1)
        got_pty, master_fd, slave_fd = \
         _create_pty_or_pipe(copy_term_size=stdout_pipe)
        return (master_fd, slave_fd)

    def _can_log(self, slave_fd):
        # With sesandbox, logging works through a pty but not through a
        # normal pipe. So, disable logging if ptys are broken.
        # See Bug #162404.
        # TODO: Add support for logging via named pipe (fifo) with
        # sesandbox, since EbuildIpcDaemon uses a fifo and it's known
        # to be compatible with sesandbox.
        return not ('sesandbox' in self.settings.features \
         and self.settings.selinux_enabled()) or os.isatty(slave_fd)

    def _killed_by_signal(self, signum):
        msg = _("The ebuild phase '%s' has been "
                "killed by signal %s.") % (self.phase, signum)
        self._eerror(textwrap.wrap(msg, 72))

    def _unexpected_exit(self):

        phase = self.phase

        msg = _(
            "The ebuild phase '%s' has exited "
            "unexpectedly. This type of behavior "
            "is known to be triggered "
            "by things such as failed variable "
            "assignments (bug #190128) or bad substitution "
            "errors (bug #200313). Normally, before exiting, bash should "
            "have displayed an error message above. If bash did not "
            "produce an error message above, it's possible "
            "that the ebuild has called `exit` when it "
            "should have called `die` instead. This behavior may also "
            "be triggered by a corrupt bash binary or a hardware "
            "problem such as memory or cpu malfunction. If the problem is not "
            "reproducible or it appears to occur randomly, then it is likely "
            "to be triggered by a hardware problem. "
            "If you suspect a hardware problem then you should "
            "try some basic hardware diagnostics such as memtest. "
            "Please do not report this as a bug unless it is consistently "
            "reproducible and you are sure that your bash binary and hardware "
            "are functioning properly.") % phase

        self._eerror(textwrap.wrap(msg, 72))

    def _eerror(self, lines):
        self._elog('eerror', lines)

    def _elog(self, elog_funcname, lines):
        out = io.StringIO()
        phase = self.phase
        elog_func = getattr(elog_messages, elog_funcname)
        global_havecolor = portage.output.havecolor
        try:
            portage.output.havecolor = \
             self.settings.get('NOCOLOR', 'false').lower() in ('no', 'false')
            for line in lines:
                elog_func(line, phase=phase, key=self.settings.mycpv, out=out)
        finally:
            portage.output.havecolor = global_havecolor
        msg = out.getvalue()
        if msg:
            log_path = None
            if self.settings.get("PORTAGE_BACKGROUND") != "subprocess":
                log_path = self.settings.get("PORTAGE_LOG_FILE")
            self.scheduler.output(msg, log_path=log_path)

    def _log_poll_exception(self, event):
        self._elog("eerror",
         ["%s received strange poll event: %s\n" % \
         (self.__class__.__name__, event,)])

    def _set_returncode(self, wait_retval):
        SpawnProcess._set_returncode(self, wait_retval)

        if self.cgroup is not None:
            try:
                shutil.rmtree(self.cgroup)
            except EnvironmentError as e:
                if e.errno != errno.ENOENT:
                    raise

        if self._exit_timeout_id is not None:
            self.scheduler.source_remove(self._exit_timeout_id)
            self._exit_timeout_id = None

        if self._ipc_daemon is not None:
            self._ipc_daemon.cancel()
            if self._exit_command.exitcode is not None:
                self.returncode = self._exit_command.exitcode
            else:
                if self.returncode < 0:
                    if not self.cancelled:
                        self._killed_by_signal(-self.returncode)
                else:
                    self.returncode = 1
                    if not self.cancelled:
                        self._unexpected_exit()
            if self._build_dir is not None:
                self._build_dir.unlock()
                self._build_dir = None
        elif not self.cancelled:
            exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
            if exit_file and not os.path.exists(exit_file):
                if self.returncode < 0:
                    if not self.cancelled:
                        self._killed_by_signal(-self.returncode)
                else:
                    self.returncode = 1
                    if not self.cancelled:
                        self._unexpected_exit()
Пример #10
0
    def testIpcDaemon(self):
        event_loop = global_event_loop()
        tmpdir = tempfile.mkdtemp()
        build_dir = None
        try:
            env = {}

            # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
            # need to be inherited by ebuild subprocesses.
            if 'PORTAGE_USERNAME' in os.environ:
                env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
            if 'PORTAGE_GRPNAME' in os.environ:
                env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']

            env['PORTAGE_PYTHON'] = _python_interpreter
            env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
            env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
            env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')
            env['PYTHONDONTWRITEBYTECODE'] = os.environ.get(
                'PYTHONDONTWRITEBYTECODE', '')

            if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
                env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
                 os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]

            build_dir = EbuildBuildDir(scheduler=event_loop, settings=env)
            event_loop.run_until_complete(build_dir.async_lock())
            ensure_dirs(env['PORTAGE_BUILDDIR'])

            input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
            output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
            os.mkfifo(input_fifo)
            os.mkfifo(output_fifo)

            for exitcode in (0, 1, 2):
                exit_command = ExitCommand()
                commands = {'exit': exit_command}
                daemon = EbuildIpcDaemon(commands=commands,
                                         input_fifo=input_fifo,
                                         output_fifo=output_fifo)
                proc = SpawnProcess(args=[
                    BASH_BINARY, "-c",
                    '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
                ],
                                    env=env)
                task_scheduler = TaskScheduler(iter([daemon, proc]),
                                               max_jobs=2,
                                               event_loop=event_loop)

                self.received_command = False

                def exit_command_callback():
                    self.received_command = True
                    task_scheduler.cancel()

                exit_command.reply_hook = exit_command_callback
                start_time = time.time()
                self._run(event_loop, task_scheduler, self._SCHEDULE_TIMEOUT)

                hardlock_cleanup(env['PORTAGE_BUILDDIR'],
                                 remove_all_locks=True)

                self.assertEqual(self.received_command, True,
                 "command not received after %d seconds" % \
                 (time.time() - start_time,))
                self.assertEqual(proc.isAlive(), False)
                self.assertEqual(daemon.isAlive(), False)
                self.assertEqual(exit_command.exitcode, exitcode)

            # Intentionally short timeout test for EventLoop/AsyncScheduler.
            # Use a ridiculously long sleep_time_s in case the user's
            # system is heavily loaded (see bug #436334).
            sleep_time_s = 600  # seconds
            short_timeout_s = 0.010  # seconds

            for i in range(3):
                exit_command = ExitCommand()
                commands = {'exit': exit_command}
                daemon = EbuildIpcDaemon(commands=commands,
                                         input_fifo=input_fifo,
                                         output_fifo=output_fifo)
                proc = SleepProcess(seconds=sleep_time_s)
                task_scheduler = TaskScheduler(iter([daemon, proc]),
                                               max_jobs=2,
                                               event_loop=event_loop)

                self.received_command = False

                def exit_command_callback():
                    self.received_command = True
                    task_scheduler.cancel()

                exit_command.reply_hook = exit_command_callback
                start_time = time.time()
                self._run(event_loop, task_scheduler, short_timeout_s)

                hardlock_cleanup(env['PORTAGE_BUILDDIR'],
                                 remove_all_locks=True)

                self.assertEqual(self.received_command, False,
                 "command received after %d seconds" % \
                 (time.time() - start_time,))
                self.assertEqual(proc.isAlive(), False)
                self.assertEqual(daemon.isAlive(), False)
                self.assertEqual(proc.returncode == os.EX_OK, False)

        finally:
            if build_dir is not None:
                event_loop.run_until_complete(build_dir.async_unlock())
            shutil.rmtree(tmpdir)
Пример #11
0
	def testIpcDaemon(self):
		tmpdir = tempfile.mkdtemp()
		build_dir = None
		try:
			env = {}

			# Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
			# need to be inherited by ebuild subprocesses.
			if 'PORTAGE_USERNAME' in os.environ:
				env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
			if 'PORTAGE_GRPNAME' in os.environ:
				env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']

			env['PORTAGE_PYTHON'] = _python_interpreter
			env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
			env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
			env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')

			task_scheduler = TaskScheduler(max_jobs=2)
			build_dir = EbuildBuildDir(
				scheduler=task_scheduler.sched_iface,
				settings=env)
			build_dir.lock()
			ensure_dirs(env['PORTAGE_BUILDDIR'])

			input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
			output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
			os.mkfifo(input_fifo)
			os.mkfifo(output_fifo)

			for exitcode in (0, 1, 2):
				exit_command = ExitCommand()
				commands = {'exit' : exit_command}
				daemon = EbuildIpcDaemon(commands=commands,
					input_fifo=input_fifo,
					output_fifo=output_fifo,
					scheduler=task_scheduler.sched_iface)
				proc = SpawnProcess(
					args=[BASH_BINARY, "-c",
					'"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode],
					env=env, scheduler=task_scheduler.sched_iface)

				self.received_command = False
				def exit_command_callback():
					self.received_command = True
					proc.cancel()
					daemon.cancel()

				exit_command.reply_hook = exit_command_callback
				task_scheduler.add(daemon)
				task_scheduler.add(proc)
				start_time = time.time()
				task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
				task_scheduler.clear()

				self.assertEqual(self.received_command, True,
					"command not received after %d seconds" % \
					(time.time() - start_time,))
				self.assertEqual(proc.isAlive(), False)
				self.assertEqual(daemon.isAlive(), False)
				self.assertEqual(exit_command.exitcode, exitcode)

			# Intentionally short timeout test for QueueScheduler.run()
			sleep_time_s = 10      # 10.000 seconds
			short_timeout_ms = 10  #  0.010 seconds

			for i in range(3):
				exit_command = ExitCommand()
				commands = {'exit' : exit_command}
				daemon = EbuildIpcDaemon(commands=commands,
					input_fifo=input_fifo,
					output_fifo=output_fifo,
					scheduler=task_scheduler.sched_iface)
				proc = SpawnProcess(
					args=[BASH_BINARY, "-c", 'exec sleep %d' % sleep_time_s],
					env=env, scheduler=task_scheduler.sched_iface)

				self.received_command = False
				def exit_command_callback():
					self.received_command = True
					proc.cancel()
					daemon.cancel()

				exit_command.reply_hook = exit_command_callback
				task_scheduler.add(daemon)
				task_scheduler.add(proc)
				start_time = time.time()
				task_scheduler.run(timeout=short_timeout_ms)
				task_scheduler.clear()

				self.assertEqual(self.received_command, False,
					"command received after %d seconds" % \
					(time.time() - start_time,))
				self.assertEqual(proc.isAlive(), False)
				self.assertEqual(daemon.isAlive(), False)
				self.assertEqual(proc.returncode == os.EX_OK, False)

		finally:
			if build_dir is not None:
				build_dir.unlock()
			shutil.rmtree(tmpdir)
Пример #12
0
class AbstractEbuildProcess(SpawnProcess):

	__slots__ = ('phase', 'settings',) + \
		('_ipc_daemon', '_exit_command',)
	_phases_without_builddir = ('clean', 'cleanrm', 'depend', 'help',)

	# Number of milliseconds to allow natural exit of the ebuild
	# process after it has called the exit command via IPC. It
	# doesn't hurt to be generous here since the scheduler
	# continues to process events during this period, and it can
	# return long before the timeout expires.
	_exit_timeout = 10000 # 10 seconds

	# The EbuildIpcDaemon support is well tested, but this variable
	# is left so we can temporarily disable it if any issues arise.
	_enable_ipc_daemon = True

	def __init__(self, **kwargs):
		SpawnProcess.__init__(self, **kwargs)
		if self.phase is None:
			phase = self.settings.get("EBUILD_PHASE")
			if not phase:
				phase = 'other'
			self.phase = phase

	def _start(self):

		if self.background:
			# Automatically prevent color codes from showing up in logs,
			# since we're not displaying to a terminal anyway.
			self.settings['NOCOLOR'] = 'true'

		if self._enable_ipc_daemon:
			self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
			if self.phase not in self._phases_without_builddir:
				self.settings['PORTAGE_IPC_DAEMON'] = "1"
				self._start_ipc_daemon()
			else:
				self.settings.pop('PORTAGE_IPC_DAEMON', None)
		else:
			# Since the IPC daemon is disabled, use a simple tempfile based
			# approach to detect unexpected exit like in bug #190128.
			self.settings.pop('PORTAGE_IPC_DAEMON', None)
			if self.phase not in self._phases_without_builddir:
				exit_file = os.path.join(
					self.settings['PORTAGE_BUILDDIR'],
					'.exit_status')
				self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
				try:
					os.unlink(exit_file)
				except OSError:
					if os.path.exists(exit_file):
						# make sure it doesn't exist
						raise
			else:
				self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

		SpawnProcess._start(self)

	def _init_ipc_fifos(self):

		input_fifo = os.path.join(
			self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
		output_fifo = os.path.join(
			self.settings['PORTAGE_BUILDDIR'], '.ipc_out')

		for p in (input_fifo, output_fifo):

			st = None
			try:
				st = os.lstat(p)
			except OSError:
				os.mkfifo(p)
			else:
				if not stat.S_ISFIFO(st.st_mode):
					st = None
					try:
						os.unlink(p)
					except OSError:
						pass
					os.mkfifo(p)

			apply_secpass_permissions(p,
				uid=os.getuid(),
				gid=portage.data.portage_gid,
				mode=0o770, stat_cached=st)

		return (input_fifo, output_fifo)

	def _start_ipc_daemon(self):
		self._exit_command = ExitCommand()
		self._exit_command.reply_hook = self._exit_command_callback
		query_command = QueryCommand(self.settings)
		commands = {
			'best_version' : query_command,
			'exit'         : self._exit_command,
			'has_version'  : query_command,
		}
		input_fifo, output_fifo = self._init_ipc_fifos()
		self._ipc_daemon = EbuildIpcDaemon(commands=commands,
			input_fifo=input_fifo,
			output_fifo=output_fifo,
			scheduler=self.scheduler)
		self._ipc_daemon.start()

	def _exit_command_callback(self):
		if self._registered:
			# Let the process exit naturally, if possible.
			self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout)
			if self._registered:
				# If it doesn't exit naturally in a reasonable amount
				# of time, kill it (solves bug #278895). We try to avoid
				# this when possible since it makes sandbox complain about
				# being killed by a signal.
				self.cancel()

	def _orphan_process_warn(self):
		phase = self.phase

		msg = _("The ebuild phase '%s' with pid %s appears "
		"to have left an orphan process running in the "
		"background.") % (phase, self.pid)

		self._eerror(textwrap.wrap(msg, 72))

	def _pipe(self, fd_pipes):
		stdout_pipe = fd_pipes.get(1)
		got_pty, master_fd, slave_fd = \
			_create_pty_or_pipe(copy_term_size=stdout_pipe)
		return (master_fd, slave_fd)

	def _can_log(self, slave_fd):
		# With sesandbox, logging works through a pty but not through a
		# normal pipe. So, disable logging if ptys are broken.
		# See Bug #162404.
		# TODO: Add support for logging via named pipe (fifo) with
		# sesandbox, since EbuildIpcDaemon uses a fifo and it's known
		# to be compatible with sesandbox.
		return not ('sesandbox' in self.settings.features \
			and self.settings.selinux_enabled()) or os.isatty(slave_fd)

	def _unexpected_exit(self):

		phase = self.phase

		msg = _("The ebuild phase '%s' has exited "
		"unexpectedly. This type of behavior "
		"is known to be triggered "
		"by things such as failed variable "
		"assignments (bug #190128) or bad substitution "
		"errors (bug #200313). Normally, before exiting, bash should "
		"have displayed an error message above. If bash did not "
		"produce an error message above, it's possible "
		"that the ebuild has called `exit` when it "
		"should have called `die` instead. This behavior may also "
		"be triggered by a corrupt bash binary or a hardware "
		"problem such as memory or cpu malfunction. If the problem is not "
		"reproducible or it appears to occur randomly, then it is likely "
		"to be triggered by a hardware problem. "
		"If you suspect a hardware problem then you should "
		"try some basic hardware diagnostics such as memtest. "
		"Please do not report this as a bug unless it is consistently "
		"reproducible and you are sure that your bash binary and hardware "
		"are functioning properly.") % phase

		self._eerror(textwrap.wrap(msg, 72))

	def _eerror(self, lines):
		out = StringIO()
		phase = self.phase
		for line in lines:
			eerror(line, phase=phase, key=self.settings.mycpv, out=out)
		msg = _unicode_decode(out.getvalue(),
			encoding=_encodings['content'], errors='replace')
		if msg:
			self.scheduler.output(msg,
				log_path=self.settings.get("PORTAGE_LOG_FILE"))

	def _set_returncode(self, wait_retval):
		SpawnProcess._set_returncode(self, wait_retval)

		if self._ipc_daemon is not None:
			self._ipc_daemon.cancel()
			if self._exit_command.exitcode is not None:
				self.returncode = self._exit_command.exitcode
			else:
				self.returncode = 1
				self._unexpected_exit()
		else:
			exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
			if exit_file and not os.path.exists(exit_file):
				self.returncode = 1
				self._unexpected_exit()
Пример #13
0
class AbstractEbuildProcess(SpawnProcess):

    __slots__ = ('phase', 'settings',) + \
     ('_ipc_daemon', '_exit_command',)
    _phases_without_builddir = (
        'clean',
        'cleanrm',
        'depend',
        'help',
    )

    # Number of milliseconds to allow natural exit of the ebuild
    # process after it has called the exit command via IPC. It
    # doesn't hurt to be generous here since the scheduler
    # continues to process events during this period, and it can
    # return long before the timeout expires.
    _exit_timeout = 10000  # 10 seconds

    # The EbuildIpcDaemon support is well tested, but this variable
    # is left so we can temporarily disable it if any issues arise.
    _enable_ipc_daemon = True

    def __init__(self, **kwargs):
        SpawnProcess.__init__(self, **kwargs)
        if self.phase is None:
            phase = self.settings.get("EBUILD_PHASE")
            if not phase:
                phase = 'other'
            self.phase = phase

    def _start(self):

        if self.background:
            # Automatically prevent color codes from showing up in logs,
            # since we're not displaying to a terminal anyway.
            self.settings['NOCOLOR'] = 'true'

        if self._enable_ipc_daemon:
            self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)
            if self.phase not in self._phases_without_builddir:
                self.settings['PORTAGE_IPC_DAEMON'] = "1"
                self._start_ipc_daemon()
            else:
                self.settings.pop('PORTAGE_IPC_DAEMON', None)
        else:
            # Since the IPC daemon is disabled, use a simple tempfile based
            # approach to detect unexpected exit like in bug #190128.
            self.settings.pop('PORTAGE_IPC_DAEMON', None)
            if self.phase not in self._phases_without_builddir:
                exit_file = os.path.join(self.settings['PORTAGE_BUILDDIR'],
                                         '.exit_status')
                self.settings['PORTAGE_EBUILD_EXIT_FILE'] = exit_file
                try:
                    os.unlink(exit_file)
                except OSError:
                    if os.path.exists(exit_file):
                        # make sure it doesn't exist
                        raise
            else:
                self.settings.pop('PORTAGE_EBUILD_EXIT_FILE', None)

        SpawnProcess._start(self)

    def _init_ipc_fifos(self):

        input_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'], '.ipc_in')
        output_fifo = os.path.join(self.settings['PORTAGE_BUILDDIR'],
                                   '.ipc_out')

        for p in (input_fifo, output_fifo):

            st = None
            try:
                st = os.lstat(p)
            except OSError:
                os.mkfifo(p)
            else:
                if not stat.S_ISFIFO(st.st_mode):
                    st = None
                    try:
                        os.unlink(p)
                    except OSError:
                        pass
                    os.mkfifo(p)

            apply_secpass_permissions(p,
                                      uid=os.getuid(),
                                      gid=portage.data.portage_gid,
                                      mode=0o770,
                                      stat_cached=st)

        return (input_fifo, output_fifo)

    def _start_ipc_daemon(self):
        self._exit_command = ExitCommand()
        self._exit_command.reply_hook = self._exit_command_callback
        query_command = QueryCommand(self.settings)
        commands = {
            'best_version': query_command,
            'exit': self._exit_command,
            'has_version': query_command,
        }
        input_fifo, output_fifo = self._init_ipc_fifos()
        self._ipc_daemon = EbuildIpcDaemon(commands=commands,
                                           input_fifo=input_fifo,
                                           output_fifo=output_fifo,
                                           scheduler=self.scheduler)
        self._ipc_daemon.start()

    def _exit_command_callback(self):
        if self._registered:
            # Let the process exit naturally, if possible.
            self.scheduler.schedule(self._reg_id, timeout=self._exit_timeout)
            if self._registered:
                # If it doesn't exit naturally in a reasonable amount
                # of time, kill it (solves bug #278895). We try to avoid
                # this when possible since it makes sandbox complain about
                # being killed by a signal.
                self.cancel()

    def _orphan_process_warn(self):
        phase = self.phase

        msg = _("The ebuild phase '%s' with pid %s appears "
                "to have left an orphan process running in the "
                "background.") % (phase, self.pid)

        self._eerror(textwrap.wrap(msg, 72))

    def _pipe(self, fd_pipes):
        stdout_pipe = fd_pipes.get(1)
        got_pty, master_fd, slave_fd = \
         _create_pty_or_pipe(copy_term_size=stdout_pipe)
        return (master_fd, slave_fd)

    def _can_log(self, slave_fd):
        # With sesandbox, logging works through a pty but not through a
        # normal pipe. So, disable logging if ptys are broken.
        # See Bug #162404.
        # TODO: Add support for logging via named pipe (fifo) with
        # sesandbox, since EbuildIpcDaemon uses a fifo and it's known
        # to be compatible with sesandbox.
        return not ('sesandbox' in self.settings.features \
         and self.settings.selinux_enabled()) or os.isatty(slave_fd)

    def _unexpected_exit(self):

        phase = self.phase

        msg = _(
            "The ebuild phase '%s' has exited "
            "unexpectedly. This type of behavior "
            "is known to be triggered "
            "by things such as failed variable "
            "assignments (bug #190128) or bad substitution "
            "errors (bug #200313). Normally, before exiting, bash should "
            "have displayed an error message above. If bash did not "
            "produce an error message above, it's possible "
            "that the ebuild has called `exit` when it "
            "should have called `die` instead. This behavior may also "
            "be triggered by a corrupt bash binary or a hardware "
            "problem such as memory or cpu malfunction. If the problem is not "
            "reproducible or it appears to occur randomly, then it is likely "
            "to be triggered by a hardware problem. "
            "If you suspect a hardware problem then you should "
            "try some basic hardware diagnostics such as memtest. "
            "Please do not report this as a bug unless it is consistently "
            "reproducible and you are sure that your bash binary and hardware "
            "are functioning properly.") % phase

        self._eerror(textwrap.wrap(msg, 72))

    def _eerror(self, lines):
        out = StringIO()
        phase = self.phase
        for line in lines:
            eerror(line, phase=phase, key=self.settings.mycpv, out=out)
        msg = _unicode_decode(out.getvalue(),
                              encoding=_encodings['content'],
                              errors='replace')
        if msg:
            self.scheduler.output(
                msg, log_path=self.settings.get("PORTAGE_LOG_FILE"))

    def _set_returncode(self, wait_retval):
        SpawnProcess._set_returncode(self, wait_retval)

        if self._ipc_daemon is not None:
            self._ipc_daemon.cancel()
            if self._exit_command.exitcode is not None:
                self.returncode = self._exit_command.exitcode
            else:
                self.returncode = 1
                self._unexpected_exit()
        else:
            exit_file = self.settings.get('PORTAGE_EBUILD_EXIT_FILE')
            if exit_file and not os.path.exists(exit_file):
                self.returncode = 1
                self._unexpected_exit()
Пример #14
0
    def testIpcDaemon(self):
        tmpdir = tempfile.mkdtemp()
        build_dir = None
        try:
            env = {}

            # Pass along PORTAGE_USERNAME and PORTAGE_GRPNAME since they
            # need to be inherited by ebuild subprocesses.
            if 'PORTAGE_USERNAME' in os.environ:
                env['PORTAGE_USERNAME'] = os.environ['PORTAGE_USERNAME']
            if 'PORTAGE_GRPNAME' in os.environ:
                env['PORTAGE_GRPNAME'] = os.environ['PORTAGE_GRPNAME']

            env['PORTAGE_PYTHON'] = _python_interpreter
            env['PORTAGE_BIN_PATH'] = PORTAGE_BIN_PATH
            env['PORTAGE_PYM_PATH'] = PORTAGE_PYM_PATH
            env['PORTAGE_BUILDDIR'] = os.path.join(tmpdir, 'cat', 'pkg-1')

            if "__PORTAGE_TEST_HARDLINK_LOCKS" in os.environ:
                env["__PORTAGE_TEST_HARDLINK_LOCKS"] = \
                 os.environ["__PORTAGE_TEST_HARDLINK_LOCKS"]

            task_scheduler = TaskScheduler(max_jobs=2)
            build_dir = EbuildBuildDir(scheduler=task_scheduler.sched_iface,
                                       settings=env)
            build_dir.lock()
            ensure_dirs(env['PORTAGE_BUILDDIR'])

            input_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_in')
            output_fifo = os.path.join(env['PORTAGE_BUILDDIR'], '.ipc_out')
            os.mkfifo(input_fifo)
            os.mkfifo(output_fifo)

            for exitcode in (0, 1, 2):
                exit_command = ExitCommand()
                commands = {'exit': exit_command}
                daemon = EbuildIpcDaemon(commands=commands,
                                         input_fifo=input_fifo,
                                         output_fifo=output_fifo,
                                         scheduler=task_scheduler.sched_iface)
                proc = SpawnProcess(args=[
                    BASH_BINARY, "-c",
                    '"$PORTAGE_BIN_PATH"/ebuild-ipc exit %d' % exitcode
                ],
                                    env=env,
                                    scheduler=task_scheduler.sched_iface)

                self.received_command = False

                def exit_command_callback():
                    self.received_command = True
                    task_scheduler.clear()
                    task_scheduler.wait()

                exit_command.reply_hook = exit_command_callback
                start_time = time.time()
                task_scheduler.add(daemon)
                task_scheduler.add(proc)
                task_scheduler.run(timeout=self._SCHEDULE_TIMEOUT)
                task_scheduler.clear()
                task_scheduler.wait()
                hardlock_cleanup(env['PORTAGE_BUILDDIR'],
                                 remove_all_locks=True)

                self.assertEqual(self.received_command, True,
                 "command not received after %d seconds" % \
                 (time.time() - start_time,))
                self.assertEqual(proc.isAlive(), False)
                self.assertEqual(daemon.isAlive(), False)
                self.assertEqual(exit_command.exitcode, exitcode)

            # Intentionally short timeout test for QueueScheduler.run()
            sleep_time_s = 10  # 10.000 seconds
            short_timeout_ms = 10  #  0.010 seconds

            for i in range(3):
                exit_command = ExitCommand()
                commands = {'exit': exit_command}
                daemon = EbuildIpcDaemon(commands=commands,
                                         input_fifo=input_fifo,
                                         output_fifo=output_fifo,
                                         scheduler=task_scheduler.sched_iface)
                proc = SpawnProcess(
                    args=[BASH_BINARY, "-c",
                          'exec sleep %d' % sleep_time_s],
                    env=env,
                    scheduler=task_scheduler.sched_iface)

                self.received_command = False

                def exit_command_callback():
                    self.received_command = True
                    task_scheduler.clear()
                    task_scheduler.wait()

                exit_command.reply_hook = exit_command_callback
                start_time = time.time()
                task_scheduler.add(daemon)
                task_scheduler.add(proc)
                task_scheduler.run(timeout=short_timeout_ms)
                task_scheduler.clear()
                task_scheduler.wait()
                hardlock_cleanup(env['PORTAGE_BUILDDIR'],
                                 remove_all_locks=True)

                self.assertEqual(self.received_command, False,
                 "command received after %d seconds" % \
                 (time.time() - start_time,))
                self.assertEqual(proc.isAlive(), False)
                self.assertEqual(daemon.isAlive(), False)
                self.assertEqual(proc.returncode == os.EX_OK, False)

        finally:
            if build_dir is not None:
                build_dir.unlock()
            shutil.rmtree(tmpdir)