Esempio n. 1
0
    def __exit__(self, *args):
        """
        Leave context.
        """

        try:
            for engine in self.engines.itervalues():
                engine.dispose()
        except:
            Raised().print_ignored()
Esempio n. 2
0
def call_capturing(arguments, input=None, preexec_fn=None):
    """
    Spawn a process and return its output and status code.
    """

    popened = None

    try:
        # launch the subprocess
        import subprocess

        from subprocess import Popen

        popened = \
            Popen(
                arguments,
                stdin      = subprocess.PIPE,
                stdout     = subprocess.PIPE,
                stderr     = subprocess.PIPE,
                preexec_fn = preexec_fn,
                )

        # wait for its natural death
        (stdout, stderr) = popened.communicate(input)
    except:
        raised = Raised()

        if popened is not None and popened.poll() is None:
            try:
                popened.kill()
                popened.wait()
            except:
                Raised().print_ignored()

        raised.re_raise()
    else:
        return (stdout, stderr, popened.returncode)
Esempio n. 3
0
def call_capturing(arguments, input = None, preexec_fn = None):
    """
    Spawn a process and return its output and status code.
    """

    popened = None

    try:
        # launch the subprocess
        import subprocess

        from subprocess import Popen

        popened = \
            Popen(
                arguments,
                stdin      = subprocess.PIPE,
                stdout     = subprocess.PIPE,
                stderr     = subprocess.PIPE,
                preexec_fn = preexec_fn,
                )

        # wait for its natural death
        (stdout, stderr) = popened.communicate(input)
    except:
        raised = Raised()

        if popened is not None and popened.poll() is None:
            try:
                popened.kill()
                popened.wait()
            except:
                Raised().print_ignored()

        raised.re_raise()
    else:
        return (stdout, stderr, popened.returncode)
Esempio n. 4
0
def spawn_pty_session(arguments, environment = {}):
    """Spawn a subprocess in its own session, with stdout routed through a pty."""

    # build a pty
    (master_fd, slave_fd) = pty.openpty()

    log.debug("opened pty %s", os.ttyname(slave_fd))

    # launch the subprocess
    try:
        popened        = \
            subprocess.Popen(
                arguments,
                close_fds  = True,
                stdin      = slave_fd,
                stdout     = slave_fd,
                stderr     = subprocess.PIPE,
                preexec_fn = lambda: _child_preexec(environment),
                )
        popened.stdout = os.fdopen(master_fd)

        os.close(slave_fd)

        return popened
    except:
        raised = Raised()

        try:
            if master_fd is not None:
                os.close(master_fd)
            if slave_fd is not None:
                os.close(slave_fd)
        except:
            Raised().print_ignored()

        raised.re_raise()
Esempio n. 5
0
def spawn_pty_session(arguments, environment={}):
    """Spawn a subprocess in its own session, with stdout routed through a pty."""

    # build a pty
    (master_fd, slave_fd) = pty.openpty()

    log.debug("opened pty %s", os.ttyname(slave_fd))

    # launch the subprocess
    try:
        popened        = \
            subprocess.Popen(
                arguments,
                close_fds  = True,
                stdin      = slave_fd,
                stdout     = slave_fd,
                stderr     = subprocess.PIPE,
                preexec_fn = lambda: _child_preexec(environment),
                )
        popened.stdout = os.fdopen(master_fd)

        os.close(slave_fd)

        return popened
    except:
        raised = Raised()

        try:
            if master_fd is not None:
                os.close(master_fd)
            if slave_fd is not None:
                os.close(slave_fd)
        except:
            Raised().print_ignored()

        raised.re_raise()
Esempio n. 6
0
def run_cpu_limited(
    arguments,
    limit,
    pty         = True,
    environment = {},
    resolution  = 0.5,
    ):
    """
    Spawn a subprocess whose process tree is granted limited CPU (user) time.

    @param environment Override specific existing environment variables.

    The subprocess must not expect input. This method is best suited to
    processes which may run for a reasonable amount of time (eg, at least
    several seconds); it will be fairly inefficient (and ineffective) at
    fine-grained limiting of CPU allocation to short-duration processes.

    We run the process and read its output. Every time we receive a chunk of
    data, or every C{resolution} seconds, we estimate the total CPU time used
    by the session---and store that information with the chunk of output, if
    any. After at least C{limit} of CPU time has been used by the spawned
    session, or after the session leader terminates, whichever is first, the
    session is (sig)killed, the session leader waited on, and any data
    remaining in the pipe is read.

    Note that the use of SIGKILL means that child processes *cannot* perform
    their own cleanup.

    If C{pty} is specified, process stdout is piped through a pty, which makes
    process output less likely to be buffered. This behavior is the default.

    Kernel-reported resource usage includes the sum of all directly and
    indirectly waited-on children. It will be accurate in the common case where
    processes terminate after correctly waiting on their children, and
    inaccurate in cases where zombies are reparented to init. Elapsed CPU time
    taken from the /proc accounting mechanism is used to do CPU time limiting,
    and will always be at least the specified limit.
    """

    log.detail("running %s for %s", arguments, limit)

    # sanity
    if not arguments:
        raise ArgumentError()

    # start the run
    from cargo.temporal import utc_now

    popened   = None
    fd_chunks = {}
    exit_pid  = None
    started   = utc_now()

    try:
        # start running the child process
        if pty:
            popened = spawn_pty_session(arguments, environment)
        else:
            popened = spawn_pipe_session(arguments, environment)

        fd_chunks = {
            popened.stdout.fileno(): [],
            popened.stderr.fileno(): [],
            }

        log.debug("spawned child with pid %i", popened.pid)

        # read the child's output while accounting (note that the session id
        # is, under Linux, the pid of the session leader)
        accountant = SessionTimeAccountant(popened.pid)
        reader     = PollingReader(fd_chunks.keys())

        while reader.fds:
            # nuke if we're past cutoff
            if accountant.total >= limit:
                popened.kill()

                break

            # read from and audit the child process
            (chunk_fd, chunk) = reader.read(resolution)

            accountant.audit()

            if chunk is not None:
                log.debug(
                    "got %i bytes at %s (user time) on fd %i; chunk follows:\n%s",
                    len(chunk),
                    accountant.total,
                    chunk_fd,
                    chunk,
                    )

                if chunk != "":
                    fd_chunks[chunk_fd].append((accountant.total, chunk))
                else:
                    reader.unregister([chunk_fd])

        # wait for our child to die
        (exit_pid, termination, usage) = os.wait4(popened.pid, 0)

        # nuke the session from orbit (it's the only way to be sure)
        kill_session(popened.pid, signal.SIGKILL)
    except:
        # something has gone awry, so we need to kill our children
        log.warning("something went awry! (our pid is %i)", os.getpid())

        raised = Raised()

        if exit_pid is None and popened is not None:
            try:
                # nuke the entire session
                kill_session(popened.pid, signal.SIGKILL)

                # and don't leave the child as a zombie
                os.waitpid(popened.pid, 0)
            except:
                Raised().print_ignored()

        raised.re_raise()
    else:
        # grab any output left in the kernel buffers
        while reader.fds:
            (chunk_fd, chunk) = reader.read(128.0)

            if chunk:
                fd_chunks[chunk_fd].append((accountant.total, chunk))
            elif chunk_fd:
                reader.unregister([chunk_fd])
            else:
                raise RuntimeError("final read from child timed out; undead child?")

        # done
        from datetime import timedelta

        return \
            CPU_LimitedRun(
                started,
                limit,
                fd_chunks[popened.stdout.fileno()],
                fd_chunks[popened.stderr.fileno()],
                timedelta(seconds = usage.ru_utime),
                accountant.total,
                os.WEXITSTATUS(termination) if os.WIFEXITED(termination) else None,
                os.WTERMSIG(termination) if os.WIFSIGNALED(termination) else None,
                )
    finally:
        # let's not leak file descriptors
        if popened is not None:
            popened.stdout.close()
            popened.stderr.close()
Esempio n. 7
0
def run_cpu_limited(
    arguments,
    limit,
    pty=True,
    environment={},
    resolution=0.5,
):
    """
    Spawn a subprocess whose process tree is granted limited CPU (user) time.

    @param environment Override specific existing environment variables.

    The subprocess must not expect input. This method is best suited to
    processes which may run for a reasonable amount of time (eg, at least
    several seconds); it will be fairly inefficient (and ineffective) at
    fine-grained limiting of CPU allocation to short-duration processes.

    We run the process and read its output. Every time we receive a chunk of
    data, or every C{resolution} seconds, we estimate the total CPU time used
    by the session---and store that information with the chunk of output, if
    any. After at least C{limit} of CPU time has been used by the spawned
    session, or after the session leader terminates, whichever is first, the
    session is (sig)killed, the session leader waited on, and any data
    remaining in the pipe is read.

    Note that the use of SIGKILL means that child processes *cannot* perform
    their own cleanup.

    If C{pty} is specified, process stdout is piped through a pty, which makes
    process output less likely to be buffered. This behavior is the default.

    Kernel-reported resource usage includes the sum of all directly and
    indirectly waited-on children. It will be accurate in the common case where
    processes terminate after correctly waiting on their children, and
    inaccurate in cases where zombies are reparented to init. Elapsed CPU time
    taken from the /proc accounting mechanism is used to do CPU time limiting,
    and will always be at least the specified limit.
    """

    log.detail("running %s for %s", arguments, limit)

    # sanity
    if not arguments:
        raise ArgumentError()

    # start the run
    from cargo.temporal import utc_now

    popened = None
    fd_chunks = {}
    exit_pid = None
    started = utc_now()

    try:
        # start running the child process
        if pty:
            popened = spawn_pty_session(arguments, environment)
        else:
            popened = spawn_pipe_session(arguments, environment)

        fd_chunks = {
            popened.stdout.fileno(): [],
            popened.stderr.fileno(): [],
        }

        log.debug("spawned child with pid %i", popened.pid)

        # read the child's output while accounting (note that the session id
        # is, under Linux, the pid of the session leader)
        accountant = SessionTimeAccountant(popened.pid)
        reader = PollingReader(fd_chunks.keys())

        while reader.fds:
            # nuke if we're past cutoff
            if accountant.total >= limit:
                popened.kill()

                break

            # read from and audit the child process
            (chunk_fd, chunk) = reader.read(resolution)

            accountant.audit()

            if chunk is not None:
                log.debug(
                    "got %i bytes at %s (user time) on fd %i; chunk follows:\n%s",
                    len(chunk),
                    accountant.total,
                    chunk_fd,
                    chunk,
                )

                if chunk != "":
                    fd_chunks[chunk_fd].append((accountant.total, chunk))
                else:
                    reader.unregister([chunk_fd])

        # wait for our child to die
        (exit_pid, termination, usage) = os.wait4(popened.pid, 0)

        # nuke the session from orbit (it's the only way to be sure)
        kill_session(popened.pid, signal.SIGKILL)
    except:
        # something has gone awry, so we need to kill our children
        log.warning("something went awry! (our pid is %i)", os.getpid())

        raised = Raised()

        if exit_pid is None and popened is not None:
            try:
                # nuke the entire session
                kill_session(popened.pid, signal.SIGKILL)

                # and don't leave the child as a zombie
                os.waitpid(popened.pid, 0)
            except:
                Raised().print_ignored()

        raised.re_raise()
    else:
        # grab any output left in the kernel buffers
        while reader.fds:
            (chunk_fd, chunk) = reader.read(128.0)

            if chunk:
                fd_chunks[chunk_fd].append((accountant.total, chunk))
            elif chunk_fd:
                reader.unregister([chunk_fd])
            else:
                raise RuntimeError(
                    "final read from child timed out; undead child?")

        # done
        from datetime import timedelta

        return \
            CPU_LimitedRun(
                started,
                limit,
                fd_chunks[popened.stdout.fileno()],
                fd_chunks[popened.stderr.fileno()],
                timedelta(seconds = usage.ru_utime),
                accountant.total,
                os.WEXITSTATUS(termination) if os.WIFEXITED(termination) else None,
                os.WTERMSIG(termination) if os.WIFSIGNALED(termination) else None,
                )
    finally:
        # let's not leak file descriptors
        if popened is not None:
            popened.stdout.close()
            popened.stderr.close()