def test_utc_now():
"""
Test construction of a UTC date and time.
"""
from time import sleep
from nose.tools import assert_true
from cargo.temporal import utc_now
then = utc_now()
sleep(2.0)
assert_true(utc_now() > then)
def test_utc_now():
"""
Test construction of a UTC date and time.
"""
from time import sleep
from nose.tools import assert_true
from cargo.temporal import utc_now
then = utc_now()
sleep(2.0)
assert_true(utc_now() > then)
def add_disk_handler(prefix, level=logging.NOTSET):
"""
Enable typical logging to disk.
"""
# generate an unused log file path
from os.path import lexists
from itertools import count
for i in count():
path = "%s.%i" % (prefix, i)
if not lexists(path):
break
# build a handler
from cargo.temporal import utc_now
handler = FileHandler(path, encoding="utf-8")
handler.setFormatter(VerboseFileFormatter())
handler.setLevel(level)
# add it
logging.root.addHandler(handler)
log.debug("added log handler for file %s at %s", path, utc_now())
return handler
def add_console_handler(level=logging.NOTSET, verbose=True):
"""
Enable typical logging to the console.
"""
# get the appropriate formatter
if verbose:
formatter = TTY_VerboseFormatter
else:
formatter = TTY_ConciseFormatter
# build a handler
from sys import stdout
from cargo.temporal import utc_now
handler = StreamHandler(stdout)
handler.setFormatter(formatter(stdout))
handler.setLevel(level)
# add it
logging.root.addHandler(handler)
log.debug("added log handler for console at %s", utc_now())
return handler
def add_disk_handler(prefix, level=logging.NOTSET):
"""
Enable typical logging to disk.
"""
# generate an unused log file path
from os.path import lexists
from itertools import count
for i in count():
path = "%s.%i" % (prefix, i)
if not lexists(path):
break
# build a handler
from cargo.temporal import utc_now
handler = FileHandler(path, encoding="utf-8")
handler.setFormatter(VerboseFileFormatter())
handler.setLevel(level)
# add it
logging.root.addHandler(handler)
log.debug("added log handler for file %s at %s", path, utc_now())
return handler
def add_console_handler(level=logging.NOTSET, verbose=True):
"""
Enable typical logging to the console.
"""
# get the appropriate formatter
if verbose:
formatter = TTY_VerboseFormatter
else:
formatter = TTY_ConciseFormatter
# build a handler
from sys import stdout
from cargo.temporal import utc_now
handler = StreamHandler(stdout)
handler.setFormatter(formatter(stdout))
handler.setLevel(level)
# add it
logging.root.addHandler(handler)
log.debug("added log handler for console at %s", utc_now())
return handler
def run_cpu_limited(
arguments,
limit,
pty = True,
environment = {},
resolution = 0.5,
):
"""
Spawn a subprocess whose process tree is granted limited CPU (user) time.
@param environment Override specific existing environment variables.
The subprocess must not expect input. This method is best suited to
processes which may run for a reasonable amount of time (eg, at least
several seconds); it will be fairly inefficient (and ineffective) at
fine-grained limiting of CPU allocation to short-duration processes.
We run the process and read its output. Every time we receive a chunk of
data, or every C{resolution} seconds, we estimate the total CPU time used
by the session---and store that information with the chunk of output, if
any. After at least C{limit} of CPU time has been used by the spawned
session, or after the session leader terminates, whichever is first, the
session is (sig)killed, the session leader waited on, and any data
remaining in the pipe is read.
Note that the use of SIGKILL means that child processes *cannot* perform
their own cleanup.
If C{pty} is specified, process stdout is piped through a pty, which makes
process output less likely to be buffered. This behavior is the default.
Kernel-reported resource usage includes the sum of all directly and
indirectly waited-on children. It will be accurate in the common case where
processes terminate after correctly waiting on their children, and
inaccurate in cases where zombies are reparented to init. Elapsed CPU time
taken from the /proc accounting mechanism is used to do CPU time limiting,
and will always be at least the specified limit.
"""
log.detail("running %s for %s", arguments, limit)
# sanity
if not arguments:
raise ArgumentError()
# start the run
from cargo.temporal import utc_now
popened = None
fd_chunks = {}
exit_pid = None
started = utc_now()
try:
# start running the child process
if pty:
popened = spawn_pty_session(arguments, environment)
else:
popened = spawn_pipe_session(arguments, environment)
fd_chunks = {
popened.stdout.fileno(): [],
popened.stderr.fileno(): [],
}
log.debug("spawned child with pid %i", popened.pid)
# read the child's output while accounting (note that the session id
# is, under Linux, the pid of the session leader)
accountant = SessionTimeAccountant(popened.pid)
reader = PollingReader(fd_chunks.keys())
while reader.fds:
# nuke if we're past cutoff
if accountant.total >= limit:
popened.kill()
break
# read from and audit the child process
(chunk_fd, chunk) = reader.read(resolution)
accountant.audit()
if chunk is not None:
log.debug(
"got %i bytes at %s (user time) on fd %i; chunk follows:\n%s",
len(chunk),
accountant.total,
chunk_fd,
chunk,
)
if chunk != "":
fd_chunks[chunk_fd].append((accountant.total, chunk))
else:
reader.unregister([chunk_fd])
# wait for our child to die
(exit_pid, termination, usage) = os.wait4(popened.pid, 0)
# nuke the session from orbit (it's the only way to be sure)
kill_session(popened.pid, signal.SIGKILL)
except:
# something has gone awry, so we need to kill our children
log.warning("something went awry! (our pid is %i)", os.getpid())
raised = Raised()
if exit_pid is None and popened is not None:
try:
# nuke the entire session
kill_session(popened.pid, signal.SIGKILL)
# and don't leave the child as a zombie
os.waitpid(popened.pid, 0)
except:
Raised().print_ignored()
raised.re_raise()
else:
# grab any output left in the kernel buffers
while reader.fds:
(chunk_fd, chunk) = reader.read(128.0)
if chunk:
fd_chunks[chunk_fd].append((accountant.total, chunk))
elif chunk_fd:
reader.unregister([chunk_fd])
else:
raise RuntimeError("final read from child timed out; undead child?")
# done
from datetime import timedelta
return \
CPU_LimitedRun(
started,
limit,
fd_chunks[popened.stdout.fileno()],
fd_chunks[popened.stderr.fileno()],
timedelta(seconds = usage.ru_utime),
accountant.total,
os.WEXITSTATUS(termination) if os.WIFEXITED(termination) else None,
os.WTERMSIG(termination) if os.WIFSIGNALED(termination) else None,
)
finally:
# let's not leak file descriptors
if popened is not None:
popened.stdout.close()
popened.stderr.close()
def run_cpu_limited(
arguments,
limit,
pty=True,
environment={},
resolution=0.5,
):
"""
Spawn a subprocess whose process tree is granted limited CPU (user) time.
@param environment Override specific existing environment variables.
The subprocess must not expect input. This method is best suited to
processes which may run for a reasonable amount of time (eg, at least
several seconds); it will be fairly inefficient (and ineffective) at
fine-grained limiting of CPU allocation to short-duration processes.
We run the process and read its output. Every time we receive a chunk of
data, or every C{resolution} seconds, we estimate the total CPU time used
by the session---and store that information with the chunk of output, if
any. After at least C{limit} of CPU time has been used by the spawned
session, or after the session leader terminates, whichever is first, the
session is (sig)killed, the session leader waited on, and any data
remaining in the pipe is read.
Note that the use of SIGKILL means that child processes *cannot* perform
their own cleanup.
If C{pty} is specified, process stdout is piped through a pty, which makes
process output less likely to be buffered. This behavior is the default.
Kernel-reported resource usage includes the sum of all directly and
indirectly waited-on children. It will be accurate in the common case where
processes terminate after correctly waiting on their children, and
inaccurate in cases where zombies are reparented to init. Elapsed CPU time
taken from the /proc accounting mechanism is used to do CPU time limiting,
and will always be at least the specified limit.
"""
log.detail("running %s for %s", arguments, limit)
# sanity
if not arguments:
raise ArgumentError()
# start the run
from cargo.temporal import utc_now
popened = None
fd_chunks = {}
exit_pid = None
started = utc_now()
try:
# start running the child process
if pty:
popened = spawn_pty_session(arguments, environment)
else:
popened = spawn_pipe_session(arguments, environment)
fd_chunks = {
popened.stdout.fileno(): [],
popened.stderr.fileno(): [],
}
log.debug("spawned child with pid %i", popened.pid)
# read the child's output while accounting (note that the session id
# is, under Linux, the pid of the session leader)
accountant = SessionTimeAccountant(popened.pid)
reader = PollingReader(fd_chunks.keys())
while reader.fds:
# nuke if we're past cutoff
if accountant.total >= limit:
popened.kill()
break
# read from and audit the child process
(chunk_fd, chunk) = reader.read(resolution)
accountant.audit()
if chunk is not None:
log.debug(
"got %i bytes at %s (user time) on fd %i; chunk follows:\n%s",
len(chunk),
accountant.total,
chunk_fd,
chunk,
)
if chunk != "":
fd_chunks[chunk_fd].append((accountant.total, chunk))
else:
reader.unregister([chunk_fd])
# wait for our child to die
(exit_pid, termination, usage) = os.wait4(popened.pid, 0)
# nuke the session from orbit (it's the only way to be sure)
kill_session(popened.pid, signal.SIGKILL)
except:
# something has gone awry, so we need to kill our children
log.warning("something went awry! (our pid is %i)", os.getpid())
raised = Raised()
if exit_pid is None and popened is not None:
try:
# nuke the entire session
kill_session(popened.pid, signal.SIGKILL)
# and don't leave the child as a zombie
os.waitpid(popened.pid, 0)
except:
Raised().print_ignored()
raised.re_raise()
else:
# grab any output left in the kernel buffers
while reader.fds:
(chunk_fd, chunk) = reader.read(128.0)
if chunk:
fd_chunks[chunk_fd].append((accountant.total, chunk))
elif chunk_fd:
reader.unregister([chunk_fd])
else:
raise RuntimeError(
"final read from child timed out; undead child?")
# done
from datetime import timedelta
return \
CPU_LimitedRun(
started,
limit,
fd_chunks[popened.stdout.fileno()],
fd_chunks[popened.stderr.fileno()],
timedelta(seconds = usage.ru_utime),
accountant.total,
os.WEXITSTATUS(termination) if os.WIFEXITED(termination) else None,
os.WTERMSIG(termination) if os.WIFSIGNALED(termination) else None,
)
finally:
# let's not leak file descriptors
if popened is not None:
popened.stdout.close()
popened.stderr.close()
