def RunTests(tests,
jobs=1,
chroot_available=True,
network=False,
dryrun=False,
failfast=False):
"""Execute |paths| with |jobs| in parallel (including |network| tests).
Args:
tests: The tests to run.
jobs: How many tests to run in parallel.
chroot_available: Whether we can run tests inside the sdk.
network: Whether to run network based tests.
dryrun: Do everything but execute the test.
failfast: Stop on first failure
Returns:
True if all tests pass, else False.
"""
finished = multiprocessing.Value('i')
testsets = []
pids = []
failed = aborted = False
def WaitOne():
(pid, status) = os.wait()
pids.remove(pid)
return status
# Launch all the tests!
try:
# Build up the testsets.
testsets = BuildTestSets(tests, chroot_available, network, jobs=jobs)
# Fork each test and add it to the list.
for test, cmd, tmpfile in testsets:
if failed and failfast:
logging.error('failure detected; stopping new tests')
break
if len(pids) >= jobs:
if WaitOne():
failed = True
pid = os.fork()
if pid == 0:
proctitle.settitle(test)
ret = 1
try:
if dryrun:
logging.info('Would have run: %s',
cros_build_lib.CmdToStr(cmd))
ret = 0
else:
ret = RunTest(test, cmd, tmpfile, finished,
len(testsets))
except KeyboardInterrupt:
pass
except BaseException:
logging.error('%s failed', test, exc_info=True)
# We cannot run clean up hooks in the child because it'll break down
# things like tempdir context managers.
os._exit(ret) # pylint: disable=protected-access
pids.append(pid)
# Wait for all of them to get cleaned up.
while pids:
if WaitOne():
failed = True
except KeyboardInterrupt:
# If the user wants to stop, reap all the pending children.
logging.warning('CTRL+C received; cleaning up tests')
aborted = True
CleanupChildren(pids)
# Walk through the results.
failed_tests = []
for test, cmd, tmpfile in testsets:
tmpfile.seek(0)
output = tmpfile.read()
if output:
failed_tests.append(test)
print()
logging.error('### LOG: %s', test)
print(output.rstrip())
print()
if failed_tests:
logging.error('The following %i tests failed:\n %s',
len(failed_tests), '\n '.join(sorted(failed_tests)))
return False
elif aborted or failed:
return False
return True
def run(self):
"""Runs the test in a proper environment (e.g. qemu)."""
# We know these pre-tests are fast (especially if they've already been run
# once), so run them automatically for the user if they test by hand.
self.pre_test()
paths_to_mount = (list(self._BIND_MOUNT_PATHS) + [
mount for mount in self._BIND_MOUNT_IF_NOT_SYMLINK_PATHS
if not os.path.islink('/' + mount)
])
for mount in paths_to_mount:
path = os.path.join(self.sysroot, mount)
osutils.SafeMakedirs(path)
osutils.Mount('/' + mount, path, 'none', osutils.MS_BIND)
positive_filters = self.gtest_filter[0]
negative_filters = self.gtest_filter[1]
if self.user_gtest_filter:
positive_filters += self.user_gtest_filter[0]
negative_filters += self.user_gtest_filter[1]
filters = (':'.join(positive_filters), ':'.join(negative_filters))
gtest_filter = '%s-%s' % filters
cmd = self.removeSysrootPrefix(self.bin)
argv = self.args[:]
argv[0] = self.removeSysrootPrefix(argv[0])
if gtest_filter != '-':
argv.append('--gtest_filter=' + gtest_filter)
# Some programs expect to find data files via $CWD, so doing a chroot
# and dropping them into / would make them fail.
cwd = self.removeSysrootPrefix(os.getcwd())
# Make orphaned child processes reparent to this process instead of the init
# process. This allows us to kill them if they do not terminate after the
# test has finished running.
_MakeProcessSubreaper()
# Fork off a child to run the test. This way we can make tweaks to the
# env that only affect the child (gid/uid/chroot/cwd/etc...). We have
# to fork anyways to run the test, so might as well do it all ourselves
# to avoid (slow) chaining through programs like:
# sudo -u $SUDO_UID -g $SUDO_GID chroot $SYSROOT bash -c 'cd $CWD; $BIN'
child = os.fork()
if child == 0:
print('chroot: %s' % self.sysroot)
print('cwd: %s' % cwd)
print('cmd: {%s} %s' % (cmd, ' '.join(map(repr, argv))))
os.chroot(self.sysroot)
os.chdir(cwd)
# Set the child's pgid to its pid, so we can kill any processes that the
# child creates after the child terminates.
os.setpgid(0, 0)
# Remove sysroot from path environment variables.
for var in ('OUT', 'SRC', 'T'):
if var in os.environ:
os.environ[var] = self.removeSysrootPrefix(os.environ[var])
# The TERM the user is leveraging might not exist in the sysroot.
# Force a sane default that supports standard color sequences.
os.environ['TERM'] = 'ansi'
# Some progs want this like bash else they get super confused.
os.environ['PWD'] = cwd
os.environ['GTEST_COLOR'] = 'yes'
if not self.run_as_root:
user, uid, gid, home = self.GetNonRootAccount()
os.setgid(gid)
os.setuid(uid)
os.environ['HOME'] = home
os.environ['USER'] = user
sys.exit(os.execvp(cmd, argv))
proctitle.settitle('sysroot watcher', cmd)
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Reap any processes that were reparented to us until the child exits.
status = _ReapUntilProcessExits(child)
leaked_children = psutil.Process().get_children(recursive=True)
if leaked_children:
# It's possible the child forked and the forked processes are still
# running. Kill the forked processes.
try:
os.killpg(child, signal.SIGTERM)
except OSError as e:
if e.errno != errno.ESRCH:
print(
'Warning: while trying to kill pgid %s caught exception\n%s'
% (child, e),
file=sys.stderr)
# Kill any orphaned processes originally created by the test that were in
# a different process group. This will also kill any processes that did
# not respond to the SIGTERM.
for child in leaked_children:
try:
child.kill()
except psutil.NoSuchProcess:
pass
failmsg = None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
failmsg = 'signal %s(%i)' % (signals.StrSignal(sig), sig)
else:
exit_status = os.WEXITSTATUS(status)
if exit_status:
failmsg = 'exit code %i' % exit_status
if failmsg:
print('Error: %s: failed with %s' % (cmd, failmsg),
file=sys.stderr)
if leaked_children:
for p in leaked_children:
print(
'Error: the test leaked process %s with pid %s (it was forcefully'
' killed)' % (p.name(), p.pid),
file=sys.stderr)
# TODO(vapier): Make this an error. We need to track down some scenarios
# where processes do leak though before we can make this fatal :(.
#sys.exit(100)
process_util.ExitAsStatus(status)
def RunTests(tests, jobs=1, chroot_available=True, network=False, dryrun=False,
failfast=False):
"""Execute |paths| with |jobs| in parallel (including |network| tests).
Args:
tests: The tests to run.
jobs: How many tests to run in parallel.
chroot_available: Whether we can run tests inside the sdk.
network: Whether to run network based tests.
dryrun: Do everything but execute the test.
failfast: Stop on first failure
Returns:
True if all tests pass, else False.
"""
finished = multiprocessing.Value('i')
testsets = []
pids = []
failed = aborted = False
def WaitOne():
(pid, status) = os.wait()
pids.remove(pid)
return status
# Launch all the tests!
try:
# Build up the testsets.
testsets = BuildTestSets(tests, chroot_available, network)
# Fork each test and add it to the list.
for test, cmd, tmpfile in testsets:
if failed and failfast:
logging.error('failure detected; stopping new tests')
break
if len(pids) >= jobs:
if WaitOne():
failed = True
pid = os.fork()
if pid == 0:
proctitle.settitle(test)
ret = 1
try:
if dryrun:
logging.info('Would have run: %s', cros_build_lib.CmdToStr(cmd))
ret = 0
else:
ret = RunTest(test, cmd, tmpfile, finished, len(testsets))
except KeyboardInterrupt:
pass
except BaseException:
logging.error('%s failed', test, exc_info=True)
# We cannot run clean up hooks in the child because it'll break down
# things like tempdir context managers.
os._exit(ret) # pylint: disable=protected-access
pids.append(pid)
# Wait for all of them to get cleaned up.
while pids:
if WaitOne():
failed = True
except KeyboardInterrupt:
# If the user wants to stop, reap all the pending children.
logging.warning('CTRL+C received; cleaning up tests')
aborted = True
CleanupChildren(pids)
# Walk through the results.
failed_tests = []
for test, cmd, tmpfile in testsets:
tmpfile.seek(0)
output = tmpfile.read()
if output:
failed_tests.append(test)
print()
logging.error('### LOG: %s', test)
print(output.rstrip())
print()
if failed_tests:
logging.error('The following %i tests failed:\n %s', len(failed_tests),
'\n '.join(sorted(failed_tests)))
return False
elif aborted or failed:
return False
return True
def CreatePidNs():
"""Start a new pid namespace
This will launch all the right manager processes. The child that returns
will be isolated in a new pid namespace.
If functionality is not available, then it will return w/out doing anything.
A note about the processes generated as a result of calling this function:
You call CreatePidNs() in pid X
- X launches Pid Y,
- Pid X will now do nothing but wait for Pid Y to finish and then sys.exit()
with that return code
- Y launches Pid Z
- Pid Y will now do nothing but wait for Pid Z to finish and then
sys.exit() with that return code
- **Pid Z returns from CreatePidNs**. So, the caller of this function
continues in a different process than the one that made the call.
- All SIGTERM/SIGINT signals are forwarded down from pid X to pid Z to
handle.
- SIGKILL will only kill pid X, and leak Pid Y and Z.
Returns:
The last pid outside of the namespace. (i.e., pid X)
"""
first_pid = os.getpid()
try:
# First create the namespace.
Unshare(CLONE_NEWPID)
except OSError as e:
if e.errno == errno.EINVAL:
# For older kernels, or the functionality is disabled in the config,
# return silently. We don't want to hard require this stuff.
return first_pid
else:
# For all other errors, abort. They shouldn't happen.
raise
# Used to make sure process groups are in the right state before we try to
# forward the controlling terminal.
lock = locking.PipeLock()
# Now that we're in the new pid namespace, fork. The parent is the master
# of it in the original namespace, so it only monitors the child inside it.
# It is only allowed to fork once too.
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'external init')
# We forward termination signals to the child and trust the child to respond
# sanely. Later, ExitAsStatus propagates the exit status back up.
_ForwardToChildPid(pid, signal.SIGINT)
_ForwardToChildPid(pid, signal.SIGTERM)
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Signal our child it can move forward.
lock.Post()
del lock
# Reap the children as the parent of the new namespace.
process_util.ExitAsStatus(_ReapChildren(pid))
else:
# Make sure to unshare the existing mount point if needed. Some distros
# create shared mount points everywhere by default.
try:
osutils.Mount('none', '/proc', 0,
osutils.MS_PRIVATE | osutils.MS_REC)
except OSError as e:
if e.errno != errno.EINVAL:
raise
# The child needs its own proc mount as it'll be different.
osutils.Mount(
'proc', '/proc', 'proc', osutils.MS_NOSUID | osutils.MS_NODEV
| osutils.MS_NOEXEC | osutils.MS_RELATIME)
# Wait for our parent to finish initialization.
lock.Wait()
del lock
# Resetup the locks for the next phase.
lock = locking.PipeLock()
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'init')
# We forward termination signals to the child and trust the child to
# respond sanely. Later, ExitAsStatus propagates the exit status back up.
_ForwardToChildPid(pid, signal.SIGINT)
_ForwardToChildPid(pid, signal.SIGTERM)
# Now that we're in a new pid namespace, start a new process group so that
# children have something valid to use. Otherwise getpgrp/etc... will get
# back 0 which tends to confuse -- you can't setpgrp(0) for example.
os.setpgrp()
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Signal our child it can move forward.
lock.Post()
del lock
# Watch all of the children. We need to act as the master inside the
# namespace and reap old processes.
process_util.ExitAsStatus(_ReapChildren(pid))
# Wait for our parent to finish initialization.
lock.Wait()
del lock
# Create a process group for the grandchild so it can manage things
# independent of the init process.
os.setpgrp()
# The grandchild will return and take over the rest of the sdk steps.
return first_pid
def CreatePidNs():
"""Start a new pid namespace
This will launch all the right manager processes. The child that returns
will be isolated in a new pid namespace.
If functionality is not available, then it will return w/out doing anything.
Returns:
The last pid outside of the namespace.
"""
first_pid = os.getpid()
try:
# First create the namespace.
Unshare(CLONE_NEWPID)
except OSError as e:
if e.errno == errno.EINVAL:
# For older kernels, or the functionality is disabled in the config,
# return silently. We don't want to hard require this stuff.
return first_pid
else:
# For all other errors, abort. They shouldn't happen.
raise
# Now that we're in the new pid namespace, fork. The parent is the master
# of it in the original namespace, so it only monitors the child inside it.
# It is only allowed to fork once too.
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'external init')
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Reap the children as the parent of the new namespace.
process_util.ExitAsStatus(_ReapChildren(pid))
else:
# Make sure to unshare the existing mount point if needed. Some distros
# create shared mount points everywhere by default.
try:
osutils.Mount('none', '/proc', 0, osutils.MS_PRIVATE | osutils.MS_REC)
except OSError as e:
if e.errno != errno.EINVAL:
raise
# The child needs its own proc mount as it'll be different.
osutils.Mount('proc', '/proc', 'proc',
osutils.MS_NOSUID | osutils.MS_NODEV | osutils.MS_NOEXEC |
osutils.MS_RELATIME)
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'init')
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Now that we're in a new pid namespace, start a new process group so that
# children have something valid to use. Otherwise getpgrp/etc... will get
# back 0 which tends to confuse -- you can't setpgrp(0) for example.
os.setpgrp()
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Watch all of the children. We need to act as the master inside the
# namespace and reap old processes.
process_util.ExitAsStatus(_ReapChildren(pid))
# Create a process group for the grandchild so it can manage things
# independent of the init process.
os.setpgrp()
# The grandchild will return and take over the rest of the sdk steps.
return first_pid
def CreatePidNs():
"""Start a new pid namespace
This will launch all the right manager processes. The child that returns
will be isolated in a new pid namespace.
If functionality is not available, then it will return w/out doing anything.
Returns:
The last pid outside of the namespace.
"""
first_pid = os.getpid()
try:
# First create the namespace.
Unshare(CLONE_NEWPID)
except OSError as e:
if e.errno == errno.EINVAL:
# For older kernels, or the functionality is disabled in the config,
# return silently. We don't want to hard require this stuff.
return first_pid
else:
# For all other errors, abort. They shouldn't happen.
raise
# Used to make sure process groups are in the right state before we try to
# forward the controlling terminal.
lock = locking.PipeLock()
# Now that we're in the new pid namespace, fork. The parent is the master
# of it in the original namespace, so it only monitors the child inside it.
# It is only allowed to fork once too.
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'external init')
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Signal our child it can move forward.
lock.Post()
del lock
# Reap the children as the parent of the new namespace.
process_util.ExitAsStatus(_ReapChildren(pid))
else:
# Make sure to unshare the existing mount point if needed. Some distros
# create shared mount points everywhere by default.
try:
osutils.Mount('none', '/proc', 0,
osutils.MS_PRIVATE | osutils.MS_REC)
except OSError as e:
if e.errno != errno.EINVAL:
raise
# The child needs its own proc mount as it'll be different.
osutils.Mount(
'proc', '/proc', 'proc', osutils.MS_NOSUID | osutils.MS_NODEV
| osutils.MS_NOEXEC | osutils.MS_RELATIME)
# Wait for our parent to finish initialization.
lock.Wait()
del lock
# Resetup the locks for the next phase.
lock = locking.PipeLock()
pid = os.fork()
if pid:
proctitle.settitle('pid ns', 'init')
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
# Now that we're in a new pid namespace, start a new process group so that
# children have something valid to use. Otherwise getpgrp/etc... will get
# back 0 which tends to confuse -- you can't setpgrp(0) for example.
os.setpgrp()
# Forward the control of the terminal to the child so it can manage input.
_SafeTcSetPgrp(sys.stdin.fileno(), pid)
# Signal our child it can move forward.
lock.Post()
del lock
# Watch all of the children. We need to act as the master inside the
# namespace and reap old processes.
process_util.ExitAsStatus(_ReapChildren(pid))
# Wait for our parent to finish initialization.
lock.Wait()
del lock
# Create a process group for the grandchild so it can manage things
# independent of the init process.
os.setpgrp()
# The grandchild will return and take over the rest of the sdk steps.
return first_pid
def run(self):
"""Runs the test in a proper environment (e.g. qemu)."""
# We know these pre-tests are fast (especially if they've already been run
# once), so run them automatically for the user if they test by hand.
self.pre_test()
for mount in self._BIND_MOUNT_PATHS:
path = os.path.join(self.sysroot, mount)
osutils.SafeMakedirs(path)
osutils.Mount('/' + mount, path, 'none', osutils.MS_BIND)
positive_filters = self.gtest_filter[0]
negative_filters = self.gtest_filter[1]
if self.user_gtest_filter:
positive_filters += self.user_gtest_filter[0]
negative_filters += self.user_gtest_filter[1]
filters = (':'.join(positive_filters), ':'.join(negative_filters))
gtest_filter = '%s-%s' % filters
cmd = self.removeSysrootPrefix(self.bin)
argv = self.args[:]
argv[0] = self.removeSysrootPrefix(argv[0])
if gtest_filter != '-':
argv.append('--gtest_filter=' + gtest_filter)
# Some programs expect to find data files via $CWD, so doing a chroot
# and dropping them into / would make them fail.
cwd = self.removeSysrootPrefix(os.getcwd())
# Fork off a child to run the test. This way we can make tweaks to the
# env that only affect the child (gid/uid/chroot/cwd/etc...). We have
# to fork anyways to run the test, so might as well do it all ourselves
# to avoid (slow) chaining through programs like:
# sudo -u $SUDO_UID -g $SUDO_GID chroot $SYSROOT bash -c 'cd $CWD; $BIN'
child = os.fork()
if child == 0:
print('chroot: %s' % self.sysroot)
print('cwd: %s' % cwd)
print('cmd: {%s} %s' % (cmd, ' '.join(map(repr, argv))))
os.chroot(self.sysroot)
os.chdir(cwd)
# The TERM the user is leveraging might not exist in the sysroot.
# Force a sane default that supports standard color sequences.
os.environ['TERM'] = 'ansi'
# Some progs want this like bash else they get super confused.
os.environ['PWD'] = cwd
if not self.run_as_root:
_, uid, gid, home = self.GetNonRootAccount()
os.setgid(gid)
os.setuid(uid)
os.environ['HOME'] = home
sys.exit(os.execvp(cmd, argv))
proctitle.settitle('sysroot watcher', cmd)
# Mask SIGINT with the assumption that the child will catch & process it.
# We'll pass that back up below.
signal.signal(signal.SIGINT, signal.SIG_IGN)
_, status = os.waitpid(child, 0)
failmsg = None
if os.WIFSIGNALED(status):
sig = os.WTERMSIG(status)
failmsg = 'signal %s(%i)' % (signals.StrSignal(sig), sig)
else:
exit_status = os.WEXITSTATUS(status)
if exit_status:
failmsg = 'exit code %i' % exit_status
if failmsg:
print('Error: %s: failed with %s' % (cmd, failmsg),
file=sys.stderr)
process_util.ExitAsStatus(status)