def check_cgroup_availability(wait=1):
"""
Basic utility to check the availability and permissions of cgroups.
This will log some warnings for the user if necessary.
On some systems, daemons such as cgrulesengd might interfere with the cgroups
of a process soon after it was started. Thus this function starts a process,
waits a configurable amount of time, and check whether the cgroups have been changed.
@param wait: a non-negative int that is interpreted as seconds to wait during the check
@raise SystemExit: if cgroups are not usable
"""
logging.basicConfig(format="%(levelname)s: %(message)s")
runexecutor = RunExecutor(use_namespaces=False)
my_cgroups = runexecutor.cgroups
if not (
CPUACCT in my_cgroups
and CPUSET in my_cgroups
# and FREEZER in my_cgroups # For now, we do not require freezer
and MEMORY in my_cgroups
):
sys.exit(1)
with tempfile.NamedTemporaryFile(mode="rt") as tmp:
runexecutor.execute_run(
["sh", "-c", "sleep {0}; cat /proc/self/cgroup".format(wait)],
tmp.name,
memlimit=1024 * 1024, # set memlimit to force check for swapaccount
# set cores and memory_nodes to force usage of CPUSET
cores=util.parse_int_list(my_cgroups.get_value(CPUSET, "cpus")),
memory_nodes=my_cgroups.read_allowed_memory_banks(),
)
lines = []
for line in tmp:
line = line.strip()
if (
line
and not line == "sh -c 'sleep {0}; cat /proc/self/cgroup'".format(wait)
and not all(c == "-" for c in line)
):
lines.append(line)
task_cgroups = find_my_cgroups(lines)
fail = False
for subsystem in CPUACCT, CPUSET, MEMORY, FREEZER:
if subsystem in my_cgroups:
if not task_cgroups[subsystem].startswith(
os.path.join(my_cgroups[subsystem], "benchmark_")
):
logging.warning(
"Task was in cgroup %s for subsystem %s, "
"which is not the expected sub-cgroup of %s. "
"Maybe some other program is interfering with cgroup management?",
task_cgroups[subsystem],
subsystem,
my_cgroups[subsystem],
)
fail = True
if fail:
sys.exit(1)
class TestRunExecutorWithSudo(TestRunExecutor):
"""
Run tests using the sudo mode of RunExecutor, if possible.
sudo is typically set up to allow executing as our own user,
so we try that. Note that this will not catch all problems,
for example if we forget to use "sudo kill" to send a signal
and instead send it directly, but requiring a second user for tests
would not be good, either.
"""
# Use user name defined in environment variable if present,
# or fall back to current user (sudo always allows this).
# sudo allows refering to numerical uids with '#'.
user = os.environ.get('BENCHEXEC_TEST_USER', '#' + str(os.getuid()))
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(user=self.user, *args, **kwargs)
except SystemExit as e:
# sudo seems not to be available
self.skipTest(e)
def check_exitcode(self, result, expected, msg=None):
actual = int(result['exitcode'])
if expected == 15 and actual == 0:
# On Ubuntu 16.04, sudo returns 0 if process is killed with signal 15
return
# Using sudo may affect the exit code:
# what was the returnsignal is now the returnvalue.
# The distinction between returnsignal and returnvalue of the actual
# process is lost.
# If the returnsignal (of the sudo process) is 0,
# we replace the exit code with the mixed returnsignal/returnvalue of
# the actual process (with bit for core dump cleared).
returnsignal = actual & 0x7F
returnvalue = (actual >> 8) & 0x7F
if returnsignal == 0:
actual = returnvalue
self.assertEqual(actual, expected, msg)
def check_command_in_output(self, output, cmd):
self.assertTrue(output[0].endswith(cmd), 'run output misses executed command')
def test_detect_new_files_in_home(self):
if not os.path.exists('/usr/bin/mktemp'):
self.skipTest('missing /usr/bin/mktemp')
home_dir = runexecutor._get_user_account_info(self.user).pw_dir
tmp_file_pattern = '.BenchExec_test_runexecutor_'+unichr(0xe4)+unichr(0xf6)+unichr(0xfc)+'_XXXXXXXXXX'
(result, output) = self.execute_run(
'/usr/bin/mktemp', '--tmpdir=' + home_dir, tmp_file_pattern)
try:
self.check_exitcode(result, 0, 'exit code of /usr/bin/mktemp is not zero')
tmp_file = output[-1]
self.assertIn(tmp_file, self.runexecutor.check_for_new_files_in_home(),
'runexecutor failed to detect new temporary file in home directory')
finally:
subprocess.check_call(self.runexecutor._build_cmdline(['rm', tmp_file]))
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(use_namespaces=False, *args, **kwargs)
except SystemExit as e:
if str(e).startswith(
"Cannot reliably kill sub-processes without freezer cgroup"
):
self.skipTest(e)
else:
raise e
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user)
self.setDaemon(True)
self.start()
def __init__(self, benchmark, my_cpus, my_memory_nodes, output_handler,
run_finished_callback):
threading.Thread.__init__(self) # constuctor of superclass
self.run_finished_callback = run_finished_callback
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run_exec_cmd(
environment: Environment,
cmd: Sequence[str],
env: Mapping[str, str],
cwd: Path,
dir_modes: Mapping[str, Any],
tmp_dir: Path,
time_limit: int,
mem_limit: int,
) -> Tuple[Tuple[str, ...], RunexecStats, str, str]:
stdout = tmp_dir / "info.log"
if stdout.exists():
stdout.unlink()
stderr = tmp_dir / "error.log"
if stderr.exists():
stderr.unlink()
combined_cmd = combine_cmd(*environment.run(cmd, env, cwd))
stderr.write_text(" \\\n ".join(
shlex.quote(part) for part in combined_cmd) + "\n\n")
run_executor = RunExecutor(
use_namespaces=False,
# dir_modes=dir_modes,
# network_access=True,
# Need to system config so DNS works.
# container_system_config=True,
)
logger = logging.getLogger("root")
with runexec_catch_signals(run_executor), capture_logs(
logger, logging.WARNING) as logs:
run_exec_run = run_executor.execute_run(
args=combined_cmd,
environments={
"keepEnv": {},
},
workingDir="/",
write_header=False,
output_filename=stdout,
error_filename=stderr,
softtimelimit=time_limit,
hardtimelimit=int(time_limit * 1.1),
walltimelimit=int(time_limit * 1.2),
memlimit=mem_limit,
)
return (
combined_cmd,
RunexecStats.create(run_exec_run),
stdout.read_text(),
stderr.read_text() + "\n".join(record.getMessage() for record in logs),
)
def check_cgroup_availability(wait=1):
"""
Basic utility to check the availability and permissions of cgroups.
This will log some warnings for the user if necessary.
On some systems, daemons such as cgrulesengd might interfere with the cgroups
of a process soon after it was started. Thus this function starts a process,
waits a configurable amount of time, and check whether the cgroups have been changed.
@param wait: a non-negative int that is interpreted as seconds to wait during the check
@raise SystemExit: if cgroups are not usable
"""
logging.basicConfig(format="%(levelname)s: %(message)s")
runexecutor = RunExecutor()
my_cgroups = runexecutor.cgroups
if not (CPUACCT in my_cgroups and
CPUSET in my_cgroups and
# FREEZER in my_cgroups and # For now, we do not require freezer
MEMORY in my_cgroups):
sys.exit(1)
with tempfile.NamedTemporaryFile(mode='rt') as tmp:
runexecutor.execute_run(['sh', '-c', 'sleep {0}; cat /proc/self/cgroup'.format(wait)], tmp.name,
memlimit=1024*1024, # set memlimit to force check for swapaccount
# set cores and memory_nodes to force usage of CPUSET
cores=util.parse_int_list(my_cgroups.get_value(CPUSET, 'cpus')),
memory_nodes=my_cgroups.read_allowed_memory_banks())
lines = []
for line in tmp:
line = line.strip()
if line and not line == "sh -c 'sleep {0}; cat /proc/self/cgroup'".format(wait) \
and not all(c == '-' for c in line):
lines.append(line)
task_cgroups = find_my_cgroups(lines)
fail = False
for subsystem in CPUACCT, CPUSET, MEMORY, FREEZER:
if subsystem in my_cgroups:
if not task_cgroups[subsystem].startswith(os.path.join(my_cgroups[subsystem], 'benchmark_')):
logging.warning('Task was in cgroup %s for subsystem %s, '
'which is not the expected sub-cgroup of %s. '
'Maybe some other program is interfering with cgroup management?',
task_cgroups[subsystem], subsystem, my_cgroups[subsystem])
fail = True
if fail:
sys.exit(1)
def test_require_cgroup_invalid(self):
with self.assertLogs(level=logging.ERROR) as log:
with self.assertRaises(SystemExit):
RunExecutor(additional_cgroup_subsystems=["invalid"])
self.assertIn(
'Cgroup subsystem "invalid" was required but is not available',
"\n".join(log.output),
)
def setUp(self, *args, **kwargs):
try:
container.execute_in_namespace(lambda: 0)
except OSError as e:
self.skipTest("Namespaces not supported: {}".format(os.strerror(e.errno)))
self.runexecutor = RunExecutor(
use_namespaces=True,
dir_modes={"/": containerexecutor.DIR_READ_ONLY,
"/tmp": containerexecutor.DIR_HIDDEN},
container_system_config=False,
*args, **kwargs)
class Runner():
"""Base class to run runxec"""
def __init__(self):
self.executor = RunExecutor()
signal.signal(signal.SIGINT, self.stop_run)
def stop_run(self):
self.executor.stop()
def _runexec_args(self):
return []
def _run(self, output):
return self.executor.execute_run(args=self._runexec_args(),
output_filename=output)
# Should I spawn a process?
def run(self, output):
return self._run(output)
def validate(self):
raise NotImplementedError()
def _run_command(self,
command: str,
time: float,
memory=None,
file_count=None,
stdin=None):
"""Runs specified command
Arguments:
command {str} -- command to be run
time {float} -- time limit in seconds
Keyword Arguments:
memory {int} -- memory limit in bytes (default: {None})
file_count {int} -- maximum number of files to write (default: {None})
stdin {file} -- file used as standard input (default: {None})
Returns:
OrderedDict -- dict with results
"""
executor = RunExecutor(use_namespaces=False)
if memory is None:
memory = self._limits.max_memory
args = command.split(' ')
try:
result = executor.execute_run(args,
self._output_log_file,
memlimit=memory,
hardtimelimit=time,
walltimelimit=time,
files_count_limit=file_count,
stdin=stdin)
if 'cputime' in result and result['cputime'] > time:
result['terminationreason'] = 'cputime'
except:
result = {'terminationreason': 'something went very wrong'}
return result
def setUp(self, *args, **kwargs):
try:
container.execute_in_namespace(lambda: 0)
except OSError as e:
self.skipTest(f"Namespaces not supported: {os.strerror(e.errno)}")
dir_modes = kwargs.pop(
"dir_modes",
{
"/": containerexecutor.DIR_READ_ONLY,
"/home": containerexecutor.DIR_HIDDEN,
"/tmp": containerexecutor.DIR_HIDDEN,
},
)
self.runexecutor = RunExecutor(
use_namespaces=True, dir_modes=dir_modes, *args, **kwargs
)
def setUp(self, *args, **kwargs):
with self.skip_if_logs(
"Cannot reliably kill sub-processes without freezer cgroup"):
self.runexecutor = RunExecutor(use_namespaces=False,
*args,
**kwargs)
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
if not hasattr(cls, 'assertRaisesRegex'):
cls.assertRaisesRegex = cls.assertRaisesRegexp
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(list(args), output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def check_command_in_output(self, output, cmd):
self.assertEqual(output[0], cmd, 'run output misses executed command')
def check_result_keys(self, result, *additional_keys):
expected_keys = {'cputime', 'walltime', 'memory', 'exitcode',
'energy', 'energy-cpu', 'energy-core', 'energy-uncore'}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith('cputime-cpu'):
self.assertRegex(key, '^cputime-cpu[0-9]+$', 'unexpected result value ' + key)
else:
self.assertIn(key, expected_keys, 'unexpected result value ' + key)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
self.check_result_keys(result)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, cmd, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
runexecutor._reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_home_and_tmp_is_separate(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
self.assertRegex(output[-1], '/BenchExec_run_[^/]*/home .*/BenchExec_run_[^/]*/tmp',
'HOME or TMPDIR variable does not contain expected temporary directory')
def test_temp_dirs_are_removed(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
home_dir = output[-1].split(' ')[0]
temp_dir = output[-1].split(' ')[1]
self.assertFalse(os.path.exists(home_dir),
'temporary home directory {} was not cleaned up'.format(home_dir))
self.assertFalse(os.path.exists(temp_dir),
'temporary temp directory {} was not cleaned up'.format(temp_dir))
def test_no_cleanup_temp(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
self.setUp(cleanup_temp_dir=False) # create RunExecutor with desired parameter
(result, output) = self.execute_run('/bin/sh', '-c', 'echo "$TMPDIR"; echo "" > "$TMPDIR/test"')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
temp_dir = output[-1]
test_file = os.path.join(temp_dir, 'test')
subprocess.check_call(self.runexecutor._build_cmdline(['test', '-f', test_file]))
self.assertEqual('tmp', os.path.basename(temp_dir), 'unexpected name of temp dir')
self.assertNotEqual('/tmp', temp_dir, 'temp dir should not be the global temp dir')
subprocess.check_call(self.runexecutor._build_cmdline(['rm', '-r', os.path.dirname(temp_dir)]))
def test_require_cgroup_invalid(self):
self.assertRaisesRegex(SystemExit, '.*invalid.*',
lambda: RunExecutor(additional_cgroup_subsystems=['invalid']))
def test_require_cgroup_cpu(self):
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(result, output) = self.execute_run('/bin/cat', '/proc/self/cgroup')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/cat is not zero')
for line in output:
if re.match('^[0-9]*:cpu:/(.*/)?benchmark_.*$',line):
return # Success
self.fail('Not in expected cgroup for subsystem cpu:\n' + '\n'.join(output))
def test_set_cgroup_cpu_shares(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
(result, _) = self.execute_run('/bin/echo',
cgroupValues={('cpu', 'shares'): 42})
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
def setUp(self):
self.runexecutor = RunExecutor()
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(
logging.NOTSET) # need to make sure to get all messages
if not hasattr(cls, "assertRegex"):
cls.assertRegex = cls.assertRegexpMatches
def setUp(self, *args, **kwargs):
with self.skip_if_logs(
"Cannot reliably kill sub-processes without freezer cgroup"):
self.runexecutor = RunExecutor(use_namespaces=False,
*args,
**kwargs)
@contextlib.contextmanager
def skip_if_logs(self, error_msg):
"""A context manager that automatically marks the test as skipped if SystemExit
is thrown and the given error message had been logged with level ERROR."""
# Note: assertLogs checks that there is at least one log message of given level.
# This is not what we want, so we just rely on one debug message being present.
try:
with self.assertLogs(level=logging.DEBUG) as log:
yield
except SystemExit as e:
if any(record.levelno == logging.ERROR
and record.msg.startswith(error_msg)
for record in log.records):
self.skipTest(e)
raise e
def execute_run(self, *args, expect_terminationreason=None, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp(".log",
"output_",
text=True)
try:
result = self.runexecutor.execute_run(list(args), output_filename,
**kwargs)
output = os.read(output_fd, 4096).decode()
finally:
os.close(output_fd)
os.remove(output_filename)
self.check_result_keys(result, "terminationreason")
if isinstance(expect_terminationreason, list):
self.assertIn(
result.get("terminationreason"),
expect_terminationreason,
"Unexpected terminationreason, output is \n" + output,
)
else:
self.assertEqual(
result.get("terminationreason"),
expect_terminationreason,
"Unexpected terminationreason, output is \n" + output,
)
return (result, output.splitlines())
def get_runexec_cmdline(self, *args, **kwargs):
return [
"python3",
runexec,
"--no-container",
"--output",
kwargs["output_filename"],
] + list(args)
def execute_run_extern(self,
*args,
expect_terminationreason=None,
**kwargs):
(output_fd, output_filename) = tempfile.mkstemp(".log",
"output_",
text=True)
try:
runexec_output = subprocess.check_output(
args=self.get_runexec_cmdline(*args,
output_filename=output_filename),
stderr=subprocess.DEVNULL,
**kwargs,
).decode()
output = os.read(output_fd, 4096).decode()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result = {
key.strip(): value.strip()
for (key, _, value) in (line.partition("=")
for line in runexec_output.splitlines())
}
self.check_result_keys(result, "terminationreason", "returnvalue")
if isinstance(expect_terminationreason, list):
self.assertIn(
result.get("terminationreason"),
expect_terminationreason,
"Unexpected terminationreason, output is \n" + output,
)
else:
self.assertEqual(
result.get("terminationreason"),
expect_terminationreason,
"Unexpected terminationreason, output is \n" + output,
)
return (result, output.splitlines())
def check_command_in_output(self, output, cmd):
self.assertEqual(output[0], cmd, "run output misses executed command")
def check_result_keys(self, result, *additional_keys):
expected_keys = {
"cputime",
"walltime",
"memory",
"exitcode",
"cpuenergy",
"blkio-read",
"blkio-write",
"starttime",
}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith("cputime-cpu"):
self.assertRegex(
key,
"^cputime-cpu[0-9]+$",
"unexpected result entry '{}={}'".format(key, result[key]),
)
elif key.startswith("cpuenergy-"):
self.assertRegex(
key,
"^cpuenergy-pkg[0-9]+-(package|core|uncore|dram|psys)$",
"unexpected result entry '{}={}'".format(key, result[key]),
)
else:
self.assertIn(
key,
expected_keys,
"unexpected result entry '{}={}'".format(key, result[key]),
)
def check_exitcode(self, result, exitcode, msg=None):
self.assertEqual(result["exitcode"].raw, exitcode, msg)
def check_exitcode_extern(self, result, exitcode, msg=None):
exitcode = util.ProcessExitCode.from_raw(exitcode)
if exitcode.value is not None:
self.assertEqual(int(result["returnvalue"]), exitcode.value, msg)
else:
self.assertEqual(int(result["exitsignal"]), exitcode.signal, msg)
def test_command_output(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
(_, output) = self.execute_run("/bin/echo", "TEST_TOKEN")
self.check_command_in_output(output, "/bin/echo TEST_TOKEN")
self.assertEqual(output[-1], "TEST_TOKEN",
"run output misses command output")
for line in output[1:-1]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_command_error_output(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
def execute_Run_intern(*args, **kwargs):
(error_fd, error_filename) = tempfile.mkstemp(".log",
"error_",
text=True)
try:
(_, output_lines) = self.execute_run(
*args, error_filename=error_filename, **kwargs)
error_lines = os.read(error_fd, 4096).decode().splitlines()
return (output_lines, error_lines)
finally:
os.close(error_fd)
os.remove(error_filename)
(output_lines,
error_lines) = execute_Run_intern("/bin/sh", "-c",
"/bin/echo ERROR_TOKEN >&2")
self.assertEqual(error_lines[-1], "ERROR_TOKEN",
"run error output misses command output")
for line in output_lines[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
for line in error_lines[1:-1]:
self.assertRegex(line, "^-*$",
"unexpected text in run error output")
(output_lines,
error_lines) = execute_Run_intern("/bin/echo", "OUT_TOKEN")
self.check_command_in_output(output_lines, "/bin/echo OUT_TOKEN")
self.check_command_in_output(error_lines, "/bin/echo OUT_TOKEN")
self.assertEqual(output_lines[-1], "OUT_TOKEN",
"run output misses command output")
for line in output_lines[1:-1]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
for line in error_lines[1:]:
self.assertRegex(line, "^-*$",
"unexpected text in run error output")
def test_command_result(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
(result, _) = self.execute_run("/bin/echo", "TEST_TOKEN")
self.check_exitcode(result, 0, "exit code of /bin/echo is not zero")
self.assertAlmostEqual(
result["walltime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg="walltime of /bin/echo not as expected",
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
result["cputime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg="cputime of /bin/echo not as expected",
)
self.check_result_keys(result)
def test_wrong_command(self):
(result, _) = self.execute_run("/does/not/exist",
expect_terminationreason="failed")
def test_wrong_command_extern(self):
(result, _) = self.execute_run("/does/not/exist",
expect_terminationreason="failed")
def test_cputime_hardlimit(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
with self.skip_if_logs(
"Time limit cannot be specified without cpuacct cgroup"):
(result, output) = self.execute_run(
"/bin/sh",
"-c",
"i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i",
hardtimelimit=1,
expect_terminationreason="cputime",
)
self.check_exitcode(result, 9, "exit code of killed process is not 9")
self.assertAlmostEqual(
result["walltime"],
1.4,
delta=0.5,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
self.assertAlmostEqual(
result["cputime"],
1.4,
delta=0.5,
msg=
"cputime is not approximately the time after which the process should have been killed",
)
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_cputime_softlimit(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
with self.skip_if_logs(
"Soft time limit cannot be specified without cpuacct cgroup"):
(result, output) = self.execute_run(
"/bin/sh",
"-c",
"i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i",
softtimelimit=1,
expect_terminationreason="cputime-soft",
)
self.check_exitcode(result, 15,
"exit code of killed process is not 15")
self.assertAlmostEqual(
result["walltime"],
4,
delta=3,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
self.assertAlmostEqual(
result["cputime"],
4,
delta=3,
msg=
"cputime is not approximately the time after which the process should have been killed",
)
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_walltime_limit(self):
if not os.path.exists("/bin/sleep"):
self.skipTest("missing /bin/sleep")
(result,
output) = self.execute_run("/bin/sleep",
"10",
walltimelimit=1,
expect_terminationreason="walltime")
self.check_exitcode(result, 9, "exit code of killed process is not 9")
self.assertAlmostEqual(
result["walltime"],
4,
delta=3,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
result["cputime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg="cputime of /bin/sleep is not approximately zero",
)
self.check_command_in_output(output, "/bin/sleep 10")
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_cputime_walltime_limit(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
with self.skip_if_logs(
"Time limit cannot be specified without cpuacct cgroup"):
(result, output) = self.execute_run(
"/bin/sh",
"-c",
"i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i",
hardtimelimit=1,
walltimelimit=5,
expect_terminationreason="cputime",
)
self.check_exitcode(result, 9, "exit code of killed process is not 9")
self.assertAlmostEqual(
result["walltime"],
1.4,
delta=0.5,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
self.assertAlmostEqual(
result["cputime"],
1.4,
delta=0.5,
msg=
"cputime is not approximately the time after which the process should have been killed",
)
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_all_timelimits(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
with self.skip_if_logs(
"Time limit cannot be specified without cpuacct cgroup"):
(result, output) = self.execute_run(
"/bin/sh",
"-c",
"i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i",
softtimelimit=1,
hardtimelimit=2,
walltimelimit=5,
expect_terminationreason="cputime-soft",
)
self.check_exitcode(result, 15,
"exit code of killed process is not 15")
self.assertAlmostEqual(
result["walltime"],
1.4,
delta=0.5,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
self.assertAlmostEqual(
result["cputime"],
1.4,
delta=0.5,
msg=
"cputime is not approximately the time after which the process should have been killed",
)
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_input_is_redirected_from_devnull(self):
if not os.path.exists("/bin/cat"):
self.skipTest("missing /bin/cat")
(result, output) = self.execute_run("/bin/cat", walltimelimit=1)
self.check_exitcode(result, 0, "exit code of process is not 0")
self.assertAlmostEqual(
result["walltime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero',
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
result["cputime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg=
'cputime of "/bin/cat < /dev/null" is not approximately zero',
)
self.check_result_keys(result)
self.check_command_in_output(output, "/bin/cat")
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_input_is_redirected_from_file(self):
if not os.path.exists("/bin/cat"):
self.skipTest("missing /bin/cat")
with tempfile.TemporaryFile() as tmp:
tmp.write(b"TEST_TOKEN")
tmp.flush()
tmp.seek(0)
(result, output) = self.execute_run("/bin/cat",
stdin=tmp,
walltimelimit=1)
self.check_exitcode(result, 0, "exit code of process is not 0")
self.assertAlmostEqual(
result["walltime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero',
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
result["cputime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg=
'cputime of "/bin/cat < /dev/null" is not approximately zero',
)
self.check_result_keys(result)
self.check_command_in_output(output, "/bin/cat")
self.assertEqual(output[-1], "TEST_TOKEN",
"run output misses command output")
for line in output[1:-1]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_input_is_redirected_from_stdin(self):
if not os.path.exists("/bin/cat"):
self.skipTest("missing /bin/cat")
(output_fd, output_filename) = tempfile.mkstemp(".log",
"output_",
text=True)
cmd = self.get_runexec_cmdline(
"--input",
"-",
"--walltime",
"1",
"/bin/cat",
output_filename=output_filename,
)
try:
process = subprocess.Popen(
args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
try:
runexec_output, unused_err = process.communicate(b"TEST_TOKEN")
except BaseException:
# catch everything, we re-raise
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode,
cmd,
output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result = {
key.strip(): value.strip()
for (key, _,
value) in (line.partition("=")
for line in runexec_output.decode().splitlines())
}
self.check_exitcode_extern(result, 0, "exit code of process is not 0")
self.assertAlmostEqual(
float(result["walltime"].rstrip("s")),
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero',
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
float(result["cputime"].rstrip("s")),
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg=
'cputime of "/bin/cat < /dev/null" is not approximately zero',
)
self.check_result_keys(result, "returnvalue")
self.check_command_in_output(output, "/bin/cat")
self.assertEqual(output[-1], "TEST_TOKEN",
"run output misses command output")
for line in output[1:-1]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_append_environment_variable(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(_, output) = self.execute_run("/bin/sh", "-c", "echo $PATH")
path = output[-1]
(_, output) = self.execute_run(
"/bin/sh",
"-c",
"echo $PATH",
environments={"additionalEnv": {
"PATH": ":TEST_TOKEN"
}},
)
self.assertEqual(output[-1], path + ":TEST_TOKEN")
def test_new_environment_variable(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(_, output) = self.execute_run(
"/bin/sh",
"-c",
"echo $PATH",
environments={"newEnv": {
"PATH": "/usr/bin"
}})
self.assertEqual(output[-1], "/usr/bin")
def test_stop_run(self):
if not os.path.exists("/bin/sleep"):
self.skipTest("missing /bin/sleep")
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run("/bin/sleep",
"10",
expect_terminationreason="killed")
thread.join()
self.check_exitcode(result, 9, "exit code of killed process is not 9")
self.assertAlmostEqual(
result["walltime"],
1,
delta=0.5,
msg=
"walltime is not approximately the time after which the process should have been killed",
)
if "cputime" in result: # not present without cpuacct cgroup
self.assertAlmostEqual(
result["cputime"],
trivial_run_grace_time,
delta=trivial_run_grace_time,
msg="cputime of /bin/sleep is not approximately zero",
)
self.check_command_in_output(output, "/bin/sleep 10")
for line in output[1:]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode="wt") as tmp:
content = "Long line " * 500
tmp.write(content)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, "rt") as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = (
"WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
)
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode="wt") as tmp:
line = "Some text\n"
tmp.write(line * 500)
tmp.flush()
limit = 500
runexecutor._reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name),
limit + self.REDUCE_OVERHEAD)
with open(tmp.name, "rt") as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode="wt") as tmp:
line = "Some text\n"
tmp.write(line * 500)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name),
self.REDUCE_OVERHEAD)
with open(tmp.name, "rt") as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_append_crash_dump_info(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(result, output) = self.execute_run(
"/bin/sh",
"-c",
'echo "# An error report file with more information is saved as:";'
'echo "# $(pwd)/hs_err_pid_1234.txt";'
"echo TEST_TOKEN > hs_err_pid_1234.txt;"
"exit 2",
)
self.assertEqual(output[-1], "TEST_TOKEN",
"log file misses content from crash dump file")
def test_integration(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
(result, output) = self.execute_run_extern("/bin/echo", "TEST_TOKEN")
self.check_exitcode_extern(result, 0,
"exit code of /bin/echo is not zero")
self.check_result_keys(result, "returnvalue")
self.check_command_in_output(output, "/bin/echo TEST_TOKEN")
self.assertEqual(output[-1], "TEST_TOKEN",
"run output misses command output")
for line in output[1:-1]:
self.assertRegex(line, "^-*$", "unexpected text in run output")
def test_home_and_tmp_is_separate(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(result, output) = self.execute_run("/bin/sh", "-c",
"echo $HOME $TMPDIR")
self.check_exitcode(result, 0, "exit code of /bin/sh is not zero")
self.assertRegex(
output[-1],
"/BenchExec_run_[^/]*/home .*/BenchExec_run_[^/]*/tmp",
"HOME or TMPDIR variable does not contain expected temporary directory",
)
def test_temp_dirs_are_removed(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(result, output) = self.execute_run("/bin/sh", "-c",
"echo $HOME $TMPDIR")
self.check_exitcode(result, 0, "exit code of /bin/sh is not zero")
home_dir = output[-1].split(" ")[0]
temp_dir = output[-1].split(" ")[1]
self.assertFalse(
os.path.exists(home_dir),
"temporary home directory {} was not cleaned up".format(home_dir),
)
self.assertFalse(
os.path.exists(temp_dir),
"temporary temp directory {} was not cleaned up".format(temp_dir),
)
def test_home_is_writable(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
(result, output) = self.execute_run("/bin/sh", "-c",
"touch $HOME/TEST_FILE")
self.check_exitcode(
result,
0,
"Failed to write to $HOME/TEST_FILE, output was\n{}".format(
output),
)
def test_no_cleanup_temp(self):
if not os.path.exists("/bin/sh"):
self.skipTest("missing /bin/sh")
self.setUp(cleanup_temp_dir=False
) # create RunExecutor with desired parameter
(result, output) = self.execute_run(
"/bin/sh", "-c", 'echo "$TMPDIR"; echo "" > "$TMPDIR/test"')
self.check_exitcode(result, 0, "exit code of /bin/sh is not zero")
temp_dir = output[-1]
test_file = os.path.join(temp_dir, "test")
subprocess.check_call(["test", "-f", test_file])
self.assertEqual("tmp", os.path.basename(temp_dir),
"unexpected name of temp dir")
self.assertNotEqual("/tmp", temp_dir,
"temp dir should not be the global temp dir")
subprocess.check_call(["rm", "-r", os.path.dirname(temp_dir)])
def test_require_cgroup_invalid(self):
with self.assertLogs(level=logging.ERROR) as log:
with self.assertRaises(SystemExit):
RunExecutor(additional_cgroup_subsystems=["invalid"])
self.assertIn(
'Cgroup subsystem "invalid" was required but is not available',
"\n".join(log.output),
)
def test_require_cgroup_cpu(self):
try:
self.setUp(additional_cgroup_subsystems=["cpu"])
except SystemExit as e:
self.skipTest(e)
if not os.path.exists("/bin/cat"):
self.skipTest("missing /bin/cat")
(result, output) = self.execute_run("/bin/cat", "/proc/self/cgroup")
self.check_exitcode(result, 0, "exit code of /bin/cat is not zero")
for line in output:
if re.match(r"^[0-9]*:([^:]*,)?cpu(,[^:]*)?:/(.*/)?benchmark_.*$",
line):
return # Success
self.fail("Not in expected cgroup for subsystem cpu:\n" +
"\n".join(output))
def test_set_cgroup_cpu_shares(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
try:
self.setUp(additional_cgroup_subsystems=["cpu"])
except SystemExit as e:
self.skipTest(e)
(result, _) = self.execute_run("/bin/echo",
cgroupValues={("cpu", "shares"): 42})
self.check_exitcode(result, 0, "exit code of /bin/echo is not zero")
# Just assert that execution was successful,
# testing that the value was actually set is much more difficult.
def test_nested_runexec(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
self.setUp(
dir_modes={
# Do not mark /home hidden, would fail with python from virtualenv
"/": containerexecutor.DIR_READ_ONLY,
"/tmp":
containerexecutor.DIR_FULL_ACCESS, # for inner_output_file
"/sys/fs/cgroup": containerexecutor.DIR_FULL_ACCESS,
})
inner_args = ["--", "/bin/echo", "TEST_TOKEN"]
with tempfile.NamedTemporaryFile(mode="r",
prefix="inner_output_",
suffix=".log") as inner_output_file:
inner_cmdline = self.get_runexec_cmdline(
*inner_args, output_filename=inner_output_file.name)
outer_result, outer_output = self.execute_run(*inner_cmdline)
inner_output = inner_output_file.read().strip().splitlines()
logging.info("Outer output:\n" + "\n".join(outer_output))
logging.info("Inner output:\n" + "\n".join(inner_output))
self.check_result_keys(outer_result, "returnvalue")
self.check_exitcode(outer_result, 0,
"exit code of inner runexec is not zero")
self.check_command_in_output(inner_output, "/bin/echo TEST_TOKEN")
self.assertEqual(inner_output[-1], "TEST_TOKEN",
"run output misses command output")
def test_starttime(self):
if not os.path.exists("/bin/echo"):
self.skipTest("missing /bin/echo")
before = util.read_local_time()
(result, _) = self.execute_run("/bin/echo")
after = util.read_local_time()
self.check_result_keys(result)
run_starttime = result["starttime"]
self.assertIsNotNone(run_starttime.tzinfo,
"start time is not a local time")
self.assertLessEqual(before, run_starttime)
self.assertLessEqual(run_starttime, after)
def __init__(self):
self.executor = RunExecutor()
signal.signal(signal.SIGINT, self.stop_run)
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
def setUp(self):
self.runexecutor = RunExecutor()
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log',
'output_',
text=True)
try:
result = self.runexecutor.execute_run(args, output_filename,
**kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log',
'output_',
text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) +
['--output', output_filename],
stderr=DEVNULL,
**kwargs).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result = {
key.strip(): value.strip()
for (key, _, value) in (line.partition('=')
for line in runexec_output.splitlines())
}
return (result, output_lines)
def check_result_keys(self, result, *additional_keys):
expected_keys = {'cputime', 'walltime', 'memory', 'exitcode'}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith('cputime-cpu'):
self.assertRegex(key, '^cputime-cpu[0-9]+$',
'unexpected result value ' + key)
else:
self.assertIn(key, expected_keys,
'unexpected result value ' + key)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(output[0], '/bin/echo TEST_TOKEN',
'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN',
'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0,
'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'],
0.2,
delta=0.2,
msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'],
0.2,
delta=0.2,
msg='cputime of /bin/echo not as expected')
self.check_result_keys(result)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run(
'/bin/sh',
'-c',
'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9,
'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime',
'termination reason is not "cputime"')
self.assertAlmostEqual(
result['walltime'],
1.4,
delta=0.5,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
1.4,
delta=0.5,
msg=
'cputime is not approximately the time after which the process should have been killed'
)
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run(
'/bin/sh',
'-c',
'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(
str(e),
'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15,
'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft',
'termination reason is not "cputime-soft"')
self.assertAlmostEqual(
result['walltime'],
4,
delta=3,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
4,
delta=3,
msg=
'cputime is not approximately the time after which the process should have been killed'
)
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep',
'10',
walltimelimit=1)
except SystemExit as e:
self.assertEqual(
str(e),
'Wall time limit is not implemented for systems without cpuacct cgroup.'
)
self.skipTest(e)
self.assertEqual(result['exitcode'], 9,
'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime',
'termination reason is not "walltime"')
self.assertAlmostEqual(
result['walltime'],
4,
delta=3,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
0.2,
delta=0.2,
msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.assertEqual(output[0], '/bin/sleep 10',
'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run(
'/bin/sh',
'-c',
'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1,
walltimelimit=5)
self.assertEqual(result['exitcode'], 9,
'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime',
'termination reason is not "cputime"')
self.assertAlmostEqual(
result['walltime'],
1.4,
delta=0.5,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
1.4,
delta=0.5,
msg=
'cputime is not approximately the time after which the process should have been killed'
)
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run(
'/bin/sh',
'-c',
'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1,
hardtimelimit=2,
walltimelimit=5)
except SystemExit as e:
self.assertEqual(
str(e),
'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15,
'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft',
'termination reason is not "cputime-soft"')
self.assertAlmostEqual(
result['walltime'],
1.4,
delta=0.5,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
1.4,
delta=0.5,
msg=
'cputime is not approximately the time after which the process should have been killed'
)
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(
str(e),
'Wall time limit is not implemented for systems without cpuacct cgroup.'
)
self.skipTest(e)
self.assertEqual(result['exitcode'], 0,
'exit code of process is not 0')
self.assertAlmostEqual(
result['walltime'],
0.2,
delta=0.2,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(
result['cputime'],
0.2,
delta=0.2,
msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.assertEqual(output[0], '/bin/cat',
'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat',
stdin=tmp,
walltimelimit=1)
except SystemExit as e:
self.assertEqual(
str(e),
'Wall time limit is not implemented for systems without cpuacct cgroup.'
)
self.skipTest(e)
self.assertEqual(result['exitcode'], 0,
'exit code of process is not 0')
self.assertAlmostEqual(
result['walltime'],
0.2,
delta=0.2,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(
result['cputime'],
0.2,
delta=0.2,
msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.assertEqual(output[0], '/bin/cat',
'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN',
'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log',
'output_',
text=True)
cmd = [
runexec, '--input', '-', '--output', output_filename, '--walltime',
'1', '/bin/cat'
]
try:
process = subprocess.Popen(args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode,
process.args,
output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result = {
key.strip(): value.strip()
for (key, _,
value) in (line.partition('=')
for line in runexec_output.decode().splitlines())
}
self.assertEqual(int(result['exitcode']), 0,
'exit code of process is not 0')
self.assertAlmostEqual(
float(result['walltime'].rstrip('s')),
0.2,
delta=0.2,
msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(
float(result['cputime'].rstrip('s')),
0.2,
delta=0.2,
msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result, 'returnvalue')
self.assertEqual(output[0], '/bin/cat',
'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN',
'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9,
'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed',
'termination reason is not "killed"')
self.assertAlmostEqual(
result['walltime'],
1,
delta=0.5,
msg=
'walltime is not approximately the time after which the process should have been killed'
)
self.assertAlmostEqual(
result['cputime'],
0.2,
delta=0.2,
msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.assertEqual(output[0], '/bin/sleep 10',
'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
_reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name),
limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name),
self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0,
'exit code of /bin/echo is not zero')
self.check_result_keys(result, 'returnvalue')
self.assertEqual(output[0], '/bin/echo TEST_TOKEN',
'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN',
'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user)
self.setDaemon(True)
self.start()
def run(self):
while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT:
currentRun = _Worker.working_queue.get_nowait()
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BaseException as e:
logging.exception('Exception during run execution')
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
memlimit = benchmark.rlimits.get(MEMLIMIT)
args = run.cmdline()
logging.debug('Command line of run is %s', args)
result = \
self.run_executor.execute_run(
args, run.log_file,
hardtimelimit=benchmark.rlimits.get(TIMELIMIT),
softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT),
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=memlimit,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize)
for key, value in result.items():
if key == 'walltime':
run.walltime = value
elif key == 'cputime':
run.cputime = value
elif key == 'memory':
run.values['memUsage'] = result['memory']
elif key == 'energy':
for ekey, evalue in value.items():
run.values['energy-'+ekey] = evalue
else:
run.values['@' + key] = value
if self.my_cpus:
run.values['@cpuCores'] = self.my_cpus
if self.my_memory_nodes:
run.values['@memoryNodes'] = self.my_memory_nodes
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
run.walltime = 0
run.cputime = 0
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
run.after_execution(result['exitcode'], termination_reason=result.get('terminationreason', None))
self.output_handler.output_after_run(run)
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def cleanup(self):
self.run_executor.check_for_new_files_in_home()
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = queue.Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, output_handler,
run_finished_callback):
threading.Thread.__init__(self) # constuctor of superclass
self.run_finished_callback = run_finished_callback
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run(self):
while not STOPPED_BY_INTERRUPT:
try:
currentRun = _Worker.working_queue.get_nowait()
except queue.Empty:
return
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BenchExecException as e:
logging.critical(e)
except BaseException:
logging.exception("Exception during run execution")
self.run_finished_callback()
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
args = run.cmdline()
logging.debug("Command line of run is %s", args)
pqos = Pqos()
if self.my_cpus:
pqos.start_monitoring([self.my_cpus])
run_result = self.run_executor.execute_run(
args,
output_filename=run.log_file,
output_dir=run.result_files_folder,
result_files_patterns=benchmark.result_files_patterns,
hardtimelimit=benchmark.rlimits.cputime_hard,
softtimelimit=benchmark.rlimits.cputime,
walltimelimit=benchmark.rlimits.walltime,
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=benchmark.rlimits.memory,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize,
files_count_limit=benchmark.config.filesCountLimit,
files_size_limit=benchmark.config.filesSizeLimit,
)
mon_data = pqos.stop_monitoring()
run_result.update(mon_data)
if not mon_data:
logging.debug(
"Could not monitor cache and memory bandwidth events for run: %s",
run.identifier,
)
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
if self.my_cpus:
run_result["cpuCores"] = self.my_cpus
if self.my_memory_nodes:
run_result["memoryNodes"] = self.my_memory_nodes
run.set_result(run_result)
self.output_handler.output_after_run(run)
return None
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def setUp(self):
self.runexecutor = RunExecutor()
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user,
output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user,
**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run(self):
while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT:
currentRun = _Worker.working_queue.get_nowait()
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BenchExecException as e:
logging.critical(e)
except BaseException as e:
logging.exception('Exception during run execution')
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
memlimit = benchmark.rlimits.get(MEMLIMIT)
args = run.cmdline()
logging.debug('Command line of run is %s', args)
run_result = \
self.run_executor.execute_run(
args,
output_filename=run.log_file,
output_dir=run.result_files_folder,
result_files_patterns=benchmark.result_files_patterns,
hardtimelimit=benchmark.rlimits.get(TIMELIMIT),
softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT),
walltimelimit=benchmark.rlimits.get(WALLTIMELIMIT),
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=memlimit,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize,
files_count_limit=benchmark.config.filesCountLimit,
files_size_limit=benchmark.config.filesSizeLimit)
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
if self.my_cpus:
run_result['cpuCores'] = self.my_cpus
if self.my_memory_nodes:
run_result['memoryNodes'] = self.my_memory_nodes
run.set_result(run_result)
self.output_handler.output_after_run(run)
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def cleanup(self):
self.run_executor.check_for_new_files_in_home()
def test_require_cgroup_invalid(self):
self.assertRaisesRegex(SystemExit, '.*invalid.*',
lambda: RunExecutor(additional_cgroup_subsystems=['invalid']))
class TestRunExecutorWithSudo(TestRunExecutor):
"""
Run tests using the sudo mode of RunExecutor, if possible.
sudo is typically set up to allow executing as our own user,
so we try that. Note that this will not catch all problems,
for example if we forget to use "sudo kill" to send a signal
and instead send it directly, but requiring a second user for tests
would not be good, either.
"""
# Use user name defined in environment variable if present,
# or fall back to current user (sudo always allows this).
# sudo allows refering to numerical uids with '#'.
user = os.environ.get('BENCHEXEC_TEST_USER', '#' + str(os.getuid()))
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(user=self.user, *args, **kwargs)
except SystemExit as e:
# sudo seems not to be available
self.skipTest(e)
def check_exitcode(self, result, expected, msg=None):
actual = int(result['exitcode'])
if expected == 15 and actual == 0:
# On Ubuntu 16.04, sudo returns 0 if process is killed with signal 15
return
# Using sudo may affect the exit code:
# what was the returnsignal is now the returnvalue.
# The distinction between returnsignal and returnvalue of the actual
# process is lost.
# If the returnsignal (of the sudo process) is 0,
# we replace the exit code with the mixed returnsignal/returnvalue of
# the actual process (with bit for core dump cleared).
returnsignal = actual & 0x7F
returnvalue = (actual >> 8) & 0x7F
if returnsignal == 0:
actual = returnvalue
self.assertEqual(actual, expected, msg)
def check_command_in_output(self, output, cmd):
self.assertTrue(output[0].endswith(cmd),
'run output misses executed command')
def test_detect_new_files_in_home(self):
if not os.path.exists('/usr/bin/mktemp'):
self.skipTest('missing /usr/bin/mktemp')
home_dir = runexecutor._get_user_account_info(self.user).pw_dir
tmp_file_pattern = '.BenchExec_test_runexecutor_' + unichr(
0xe4) + unichr(0xf6) + unichr(0xfc) + '_XXXXXXXXXX'
(result, output) = self.execute_run('/usr/bin/mktemp',
'--tmpdir=' + home_dir,
tmp_file_pattern)
try:
self.check_exitcode(result, 0,
'exit code of /usr/bin/mktemp is not zero')
tmp_file = output[-1]
self.assertIn(
tmp_file, self.runexecutor.check_for_new_files_in_home(),
'runexecutor failed to detect new temporary file in home directory'
)
finally:
subprocess.check_call(
self.runexecutor._build_cmdline(['rm', tmp_file]))
def test_append_environment_variable(self):
# sudo-mode has a suboptimal implementation for additionalEnv:
# If an environment variable is not modified, it will be cleared completely and in case of
# PATH sudo will set it. If PATH is specified in additionalEnv, we will copy the value
# from the current process (which is different than what sudo would set)
# and append the given string.
pass
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
def setUp(self):
self.runexecutor = RunExecutor()
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(args, output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, process.args, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'returnvalue'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
_reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'returnvalue'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(user=self.user, *args, **kwargs)
except SystemExit as e:
# sudo seems not to be available
self.skipTest(e)
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(user=self.user, *args, **kwargs)
except SystemExit as e:
# sudo seems not to be available
self.skipTest(e)
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
if not hasattr(cls, 'assertRaisesRegex'):
cls.assertRaisesRegex = cls.assertRaisesRegexp
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(list(args), output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def check_command_in_output(self, output, cmd):
self.assertEqual(output[0], cmd, 'run output misses executed command')
def check_result_keys(self, result, *additional_keys):
expected_keys = {'cputime', 'walltime', 'memory', 'exitcode'}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith('cputime-cpu'):
self.assertRegex(key, '^cputime-cpu[0-9]+$', 'unexpected result value ' + key)
else:
self.assertIn(key, expected_keys, 'unexpected result value ' + key)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
self.check_result_keys(result)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, cmd, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
runexecutor._reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_home_and_tmp_is_separate(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
self.assertRegex(output[-1], '/BenchExec_run_[^/]*/home .*/BenchExec_run_[^/]*/tmp',
'HOME or TMPDIR variable does not contain expected temporary directory')
def test_temp_dirs_are_removed(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
home_dir = output[-1].split(' ')[0]
temp_dir = output[-1].split(' ')[1]
self.assertFalse(os.path.exists(home_dir),
'temporary home directory {} was not cleaned up'.format(home_dir))
self.assertFalse(os.path.exists(temp_dir),
'temporary temp directory {} was not cleaned up'.format(temp_dir))
def test_no_cleanup_temp(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
self.setUp(cleanup_temp_dir=False) # create RunExecutor with desired parameter
(result, output) = self.execute_run('/bin/sh', '-c', 'echo "$TMPDIR"; echo "" > "$TMPDIR/test"')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
temp_dir = output[-1]
test_file = os.path.join(temp_dir, 'test')
subprocess.check_call(self.runexecutor._build_cmdline(['test', '-f', test_file]))
self.assertEqual('tmp', os.path.basename(temp_dir), 'unexpected name of temp dir')
self.assertNotEqual('/tmp', temp_dir, 'temp dir should not be the global temp dir')
subprocess.check_call(self.runexecutor._build_cmdline(['rm', '-r', os.path.dirname(temp_dir)]))
def test_require_cgroup_invalid(self):
self.assertRaisesRegex(SystemExit, '.*invalid.*',
lambda: RunExecutor(additional_cgroup_subsystems=['invalid']))
def test_require_cgroup_cpu(self):
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(result, output) = self.execute_run('/bin/cat', '/proc/self/cgroup')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/cat is not zero')
for line in output:
if re.match('^[0-9]*:cpu:/(.*/)?benchmark_.*$',line):
return # Success
self.fail('Not in expected cgroup for subsystem cpu:\n' + '\n'.join(output))
def test_set_cgroup_cpu_shares(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
(result, _) = self.execute_run('/bin/echo',
cgroupValues={('cpu', 'shares'): 42})
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
class TestRunExecutorWithSudo(TestRunExecutor):
"""
Run tests using the sudo mode of RunExecutor, if possible.
sudo is typically set up to allow executing as our own user,
so we try that. Note that this will not catch all problems,
for example if we forget to use "sudo kill" to send a signal
and instead send it directly, but requiring a second user for tests
would not be good, either.
"""
# Use user name defined in environment variable if present,
# or fall back to current user (sudo always allows this).
# sudo allows refering to numerical uids with '#'.
user = os.environ.get('BENCHEXEC_TEST_USER', '#' + str(os.getuid()))
def setUp(self, *args, **kwargs):
try:
self.runexecutor = RunExecutor(user=self.user, *args, **kwargs)
except SystemExit as e:
# sudo seems not to be available
self.skipTest(e)
def execute_run(self, *args, **kwargs):
result, output = super(TestRunExecutorWithSudo, self).execute_run(*args, **kwargs)
self.fix_exitcode(result)
return (result, output)
def execute_run_extern(self, *args, **kwargs):
result, output = super(TestRunExecutorWithSudo, self) \
.execute_run_extern('--user', self.user, *args, **kwargs)
self.fix_exitcode(result)
return (result, output)
def fix_exitcode(self, result):
# Using sudo may affect the exit code:
# what was the returnsignal is now the returnvalue.
# The distinction between returnsignal and returnvalue of the actual
# process is lost.
# If the returnsignal (of the sudo process) is 0,
# we replace the exit code with the mixed returnsignal/returnvalue of
# the actual process (with bit for core dump cleared).
exitcode = int(result['exitcode'])
returnsignal = exitcode & 0x7F
returnvalue = (exitcode >> 8) & 0x7F
if returnsignal == 0:
result['exitcode'] = returnvalue
def check_command_in_output(self, output, cmd):
self.assertTrue(output[0].endswith(cmd), 'run output misses executed command')
def test_detect_new_files_in_home(self):
if not os.path.exists('/usr/bin/mktemp'):
self.skipTest('missing /usr/bin/mktemp')
home_dir = runexecutor._get_user_account_info(self.user).pw_dir
tmp_file_pattern = '.BenchExec_test_runexecutor_'+unichr(0xe4)+unichr(0xf6)+unichr(0xfc)+'_XXXXXXXXXX'
(result, output) = self.execute_run(
'/usr/bin/mktemp', '--tmpdir=' + home_dir, tmp_file_pattern)
try:
self.assertEqual(int(result['exitcode']), 0, 'exit code of /usr/bin/mktemp is not zero')
tmp_file = output[-1]
self.assertIn(tmp_file, self.runexecutor.check_for_new_files_in_home(),
'runexecutor failed to detect new temporary file in home directory')
finally:
subprocess.check_call(self.runexecutor._build_cmdline('rm', tmp_file))
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)