def check_cgroup_availability(wait=1):
"""
Basic utility to check the availability and permissions of cgroups.
This will log some warnings for the user if necessary.
On some systems, daemons such as cgrulesengd might interfere with the cgroups
of a process soon after it was started. Thus this function starts a process,
waits a configurable amount of time, and check whether the cgroups have been changed.
@param wait: a non-negative int that is interpreted as seconds to wait during the check
@raise SystemExit: if cgroups are not usable
"""
logging.basicConfig(format="%(levelname)s: %(message)s")
runexecutor = RunExecutor(use_namespaces=False)
my_cgroups = runexecutor.cgroups
if not (
CPUACCT in my_cgroups
and CPUSET in my_cgroups
# and FREEZER in my_cgroups # For now, we do not require freezer
and MEMORY in my_cgroups
):
sys.exit(1)
with tempfile.NamedTemporaryFile(mode="rt") as tmp:
runexecutor.execute_run(
["sh", "-c", "sleep {0}; cat /proc/self/cgroup".format(wait)],
tmp.name,
memlimit=1024 * 1024, # set memlimit to force check for swapaccount
# set cores and memory_nodes to force usage of CPUSET
cores=util.parse_int_list(my_cgroups.get_value(CPUSET, "cpus")),
memory_nodes=my_cgroups.read_allowed_memory_banks(),
)
lines = []
for line in tmp:
line = line.strip()
if (
line
and not line == "sh -c 'sleep {0}; cat /proc/self/cgroup'".format(wait)
and not all(c == "-" for c in line)
):
lines.append(line)
task_cgroups = find_my_cgroups(lines)
fail = False
for subsystem in CPUACCT, CPUSET, MEMORY, FREEZER:
if subsystem in my_cgroups:
if not task_cgroups[subsystem].startswith(
os.path.join(my_cgroups[subsystem], "benchmark_")
):
logging.warning(
"Task was in cgroup %s for subsystem %s, "
"which is not the expected sub-cgroup of %s. "
"Maybe some other program is interfering with cgroup management?",
task_cgroups[subsystem],
subsystem,
my_cgroups[subsystem],
)
fail = True
if fail:
sys.exit(1)
def check_cgroup_availability(wait=1):
"""
Basic utility to check the availability and permissions of cgroups.
This will log some warnings for the user if necessary.
On some systems, daemons such as cgrulesengd might interfere with the cgroups
of a process soon after it was started. Thus this function starts a process,
waits a configurable amount of time, and check whether the cgroups have been changed.
@param wait: a non-negative int that is interpreted as seconds to wait during the check
@raise SystemExit: if cgroups are not usable
"""
logging.basicConfig(format="%(levelname)s: %(message)s")
runexecutor = RunExecutor()
my_cgroups = runexecutor.cgroups
if not (CPUACCT in my_cgroups and
CPUSET in my_cgroups and
# FREEZER in my_cgroups and # For now, we do not require freezer
MEMORY in my_cgroups):
sys.exit(1)
with tempfile.NamedTemporaryFile(mode='rt') as tmp:
runexecutor.execute_run(['sh', '-c', 'sleep {0}; cat /proc/self/cgroup'.format(wait)], tmp.name,
memlimit=1024*1024, # set memlimit to force check for swapaccount
# set cores and memory_nodes to force usage of CPUSET
cores=util.parse_int_list(my_cgroups.get_value(CPUSET, 'cpus')),
memory_nodes=my_cgroups.read_allowed_memory_banks())
lines = []
for line in tmp:
line = line.strip()
if line and not line == "sh -c 'sleep {0}; cat /proc/self/cgroup'".format(wait) \
and not all(c == '-' for c in line):
lines.append(line)
task_cgroups = find_my_cgroups(lines)
fail = False
for subsystem in CPUACCT, CPUSET, MEMORY, FREEZER:
if subsystem in my_cgroups:
if not task_cgroups[subsystem].startswith(os.path.join(my_cgroups[subsystem], 'benchmark_')):
logging.warning('Task was in cgroup %s for subsystem %s, '
'which is not the expected sub-cgroup of %s. '
'Maybe some other program is interfering with cgroup management?',
task_cgroups[subsystem], subsystem, my_cgroups[subsystem])
fail = True
if fail:
sys.exit(1)
def run_exec_cmd(
environment: Environment,
cmd: Sequence[str],
env: Mapping[str, str],
cwd: Path,
dir_modes: Mapping[str, Any],
tmp_dir: Path,
time_limit: int,
mem_limit: int,
) -> Tuple[Tuple[str, ...], RunexecStats, str, str]:
stdout = tmp_dir / "info.log"
if stdout.exists():
stdout.unlink()
stderr = tmp_dir / "error.log"
if stderr.exists():
stderr.unlink()
combined_cmd = combine_cmd(*environment.run(cmd, env, cwd))
stderr.write_text(" \\\n ".join(
shlex.quote(part) for part in combined_cmd) + "\n\n")
run_executor = RunExecutor(
use_namespaces=False,
# dir_modes=dir_modes,
# network_access=True,
# Need to system config so DNS works.
# container_system_config=True,
)
logger = logging.getLogger("root")
with runexec_catch_signals(run_executor), capture_logs(
logger, logging.WARNING) as logs:
run_exec_run = run_executor.execute_run(
args=combined_cmd,
environments={
"keepEnv": {},
},
workingDir="/",
write_header=False,
output_filename=stdout,
error_filename=stderr,
softtimelimit=time_limit,
hardtimelimit=int(time_limit * 1.1),
walltimelimit=int(time_limit * 1.2),
memlimit=mem_limit,
)
return (
combined_cmd,
RunexecStats.create(run_exec_run),
stdout.read_text(),
stderr.read_text() + "\n".join(record.getMessage() for record in logs),
)
class Runner():
"""Base class to run runxec"""
def __init__(self):
self.executor = RunExecutor()
signal.signal(signal.SIGINT, self.stop_run)
def stop_run(self):
self.executor.stop()
def _runexec_args(self):
return []
def _run(self, output):
return self.executor.execute_run(args=self._runexec_args(),
output_filename=output)
# Should I spawn a process?
def run(self, output):
return self._run(output)
def validate(self):
raise NotImplementedError()
def _run_command(self,
command: str,
time: float,
memory=None,
file_count=None,
stdin=None):
"""Runs specified command
Arguments:
command {str} -- command to be run
time {float} -- time limit in seconds
Keyword Arguments:
memory {int} -- memory limit in bytes (default: {None})
file_count {int} -- maximum number of files to write (default: {None})
stdin {file} -- file used as standard input (default: {None})
Returns:
OrderedDict -- dict with results
"""
executor = RunExecutor(use_namespaces=False)
if memory is None:
memory = self._limits.max_memory
args = command.split(' ')
try:
result = executor.execute_run(args,
self._output_log_file,
memlimit=memory,
hardtimelimit=time,
walltimelimit=time,
files_count_limit=file_count,
stdin=stdin)
if 'cputime' in result and result['cputime'] > time:
result['terminationreason'] = 'cputime'
except:
result = {'terminationreason': 'something went very wrong'}
return result
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, my_user, output_handler):
threading.Thread.__init__(self) # constuctor of superclass
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(user=my_user)
self.setDaemon(True)
self.start()
def run(self):
while not _Worker.working_queue.empty() and not STOPPED_BY_INTERRUPT:
currentRun = _Worker.working_queue.get_nowait()
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BaseException as e:
logging.exception('Exception during run execution')
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
memlimit = benchmark.rlimits.get(MEMLIMIT)
args = run.cmdline()
logging.debug('Command line of run is %s', args)
result = \
self.run_executor.execute_run(
args, run.log_file,
hardtimelimit=benchmark.rlimits.get(TIMELIMIT),
softtimelimit=benchmark.rlimits.get(SOFTTIMELIMIT),
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=memlimit,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize)
for key, value in result.items():
if key == 'walltime':
run.walltime = value
elif key == 'cputime':
run.cputime = value
elif key == 'memory':
run.values['memUsage'] = result['memory']
elif key == 'energy':
for ekey, evalue in value.items():
run.values['energy-'+ekey] = evalue
else:
run.values['@' + key] = value
if self.my_cpus:
run.values['@cpuCores'] = self.my_cpus
if self.my_memory_nodes:
run.values['@memoryNodes'] = self.my_memory_nodes
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
run.walltime = 0
run.cputime = 0
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
run.after_execution(result['exitcode'], termination_reason=result.get('terminationreason', None))
self.output_handler.output_after_run(run)
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
def cleanup(self):
self.run_executor.check_for_new_files_in_home()
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
if not hasattr(cls, 'assertRaisesRegex'):
cls.assertRaisesRegex = cls.assertRaisesRegexp
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(list(args), output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def check_command_in_output(self, output, cmd):
self.assertEqual(output[0], cmd, 'run output misses executed command')
def check_result_keys(self, result, *additional_keys):
expected_keys = {'cputime', 'walltime', 'memory', 'exitcode',
'energy', 'energy-cpu', 'energy-core', 'energy-uncore'}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith('cputime-cpu'):
self.assertRegex(key, '^cputime-cpu[0-9]+$', 'unexpected result value ' + key)
else:
self.assertIn(key, expected_keys, 'unexpected result value ' + key)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
self.check_result_keys(result)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, cmd, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
runexecutor._reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_home_and_tmp_is_separate(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
self.assertRegex(output[-1], '/BenchExec_run_[^/]*/home .*/BenchExec_run_[^/]*/tmp',
'HOME or TMPDIR variable does not contain expected temporary directory')
def test_temp_dirs_are_removed(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
home_dir = output[-1].split(' ')[0]
temp_dir = output[-1].split(' ')[1]
self.assertFalse(os.path.exists(home_dir),
'temporary home directory {} was not cleaned up'.format(home_dir))
self.assertFalse(os.path.exists(temp_dir),
'temporary temp directory {} was not cleaned up'.format(temp_dir))
def test_no_cleanup_temp(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
self.setUp(cleanup_temp_dir=False) # create RunExecutor with desired parameter
(result, output) = self.execute_run('/bin/sh', '-c', 'echo "$TMPDIR"; echo "" > "$TMPDIR/test"')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
temp_dir = output[-1]
test_file = os.path.join(temp_dir, 'test')
subprocess.check_call(self.runexecutor._build_cmdline(['test', '-f', test_file]))
self.assertEqual('tmp', os.path.basename(temp_dir), 'unexpected name of temp dir')
self.assertNotEqual('/tmp', temp_dir, 'temp dir should not be the global temp dir')
subprocess.check_call(self.runexecutor._build_cmdline(['rm', '-r', os.path.dirname(temp_dir)]))
def test_require_cgroup_invalid(self):
self.assertRaisesRegex(SystemExit, '.*invalid.*',
lambda: RunExecutor(additional_cgroup_subsystems=['invalid']))
def test_require_cgroup_cpu(self):
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(result, output) = self.execute_run('/bin/cat', '/proc/self/cgroup')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/cat is not zero')
for line in output:
if re.match('^[0-9]*:cpu:/(.*/)?benchmark_.*$',line):
return # Success
self.fail('Not in expected cgroup for subsystem cpu:\n' + '\n'.join(output))
def test_set_cgroup_cpu_shares(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
(result, _) = self.execute_run('/bin/echo',
cgroupValues={('cpu', 'shares'): 42})
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
if not hasattr(cls, 'assertRaisesRegex'):
cls.assertRaisesRegex = cls.assertRaisesRegexp
def setUp(self, *args, **kwargs):
self.runexecutor = RunExecutor(*args, **kwargs)
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(list(args), output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def check_command_in_output(self, output, cmd):
self.assertEqual(output[0], cmd, 'run output misses executed command')
def check_result_keys(self, result, *additional_keys):
expected_keys = {'cputime', 'walltime', 'memory', 'exitcode'}
expected_keys.update(additional_keys)
for key in result.keys():
if key.startswith('cputime-cpu'):
self.assertRegex(key, '^cputime-cpu[0-9]+$', 'unexpected result value ' + key)
else:
self.assertIn(key, expected_keys, 'unexpected result value ' + key)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
self.check_result_keys(result)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
self.check_result_keys(result, 'terminationreason')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result)
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, cmd, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/cat')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
self.check_result_keys(result, 'terminationreason')
self.check_command_in_output(output, '/bin/sleep 10')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
runexecutor._reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
runexecutor._reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
self.check_result_keys(result, 'returnvalue')
self.check_command_in_output(output, '/bin/echo TEST_TOKEN')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_home_and_tmp_is_separate(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
self.assertRegex(output[-1], '/BenchExec_run_[^/]*/home .*/BenchExec_run_[^/]*/tmp',
'HOME or TMPDIR variable does not contain expected temporary directory')
def test_temp_dirs_are_removed(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'echo $HOME $TMPDIR')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
home_dir = output[-1].split(' ')[0]
temp_dir = output[-1].split(' ')[1]
self.assertFalse(os.path.exists(home_dir),
'temporary home directory {} was not cleaned up'.format(home_dir))
self.assertFalse(os.path.exists(temp_dir),
'temporary temp directory {} was not cleaned up'.format(temp_dir))
def test_no_cleanup_temp(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
self.setUp(cleanup_temp_dir=False) # create RunExecutor with desired parameter
(result, output) = self.execute_run('/bin/sh', '-c', 'echo "$TMPDIR"; echo "" > "$TMPDIR/test"')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/sh is not zero')
temp_dir = output[-1]
test_file = os.path.join(temp_dir, 'test')
subprocess.check_call(self.runexecutor._build_cmdline(['test', '-f', test_file]))
self.assertEqual('tmp', os.path.basename(temp_dir), 'unexpected name of temp dir')
self.assertNotEqual('/tmp', temp_dir, 'temp dir should not be the global temp dir')
subprocess.check_call(self.runexecutor._build_cmdline(['rm', '-r', os.path.dirname(temp_dir)]))
def test_require_cgroup_invalid(self):
self.assertRaisesRegex(SystemExit, '.*invalid.*',
lambda: RunExecutor(additional_cgroup_subsystems=['invalid']))
def test_require_cgroup_cpu(self):
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(result, output) = self.execute_run('/bin/cat', '/proc/self/cgroup')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/cat is not zero')
for line in output:
if re.match('^[0-9]*:cpu:/(.*/)?benchmark_.*$',line):
return # Success
self.fail('Not in expected cgroup for subsystem cpu:\n' + '\n'.join(output))
def test_set_cgroup_cpu_shares(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
try:
self.setUp(additional_cgroup_subsystems=['cpu'])
except SystemExit as e:
self.skipTest(e)
(result, _) = self.execute_run('/bin/echo',
cgroupValues={('cpu', 'shares'): 42})
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
class _Worker(threading.Thread):
"""
A Worker is a deamonic thread, that takes jobs from the working_queue and runs them.
"""
working_queue = queue.Queue()
def __init__(self, benchmark, my_cpus, my_memory_nodes, output_handler,
run_finished_callback):
threading.Thread.__init__(self) # constuctor of superclass
self.run_finished_callback = run_finished_callback
self.benchmark = benchmark
self.my_cpus = my_cpus
self.my_memory_nodes = my_memory_nodes
self.output_handler = output_handler
self.run_executor = RunExecutor(**benchmark.config.containerargs)
self.setDaemon(True)
self.start()
def run(self):
while not STOPPED_BY_INTERRUPT:
try:
currentRun = _Worker.working_queue.get_nowait()
except queue.Empty:
return
try:
logging.debug('Executing run "%s"', currentRun.identifier)
self.execute(currentRun)
logging.debug('Finished run "%s"', currentRun.identifier)
except SystemExit as e:
logging.critical(e)
except BenchExecException as e:
logging.critical(e)
except BaseException:
logging.exception("Exception during run execution")
self.run_finished_callback()
_Worker.working_queue.task_done()
def execute(self, run):
"""
This function executes the tool with a sourcefile with options.
It also calls functions for output before and after the run.
"""
self.output_handler.output_before_run(run)
benchmark = self.benchmark
args = run.cmdline()
logging.debug("Command line of run is %s", args)
pqos = Pqos()
if self.my_cpus:
pqos.start_monitoring([self.my_cpus])
run_result = self.run_executor.execute_run(
args,
output_filename=run.log_file,
output_dir=run.result_files_folder,
result_files_patterns=benchmark.result_files_patterns,
hardtimelimit=benchmark.rlimits.cputime_hard,
softtimelimit=benchmark.rlimits.cputime,
walltimelimit=benchmark.rlimits.walltime,
cores=self.my_cpus,
memory_nodes=self.my_memory_nodes,
memlimit=benchmark.rlimits.memory,
environments=benchmark.environment(),
workingDir=benchmark.working_directory(),
maxLogfileSize=benchmark.config.maxLogfileSize,
files_count_limit=benchmark.config.filesCountLimit,
files_size_limit=benchmark.config.filesSizeLimit,
)
mon_data = pqos.stop_monitoring()
run_result.update(mon_data)
if not mon_data:
logging.debug(
"Could not monitor cache and memory bandwidth events for run: %s",
run.identifier,
)
if self.run_executor.PROCESS_KILLED:
# If the run was interrupted, we ignore the result and cleanup.
try:
if benchmark.config.debug:
os.rename(run.log_file, run.log_file + ".killed")
else:
os.remove(run.log_file)
except OSError:
pass
return 1
if self.my_cpus:
run_result["cpuCores"] = self.my_cpus
if self.my_memory_nodes:
run_result["memoryNodes"] = self.my_memory_nodes
run.set_result(run_result)
self.output_handler.output_after_run(run)
return None
def stop(self):
# asynchronous call to runexecutor,
# the worker will stop asap, but not within this method.
self.run_executor.stop()
class TestRunExecutor(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.longMessage = True
cls.maxDiff = None
logging.disable(logging.CRITICAL)
if not hasattr(cls, 'assertRegex'):
cls.assertRegex = cls.assertRegexpMatches
def setUp(self):
self.runexecutor = RunExecutor()
def execute_run(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
result = self.runexecutor.execute_run(args, output_filename, **kwargs)
output_lines = os.read(output_fd, 4096).decode().splitlines()
return (result, output_lines)
finally:
os.close(output_fd)
os.remove(output_filename)
def execute_run_extern(self, *args, **kwargs):
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
try:
runexec_output = subprocess.check_output(
args=[python, runexec] + list(args) + ['--output', output_filename],
stderr=DEVNULL,
**kwargs
).decode()
output_lines = os.read(output_fd, 4096).decode().splitlines()
except subprocess.CalledProcessError as e:
print(e.output.decode())
raise e
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
return (result, output_lines)
def test_command_output(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_command_result(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
def test_cputime_hardlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_softlimit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_walltime_limit(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
try:
(result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_cputime_walltime_limit(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
hardtimelimit=1, walltimelimit=5)
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
if 'terminationreason' in result:
# not produced currently if killed by ulimit
self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_all_timelimits(self):
if not os.path.exists('/bin/sh'):
self.skipTest('missing /bin/sh')
try:
(result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
softtimelimit=1, hardtimelimit=2, walltimelimit=5)
except SystemExit as e:
self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_devnull(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
try:
(result, output) = self.execute_run('/bin/cat', walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_file(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
with tempfile.TemporaryFile() as tmp:
tmp.write(b'TEST_TOKEN')
tmp.flush()
tmp.seek(0)
try:
(result, output) = self.execute_run('/bin/cat', stdin=tmp, walltimelimit=1)
except SystemExit as e:
self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
self.skipTest(e)
self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_input_is_redirected_from_stdin(self):
if not os.path.exists('/bin/cat'):
self.skipTest('missing /bin/cat')
(output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
cmd = [runexec, '--input', '-', '--output', output_filename, '--walltime', '1', '/bin/cat']
try:
process = subprocess.Popen(args=cmd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=DEVNULL)
try:
runexec_output, unused_err = process.communicate(b'TEST_TOKEN')
except:
process.kill()
process.wait()
raise
retcode = process.poll()
if retcode:
print(runexec_output.decode())
raise subprocess.CalledProcessError(retcode, process.args, output=runexec_output)
output = os.read(output_fd, 4096).decode().splitlines()
finally:
os.close(output_fd)
os.remove(output_filename)
result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.decode().splitlines())}
self.assertEqual(int(result['exitcode']), 0, 'exit code of process is not 0')
self.assertAlmostEqual(float(result['walltime'].rstrip('s')), 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
self.assertAlmostEqual(float(result['cputime'].rstrip('s')), 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'returnvalue'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_stop_run(self):
if not os.path.exists('/bin/sleep'):
self.skipTest('missing /bin/sleep')
thread = _StopRunThread(1, self.runexecutor)
thread.start()
(result, output) = self.execute_run('/bin/sleep', '10')
thread.join()
self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
for line in output[1:]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')
def test_reduce_file_size_empty_file(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_empty_file2(self):
with tempfile.NamedTemporaryFile() as tmp:
_reduce_file_size_if_necessary(tmp.name, 500)
self.assertEqual(os.path.getsize(tmp.name), 0)
def test_reduce_file_size_long_line_not_truncated(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
content = 'Long line ' * 500
tmp.write(content)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 500)
with open(tmp.name, 'rt') as tmp2:
self.assertMultiLineEqual(tmp2.read(), content)
REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
REDUCE_OVERHEAD = 100
def test_reduce_file_size(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
limit = 500
_reduce_file_size_if_necessary(tmp.name, limit)
self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
self.assertTrue(new_content.endswith(line))
def test_reduce_file_size_limit_zero(self):
with tempfile.NamedTemporaryFile(mode='wt') as tmp:
line = 'Some text\n'
tmp.write(line * 500)
tmp.flush()
_reduce_file_size_if_necessary(tmp.name, 0)
self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
with open(tmp.name, 'rt') as tmp2:
new_content = tmp2.read()
self.assertIn(self.REDUCE_WARNING_MSG, new_content)
self.assertTrue(new_content.startswith(line))
def test_integration(self):
if not os.path.exists('/bin/echo'):
self.skipTest('missing /bin/echo')
(result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
for key in result.keys():
self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'returnvalue'}, 'unexpected result value ' + key)
self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
for line in output[1:-1]:
self.assertRegex(line, '^-*$', 'unexpected text in run output')