def test_check_and_parse_execution_results():
stdout = "[0.000403]"
stderr = "[iterations_runner.py] iteration 1/1"
assert check_and_parse_execution_results(stdout, stderr, 0) == json.loads(stdout)
# Non-zero return code.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 1)
expected = """Benchmark returned non-zero or didn't emit JSON list. return code: 1
stdout:
--------------------------------------------------
[0.000403]
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
# Corrupt Json in STDOUT.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results("[0.000403[", stderr, 0)
expected = """Benchmark returned non-zero or didn't emit JSON list. Exception string: Expecting , delimiter: line 1 column 10 (char 9)
return code: 0
stdout:
--------------------------------------------------
[0.000403[
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
def test_check_and_parse_execution_results0005(caplog):
stdout = json.dumps({
"wallclock_times": [.5, .5],
"core_cycle_counts": [[1000, 500], [50, 50], [30, 30], [40, 40]],
"aperf_counts": [[20, 15], [5, 5], [5, 5], [5, 5]],
"mperf_counts": [[20, 20], [5, 5], [5, 5], [5, 5]],
})
stderr = "[iterations_runner.py] iteration 1/1"
with pytest.raises(RerunExecution) as e:
check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
assert e.value.message == \
"APERF/MPERF ratio badness detected\n" \
" in_proc_iter=1, core=0, type=throttle, ratio=0.75\n\n" \
"The process execution will be retried until the ratios are OK."
def test_check_and_parse_execution_results0005(caplog):
stdout = json.dumps({
"wallclock_times": [.5, .5],
"core_cycle_counts": [[1000, 500], [50, 50], [30, 30], [40, 40]],
"aperf_counts": [[20, 15], [5, 5], [5, 5], [5, 5]],
"mperf_counts": [[20, 20], [5, 5], [5, 5], [5, 5]],
})
stderr = "[iterations_runner.py] iteration 1/1"
with pytest.raises(RerunExecution) as e:
check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
assert e.value.message == \
"APERF/MPERF ratio badness detected\n" \
" in_proc_iter=1, core=0, type=throttle, ratio=0.75\n\n" \
"The process execution will be retried until the ratios are OK."
def run(self, mailer, dry_run=False):
"""Runs this job (execution)"""
entry_point = self.sched.config.VARIANTS[self.variant]
vm_def = self.vm_info["vm_def"]
vm_def.dry_run = dry_run
# Set heap limit
heap_limit_kb = self.sched.config.HEAP_LIMIT
stack_limit_kb = self.sched.config.STACK_LIMIT
stdout, stderr, rc = vm_def.run_exec(entry_point, self.benchmark,
self.vm_info["n_iterations"],
self.parameter, heap_limit_kb,
stack_limit_kb)
if not dry_run:
try:
iterations_results = util.check_and_parse_execution_results(
stdout, stderr, rc)
except util.ExecutionFailed as e:
util.log_and_mail(mailer, error,
"Benchmark failure: %s" % self.key,
e.message)
iterations_results = []
else:
iterations_results = []
# We print the status *after* benchmarking, so that I/O cannot be
# commited during benchmarking. In production, we will be rebooting
# before the next execution, so we are grand.
info("Finished '%s(%d)' (%s variant) under '%s'" %
(self.benchmark, self.parameter, self.variant, self.vm_name))
return iterations_results
def run(self, mailer, dry_run=False):
"""Runs this job (execution)"""
entry_point = self.sched.config.VARIANTS[self.variant]
vm_def = self.vm_info["vm_def"]
vm_def.dry_run = dry_run
# Set heap limit
heap_limit_kb = self.sched.config.HEAP_LIMIT
stack_limit_kb = self.sched.config.STACK_LIMIT
stdout, stderr, rc = vm_def.run_exec(
entry_point, self.benchmark, self.vm_info["n_iterations"],
self.parameter, heap_limit_kb, stack_limit_kb)
if not dry_run:
try:
iterations_results = util.check_and_parse_execution_results(stdout, stderr, rc)
except util.ExecutionFailed as e:
util.log_and_mail(mailer, error, "Benchmark failure: %s" % self.key, e.message)
iterations_results = []
else:
iterations_results = []
# We print the status *after* benchmarking, so that I/O cannot be
# commited during benchmarking. In production, we will be rebooting
# before the next execution, so we are grand.
info("Finished '%s(%d)' (%s variant) under '%s'" %
(self.benchmark, self.parameter, self.variant, self.vm_name))
return iterations_results
def test_check_and_parse_execution_results0004(caplog):
stdout = json.dumps({
"wallclock_times": [1],
"core_cycle_counts": [[1], [1], [1], [1]],
"aperf_counts": [[30], [20], [5], [3]],
"mperf_counts": [[20], [30], [5], [5]],
})
stderr = "[iterations_runner.py] iteration 1/1"
with pytest.raises(RerunExecution) as e:
check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG, "a:b:c")
assert e.value.args[0] == \
"APERF/MPERF ratio badness detected\n" \
" in_proc_iter=0, core=0, type=turbo, ratio=1.5\n" \
" in_proc_iter=0, core=1, type=throttle, ratio=0.666666666667\n\n" \
"The process execution will be retried until the ratios are OK."
def test_check_and_parse_execution_results0003():
stderr = "[iterations_runner.py] iteration 1/1"
stdout = "[0.000403["
# Corrupt JSON in STDOUT.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
expected = """Benchmark returned non-zero or emitted invalid JSON.\nException string: Expecting , delimiter: line 1 column 10 (char 9)
return code: 0
stdout:
--------------------------------------------------
[0.000403[
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
def test_check_and_parse_execution_results0001():
stdout = json.dumps({
"wallclock_times": [1],
"core_cycle_counts": [[1], [2], [3], [4]],
"aperf_counts": [[5], [5], [5], [5]],
"mperf_counts": [[5], [5], [5], [5]],
})
stderr = "[iterations_runner.py] iteration 1/1"
js = check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
assert js == json.loads(stdout)
def test_check_and_parse_execution_results0003():
stderr = "[iterations_runner.py] iteration 1/1"
stdout = "[0.000403["
# Corrupt JSON in STDOUT.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
expected = """Benchmark returned non-zero or emitted invalid JSON.\nException string: Expecting , delimiter: line 1 column 10 (char 9)
return code: 0
stdout:
--------------------------------------------------
[0.000403[
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
def test_check_and_parse_execution_results0001():
stdout = json.dumps({
"wallclock_times": [1],
"core_cycle_counts": [[1], [2], [3], [4]],
"aperf_counts": [[5], [5], [5], [5]],
"mperf_counts": [[5], [5], [5], [5]],
})
stderr = "[iterations_runner.py] iteration 1/1"
js = check_and_parse_execution_results(stdout, stderr, 0, DUMMY_CONFIG)
assert js == json.loads(stdout)
def test_check_and_parse_execution_results():
stdout = "[0.000403]"
stderr = "[iterations_runner.py] iteration 1/1"
assert check_and_parse_execution_results(stdout, stderr,
0) == json.loads(stdout)
# Non-zero return code.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 1)
expected = """Benchmark returned non-zero or didn't emit JSON list. return code: 1
stdout:
--------------------------------------------------
[0.000403]
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
# Corrupt Json in STDOUT.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results("[0.000403[", stderr, 0)
expected = """Benchmark returned non-zero or didn't emit JSON list. Exception string: Expecting , delimiter: line 1 column 10 (char 9)
return code: 0
stdout:
--------------------------------------------------
[0.000403[
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
"""
assert excinfo.value.message == expected
def test_check_and_parse_execution_results0002():
stdout = json.dumps({
"wallclock_times": [123.4],
"core_cycle_counts": [[1], [2], [3], [4]],
"aperf_counts": [[5], [6], [7], [8]],
"mperf_counts": [[9], [10], [11], [12]],
})
stderr = "[iterations_runner.py] iteration 1/1"
# Non-zero return code.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 1, DUMMY_CONFIG)
expected = """Benchmark returned non-zero or emitted invalid JSON.
return code: 1
stdout:
--------------------------------------------------
%s
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
""" % stdout
assert excinfo.value.message == expected
def test_check_and_parse_execution_results0002():
stdout = json.dumps({
"wallclock_times": [123.4],
"core_cycle_counts": [[1], [2], [3], [4]],
"aperf_counts": [[5], [6], [7], [8]],
"mperf_counts": [[9], [10], [11], [12]],
})
stderr = "[iterations_runner.py] iteration 1/1"
# Non-zero return code.
with pytest.raises(ExecutionFailed) as excinfo:
check_and_parse_execution_results(stdout, stderr, 1, DUMMY_CONFIG)
expected = """Benchmark returned non-zero or emitted invalid JSON.
return code: 1
stdout:
--------------------------------------------------
%s
--------------------------------------------------
stderr:
--------------------------------------------------
[iterations_runner.py] iteration 1/1
--------------------------------------------------
""" % stdout
assert excinfo.value.message == expected
def run(self, mailer, dry_run=False):
"""Runs this job (execution)"""
flag = None
entry_point = self.sched.config.VARIANTS[self.variant]
vm_def = self.vm_info["vm_def"]
vm_def.dry_run = dry_run
# Set heap limit
heap_limit_kb = self.sched.config.HEAP_LIMIT
stack_limit_kb = self.sched.config.STACK_LIMIT
in_proc_iters = self.vm_info["n_iterations"]
if not dry_run:
self.sched.platform.collect_starting_throttle_counts()
stdout, stderr, rc, envlog_filename, timed_out = \
vm_def.run_exec(entry_point, in_proc_iters, self.parameter,
heap_limit_kb, stack_limit_kb, self.key,
self.key_pexec_idx)
if timed_out:
measurements = self.empty_measurements
instr_data = {}
flag = "T"
elif not dry_run:
try:
self.sched.platform.check_throttle_counts(self.sched.manifest)
measurements = util.check_and_parse_execution_results(
stdout, stderr, rc, self.sched.config, self.key, instrument=vm_def.instrument)
flag = "C"
except util.RerunExecution as e:
subject = ("Benchmark needs to be re-run: %s (exec_idx=%s)" %
(self.key, self.sched.manifest.next_exec_idx))
util.log_and_mail(mailer, warn, subject,
e.message, manifest=self.sched.manifest,
bypass_limiter=True)
measurements = self.empty_measurements
flag = "O" # i.e. still outstanding
except util.ExecutionFailed as e:
util.log_and_mail(mailer, error, "Benchmark failure: %s" %
self.key, e.message,
manifest=self.sched.manifest)
measurements = self.empty_measurements
flag = "E"
# Collect instrumentation data
if vm_def.instrument and flag == "C":
instr_data = vm_def.get_instr_data()
for k, v in instr_data.iteritems():
assert len(instr_data[k]) == in_proc_iters
else:
# The benchmark either failed, needs to be re-run, or had
# instrumentation turned off.
instr_data = {}
else:
measurements = self.empty_measurements
instr_data = {}
flag = "C"
# We print the status *after* benchmarking, so that I/O cannot be
# committed during benchmarking. In production, we will be rebooting
# before the next execution, so we are grand.
info("Finished '%s(%d)' (%s variant) under '%s'" %
(self.benchmark, self.parameter, self.variant, self.vm_name))
# Move the environment log out of /tmp.
#
# We don't do this for re-runs (O) as the log for the re-run pexec is
# the one we want.
#
# We don't do this for timeouts (T) because the wrapper script is
# killed upon timeout, and thus doesn't get a chance to log the
# environment.
if not dry_run and flag not in ("O", "T"):
key_exec_num = self.sched.manifest.completed_exec_counts[self.key]
util.stash_envlog(envlog_filename, self.sched.config,
self.sched.platform, self.key, key_exec_num)
assert flag is not None
return measurements, instr_data, flag