def test_run_schedule0005(self, mock_platform, monkeypatch,
no_results_instantiation_check):
def dummy_execjob_run(self, mailer, dryrun=False):
return self.empty_measurements, {}, "E" # pretend jobs fail
monkeypatch.setattr(ExecutionJob, 'run', dummy_execjob_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config,
mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to error
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_audit_differs0001(self, monkeypatch, mock_platform, caplog):
"""Check that if the audit differs, a crash occurs"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# mutate the audit, so it won't match later
results = Results(config, mock_platform, results_file=results_path)
results.audit._audit["wibble"] = "wut"
results.write_to_file()
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = "This is only valid if the machine you are using is identical"
assert expect in caplog.text()
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_boot_loop0001(self, monkeypatch, mock_platform, caplog):
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
# Simulate a boot loop
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
sched.manifest.num_reboots = 9998 # way too many
sched.manifest.update_num_reboots() # increments and writes out file
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = ("HALTING now to prevent an infinite reboot loop: "
"INVARIANT num_reboots <= num_jobs violated. Krun was about "
"to execute reboot number: 10000. 1 jobs have been "
"completed, 7 are left to go.")
assert expect in caplog.text()
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_pre_and_post_exec_cmds0002(self, monkeypatch, mock_platform,
no_results_instantiation_check):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE}:${KRUN_MANIFEST_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
os.unlink(sched.manifest.path)
assert n_reboots == 1
elems = got.split(":")
assert os.path.basename(elems[0]) == "one_exec_results.json.bz2"
assert os.path.basename(elems[1]) == "one_exec.log"
assert os.path.basename(elems[2]) == "one_exec.manifest"
# all paths should be in the same dir
dirnames = [os.path.dirname(x) for x in elems]
assert dirnames[0] == dirnames[1] == dirnames[2]
def test_pre_and_post_exec_cmds0002(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE}:${KRUN_MANIFEST_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
os.unlink(sched.manifest.path)
assert n_reboots == 1
elems = got.split(":")
assert os.path.basename(elems[0]) == "one_exec_results.json.bz2"
assert os.path.basename(elems[1]) == "one_exec.log"
assert os.path.basename(elems[2]) == "one_exec.manifest"
# all paths should be in the same dir
dirnames = [os.path.dirname(x) for x in elems]
assert dirnames[0] == dirnames[1] == dirnames[2]
def test_boot_loop0001(self, monkeypatch, mock_platform, caplog,
no_results_instantiation_check):
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
# Simulate a boot loop
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
sched.manifest.num_reboots = 9998 # way too many
sched.manifest.update_num_reboots() # increments and writes out file
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = ("HALTING now to prevent an infinite reboot loop: "
"INVARIANT num_reboots <= num_jobs violated. Krun was about "
"to execute reboot number: 10000. 1 jobs have been "
"completed, 7 are left to go.")
assert expect in caplog.text
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_audit_differs0001(self, monkeypatch, mock_platform, caplog,
no_results_instantiation_check):
"""Check that if the audit differs, a crash occurs"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
results_path = config.results_filename()
# mutate the audit, so it won't match later
results = Results(config, mock_platform, results_file=results_path)
results.audit._audit["wibble"] = "wut"
results.write_to_file()
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = "This is only valid if the machine you are using is identical"
assert expect in caplog.text
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_results_filename():
example = os.path.join(TEST_DIR, "example.krun")
touch(example)
example_config = Config(example)
# not exact match due to absolute path
assert example_config.results_filename().endswith(
"example_results.json.bz2")
def test_run_schedule0003(self, mock_platform, monkeypatch):
config = Config(os.path.join(TEST_DIR, "example_all_skip.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 0 # all skipped!
os.unlink(sched.manifest.path)
os.unlink(config.results_filename())
def test_run_schedule0002(self, mock_platform, monkeypatch):
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to dry run
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def to_strip():
from krun.platform import detect_platform
from krun.results import Results
path = os.path.join(TEST_DIR, "quick.krun")
config = Config(path)
platform = detect_platform(None)
results = Results(config, platform, results_file=config.results_filename())
return results
def to_strip():
from krun.platform import detect_platform
from krun.results import Results
path = os.path.join(TEST_DIR, "quick.krun")
config = Config(path)
platform = detect_platform(None)
results = Results(config, platform,
results_file=config.results_filename())
return results
def test_run_schedule0001(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert sched.manifest.total_num_execs == 1
assert sched.manifest.num_execs_left == 0
assert n_reboots == 1
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 1 # 1 benchmark, 1 vm
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 1
for _exec in execs:
assert len(_exec) == 0 # due to dry run
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_run_schedule0005(self, mock_platform, monkeypatch):
def dummy_execjob_run(self, mailer, dryrun=False):
return self.empty_measurements, {}, "E" # pretend jobs fail
monkeypatch.setattr(ExecutionJob, 'run', dummy_execjob_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to error
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_run_schedule0004(self, mock_platform, monkeypatch,
no_results_instantiation_check):
config = Config(os.path.join(TEST_DIR, "example_skip_1vm.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 4 # 2 benchmarks, 2 vms, 2 execs, one VM skipped
results = Results(config,
mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
if "CPython" in key:
assert len(execs) == 0
else:
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to dry run
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_pre_and_post_exec_cmds0001(self, monkeypatch, mock_platform):
cap_cmds = []
def dummy_run_shell_cmd(cmd, failure_fatal=False, extra_env=None):
cap_cmds.append(cmd)
return "", "", 0
monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_run_shell_cmd)
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
config.PRE_EXECUTION_CMDS = ["cmd1", "cmd2"]
config.POST_EXECUTION_CMDS = ["cmd3", "cmd4"]
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
assert n_reboots == 1
expect = ["cmd1", "cmd2", "cmd3", "cmd4"]
assert cap_cmds == expect
def test_error_flag_persists0002(self, monkeypatch, mock_platform,
no_results_instantiation_check):
"""Check a changed dmesg will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend dmesg changes
def dummy_check_for_dmesg_changes(self):
return True
monkeypatch.setattr(mock_platform, 'check_dmesg_for_changes',
dummy_check_for_dmesg_changes)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run an execution where the dmesg changes
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_error_flag_persists0001(self, monkeypatch, mock_platform,
no_results_instantiation_check):
"""Check a failing exec will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend exec fails
def dummy_job_run(self, mailer, dry):
measurements = self.make_empty_measurement()
return measurements, None, 'E' # measurements, instr_data, flag
monkeypatch.setattr(ExecutionJob, 'run', dummy_job_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run a (failing) execution, which will dump the results file
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_pre_and_post_cmds0003(self, monkeypatch, mock_platform):
"""Check that the pre/post commands use a shell and don't just exec(3)"""
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
tmp_file = os.path.join(TEST_DIR, "prepost.txt")
# commands use shell syntax
config.PRE_EXECUTION_CMDS = ["echo 'pre' > %s" % tmp_file]
config.POST_EXECUTION_CMDS = ["echo 'post' >> %s" % tmp_file]
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 1
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
with open(tmp_file) as fh:
got = fh.read()
os.unlink(tmp_file)
assert got == "pre\npost\n"
def test_error_flag_persists0001(self, monkeypatch, mock_platform):
"""Check a failing exec will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend exec fails
def dummy_job_run(self, mailer, dry):
measurements = self.make_empty_measurement()
return measurements, None, 'E' # measurements, instr_data, flag
monkeypatch.setattr(ExecutionJob, 'run', dummy_job_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run a (failing) execution, which will dump the results file
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_error_flag_persists0002(self, monkeypatch, mock_platform):
"""Check a changed dmesg will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend dmesg changes
def dummy_check_for_dmesg_changes(self):
return True
monkeypatch.setattr(mock_platform, 'check_dmesg_for_changes',
dummy_check_for_dmesg_changes)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run an execution where the dmesg changes
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_results_filename():
example = os.path.join(TEST_DIR, "example.krun")
touch(example)
example_config = Config(example)
# not exact match due to absolute path
assert example_config.results_filename().endswith("example_results.json.bz2")
