def test_boot_loop0001(self, monkeypatch, mock_platform, caplog):
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
# Simulate a boot loop
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
sched.manifest.num_reboots = 9998 # way too many
sched.manifest.update_num_reboots() # increments and writes out file
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = ("HALTING now to prevent an infinite reboot loop: "
"INVARIANT num_reboots <= num_jobs violated. Krun was about "
"to execute reboot number: 10000. 1 jobs have been "
"completed, 7 are left to go.")
assert expect in caplog.text()
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_skip0007():
config = Config()
with pytest.raises(ValueError) as e:
config.should_skip("wobble")
assert e.value.message == "bad benchmark key: wobble"
def test_read_corrupt_config_from_string():
path = os.path.join(TEST_DIR, "corrupt.krun")
config = Config(None)
with pytest.raises(Exception):
with open(path) as fp:
config_string = fp.read()
config.read_from_string(config_string)
def test_audit_differs0001(self, monkeypatch, mock_platform, caplog):
"""Check that if the audit differs, a crash occurs"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# mutate the audit, so it won't match later
results = Results(config, mock_platform, results_file=results_path)
results.audit._audit["wibble"] = "wut"
results.write_to_file()
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = "This is only valid if the machine you are using is identical"
assert expect in caplog.text()
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_pre_and_post_cmds0002(self, monkeypatch, mock_platform):
"""Check that the pre/post commands use a shell and don't just exec(3)"""
config = Config(os.path.join(TEST_DIR, "example.krun"))
tmp_file = os.path.join(TEST_DIR, "prepost.txt")
# commands use shell syntax
config.PRE_EXECUTION_CMDS = ["echo 'pre' > %s" % tmp_file]
config.POST_EXECUTION_CMDS = ["echo 'post' >> %s" % tmp_file]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform, resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
sched.run()
with open(tmp_file) as fh:
got = fh.read()
os.unlink(tmp_file)
assert got == "pre\npost\n"
def test_pre_and_post_cmds0001(self, monkeypatch, mock_platform):
cap_cmds = []
def dummy_run_shell_cmd(cmd, failure_fatal=False, extra_env=None):
cap_cmds.append(cmd)
return "", "", 0
monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_run_shell_cmd)
config = Config(os.path.join(TEST_DIR, "example.krun"))
config.PRE_EXECUTION_CMDS = ["pre1", "pre2"]
config.POST_EXECUTION_CMDS = ["post1", "post2"]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
assert len(sched) == 8
sched.run()
expect = ["pre1", "pre2", "post1", "post2"] * 8
assert cap_cmds == expect
def test_run_schedule0003(self, mock_platform, monkeypatch):
config = Config(os.path.join(TEST_DIR, "example_all_skip.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 0 # all skipped!
os.unlink(sched.manifest.path)
os.unlink(config.results_filename())
def test_post_exec_cmds0002(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "example.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
assert len(sched) == 8
sched.run()
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
elems = got.split(":")
assert elems[0].endswith(".json.bz2")
assert elems[1].endswith(".log")
def test_pre_and_post_cmds0002(self, monkeypatch, mock_platform):
"""Check that the pre/post commands use a shell and don't just exec(3)"""
config = Config(os.path.join(TEST_DIR, "example.krun"))
tmp_file = os.path.join(TEST_DIR, "prepost.txt")
# commands use shell syntax
config.PRE_EXECUTION_CMDS = ["echo 'pre' > %s" % tmp_file]
config.POST_EXECUTION_CMDS = ["echo 'post' >> %s" % tmp_file]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
sched.run()
with open(tmp_file) as fh:
got = fh.read()
os.unlink(tmp_file)
assert got == "pre\npost\n"
def test_pre_and_post_cmds0001(self, monkeypatch, mock_platform):
cap_cmds = []
def dummy_run_shell_cmd(cmd, failure_fatal=False, extra_env=None):
cap_cmds.append(cmd)
return "", "", 0
monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_run_shell_cmd)
config = Config(os.path.join(TEST_DIR, "example.krun"))
config.PRE_EXECUTION_CMDS = ["pre1", "pre2"]
config.POST_EXECUTION_CMDS = ["post1", "post2"]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform, resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
assert len(sched) == 8
sched.run()
expect = ["pre1", "pre2", "post1", "post2"] * 8
assert cap_cmds == expect
def test_num_emails_sent_persists0001(self, monkeypatch, mock_platform,
no_results_instantiation_check):
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
sched.mailer.recipients = ["noone@localhost"]
assert sched.manifest.num_mails_sent == 0
sched.mailer.send("subject", "body", manifest=sched.manifest)
assert sched.manifest.num_mails_sent == 1
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# suppose a reboot happened now
del sched
del config
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
assert sched.manifest.num_mails_sent == 1
os.unlink(sched.manifest.path)
def test_skip0007():
config = Config()
with pytest.raises(ValueError) as e:
config.should_skip("wobble")
assert e.value.args[0] == "bad benchmark key: wobble"
def test_boot_loop0001(self, monkeypatch, mock_platform, caplog,
no_results_instantiation_check):
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
# Simulate a boot loop
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
sched.manifest.num_reboots = 9998 # way too many
sched.manifest.update_num_reboots() # increments and writes out file
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = ("HALTING now to prevent an infinite reboot loop: "
"INVARIANT num_reboots <= num_jobs violated. Krun was about "
"to execute reboot number: 10000. 1 jobs have been "
"completed, 7 are left to go.")
assert expect in caplog.text
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_audit_differs0001(self, monkeypatch, mock_platform, caplog,
no_results_instantiation_check):
"""Check that if the audit differs, a crash occurs"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config)
results_path = config.results_filename()
# mutate the audit, so it won't match later
results = Results(config, mock_platform, results_file=results_path)
results.audit._audit["wibble"] = "wut"
results.write_to_file()
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
dry_run=True)
with pytest.raises(krun.util.FatalKrunError):
sched.run()
expect = "This is only valid if the machine you are using is identical"
assert expect in caplog.text
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_post_exec_cmds0002(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "example.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform, resume=False,
dry_run=True,
started_by_init=True)
sched.build_schedule()
assert len(sched) == 8
sched.run()
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
elems = got.split(":")
assert elems[0].endswith(".json.bz2")
assert elems[1].endswith(".log")
def test_pre_and_post_exec_cmds0002(self, monkeypatch, mock_platform,
no_results_instantiation_check):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE}:${KRUN_MANIFEST_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
os.unlink(sched.manifest.path)
assert n_reboots == 1
elems = got.split(":")
assert os.path.basename(elems[0]) == "one_exec_results.json.bz2"
assert os.path.basename(elems[1]) == "one_exec.log"
assert os.path.basename(elems[2]) == "one_exec.manifest"
# all paths should be in the same dir
dirnames = [os.path.dirname(x) for x in elems]
assert dirnames[0] == dirnames[1] == dirnames[2]
def test_run_schedule0005(self, mock_platform, monkeypatch,
no_results_instantiation_check):
def dummy_execjob_run(self, mailer, dryrun=False):
return self.empty_measurements, {}, "E" # pretend jobs fail
monkeypatch.setattr(ExecutionJob, 'run', dummy_execjob_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config,
mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to error
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_results_filename():
example = os.path.join(TEST_DIR, "example.krun")
touch(example)
example_config = Config(example)
# not exact match due to absolute path
assert example_config.results_filename().endswith(
"example_results.json.bz2")
def test_pre_and_post_exec_cmds0002(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
path = os.path.join(TEST_DIR, "shell-out")
cmd = "echo ${KRUN_RESULTS_FILE}:${KRUN_LOG_FILE}:${KRUN_MANIFEST_FILE} > %s" % path
config.POST_EXECUTION_CMDS = [cmd]
krun.util.assign_platform(config, mock_platform)
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
with open(path) as fh:
got = fh.read().strip()
os.unlink(path)
os.unlink(sched.manifest.path)
assert n_reboots == 1
elems = got.split(":")
assert os.path.basename(elems[0]) == "one_exec_results.json.bz2"
assert os.path.basename(elems[1]) == "one_exec.log"
assert os.path.basename(elems[2]) == "one_exec.manifest"
# all paths should be in the same dir
dirnames = [os.path.dirname(x) for x in elems]
assert dirnames[0] == dirnames[1] == dirnames[2]
def test_read_config_from_string():
path = os.path.join(TEST_DIR, "example.krun")
config0 = Config(path)
config1 = Config(None)
with open(path) as fp:
config_string = fp.read()
config1.read_from_string(config_string)
assert config0 == config1
def test_check_config_consistency_fails():
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
with open(path) as fp:
config_string = fp.read()
with pytest.raises(Exception) as excinfo:
config.check_config_consistency(
config_string + "\n# different config!", "fakefilename")
assert "+# different config!" in excinfo.value.args[0]
def test_check_config_consistency_fails():
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
with open(path) as fp:
config_string = fp.read()
with pytest.raises(Exception) as excinfo:
config.check_config_consistency(config_string + "\n# different config!",
"fakefilename")
print excinfo.value.message
assert "+# different config!" in excinfo.value.message
def to_strip():
from krun.platform import detect_platform
from krun.results import Results
path = os.path.join(TEST_DIR, "quick.krun")
config = Config(path)
platform = detect_platform(None)
results = Results(config, platform, results_file=config.results_filename())
return results
def read_from_file(self, results_file):
"""Initialise object from serialised file on disk.
"""
with bz2.BZ2File(results_file, "rb") as f:
results = json.loads(f.read())
self.__dict__.update(results)
# Ensure that self.audir and self.config have correct types.
self.config = Config(None)
self.config.read_from_string(results["config"])
self.audit = results["audit"]
def to_strip():
from krun.platform import detect_platform
from krun.results import Results
path = os.path.join(TEST_DIR, "quick.krun")
config = Config(path)
platform = detect_platform(None)
results = Results(config, platform,
results_file=config.results_filename())
return results
def test_log_filename(monkeypatch):
path = os.path.join(TEST_DIR, "example.krun")
example_config = Config(path)
tstamp = time.strftime(LOGFILE_FILENAME_TIME_FORMAT)
expect_path = os.path.join(TEST_DIR, "example_%s.log" % tstamp)
assert example_config.log_filename(False) == expect_path
def mock_mtime(path):
return 1445964109.9363003
monkeypatch.setattr(os.path, 'getmtime', mock_mtime)
tstamp = '20151027_164149'
expect_path = os.path.join(TEST_DIR, "example_%s.log" % tstamp)
assert example_config.log_filename(True) == expect_path
def test_skip0001():
path = os.path.join(TEST_DIR, "skips.krun")
config = Config(path)
expected = ["*:PyPy:*",
"*:CPython:*",
"*:Hotspot:*",
"*:Graal:*",
"*:LuaJIT:*",
"*:HHVM:*",
"*:JRubyTruffle:*",
"*:V8:*",
]
for triplet in expected:
assert config.should_skip(triplet)
assert config.should_skip("nbody:HHVM:default-php")
assert not config.should_skip("nbody:MYVM:default-php")
def test_write_results_to_disk(self, mock_platform,
no_results_instantiation_check):
config = Config("krun/tests/example.krun")
mock_platform.num_cpus = 4
mock_platform.num_per_core_measurements = mock_platform.num_cpus
out_file = "krun/tests/example_results.json.bz2"
results0 = Results(config, mock_platform)
results0.audit = dict()
results0.starting_temperatures = [4355, 9879]
results0.wallclock_times = {u"dummy:Java:default-java": [[1.000726]]}
results0.eta_estimates = {u"dummy:Java:default-java": [1.1]}
results0.core_cycle_counts = {
u"dummy:Java:default-java": [[[2], [3], [4], [5]]]
}
results0.aperf_counts = {
u"dummy:Java:default-java": [[[3], [4], [5], [6]]]
}
results0.mperf_counts = {
u"dummy:Java:default-java": [[[4], [5], [6], [7]]]
}
results0.reboots = 5
results0.error_flag = False
results0.write_to_file()
results1 = Results(config, None, results_file=out_file)
assert results0 == results1
# Clean-up generated file.
os.unlink(out_file)
def test_run_exec_popen0002(self, monkeypatch):
"""Check that writing stderr to a file works. Used for instrumentation"""
config = Config()
platform = MockPlatform(None, config)
vm_def = PythonVMDef('/dummy/bin/python')
vm_def.set_platform(platform)
args = [
sys.executable, "-c",
"import sys; sys.stdout.write('STDOUT'); sys.stderr.write('STDERR')"
]
with NamedTemporaryFile(delete=False, prefix="kruntest") as fh:
filename = fh.name
out, err, rv, timed_out = vm_def._run_exec_popen(args, fh)
assert err == "" # not here due to redirection
assert out == "STDOUT" # behaviour should be unchanged
assert rv == 0
assert timed_out == False
# stderr should be in this file
with open(filename) as fh:
assert fh.read() == "STDERR"
fh.close()
os.unlink(filename)
def test_skip0001():
path = os.path.join(TEST_DIR, "skips.krun")
config = Config(path)
expected = ["*:PyPy:*",
"*:CPython:*",
"*:Hotspot:*",
"*:Graal:*",
"*:LuaJIT:*",
"*:HHVM:*",
"*:TruffleRuby:*",
"*:V8:*",
]
for triplet in expected:
assert config.should_skip(triplet)
assert config.should_skip("nbody:HHVM:default-php")
assert not config.should_skip("nbody:MYVM:default-php")
def test_sync_disks0002(self, monkeypatch):
"""We throw away the results from sanity checks, so there's no need to
sync disks (and wait)."""
stdout = json.dumps({
"wallclock_times": [123.4],
"core_cycle_counts": [[1], [2], [3], [4]],
"aperf_counts": [[5], [6], [7], [8]],
"mperf_counts": [[9], [10], [11], [12]],
})
config = Config()
platform = MockPlatform(None, config)
ep = EntryPoint("test")
vm_def = PythonVMDef('/dummy/bin/python')
sync_called = [False]
def fake_sync_disks():
sync_called[0] = True
monkeypatch.setattr(platform, "sync_disks", fake_sync_disks)
def fake_run_exec_popen(args, stderr_file=None):
return stdout, "", 0, False # stdout, stderr, exit_code, timed_out
monkeypatch.setattr(vm_def, "_run_exec_popen", fake_run_exec_popen)
util.spawn_sanity_check(platform, ep, vm_def, "test")
assert sync_called == [False]
def test_write_new_manifest0002(mock_platform):
manifest_path = "example_000.manifest"
config_path = os.path.join(TEST_DIR, "more_complicated.krun")
config = Config(config_path)
manifest = ManifestManager(config, mock_platform, new_file=True)
assert manifest.total_num_execs == 90 # taking into account skips
_tear_down(manifest.path)
def test_skip0009():
config = Config()
config.SKIP = ["*:SomeVM:*",
"fasta:JRubyTruffle:default-ruby",
"bench:*:*",
"bench:vm:skipvariant",
"*:*:skipvariant",
]
assert config.should_skip("fasta:JRubyTruffle:default-ruby")
assert not config.should_skip("fasta:JRubyTruffle:default-ruby2")
assert config.should_skip("bench:lala:hihi")
assert config.should_skip("bench:lala:hihi2")
assert not config.should_skip("bench1:lala:hihi")
assert config.should_skip("bench1:lala:skipvariant")
assert config.should_skip("bench1:lala2:skipvariant")
def test_run_schedule_reboot(self, monkeypatch, mock_platform):
def dummy_shell_cmd(text):
pass
def dummy_execv(text, lst):
pass
monkeypatch.setattr(os, "execv", dummy_execv)
monkeypatch.setattr(subprocess, "call", dummy_shell_cmd)
monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_shell_cmd)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
sched = ExecutionScheduler(config,
mock_platform.mailer,
mock_platform,
resume=False,
reboot=True,
dry_run=True,
started_by_init=True)
sched.build_schedule()
assert len(sched) == 8
with pytest.raises(AssertionError):
sched.run()
assert len(sched) == 7
os.unlink(os.path.join(TEST_DIR, "example_results.json.bz2"))
def main(parser):
args = parser.parse_args()
if args.dump is not None:
if not args.filename.endswith(".json.bz2"):
usage(parser)
else:
results = Results(None, None, results_file=args.filename)
if args.dump == "config" or "audit":
text = unicode(results.__getattribute__(args.dump))
else:
text = json.dumps(results.__getattribute__(args.dump),
sort_keys=True, indent=2)
# String data read in from JSON are unicode objects. This matters
# for us as some data in the audit includes unicode characters.
# If it does, a simple print no longer suffices if the system
# locale is (e.g.) ASCII. In this case print will raise an
# exception. The correct thing to do is to encode() the unicode to
# the system locale.
print(text.encode(locale.getpreferredencoding()))
sys.exit(0)
if not args.filename.endswith(".krun"):
usage(parser)
try:
if os.stat(args.filename).st_size <= 0:
util.fatal('Krun configuration file %s is empty.' % args.filename)
except OSError:
util.fatal('Krun configuration file %s does not exist.' % args.filename)
config = Config(args.filename)
if args.info:
# Info mode doesn't run the experiment.
# Just prints some metrics and exits.
util.print_session_info(config)
return
if args.strip_results:
util.strip_results(config, args.strip_results)
return
attach_log_file(config, args.resume)
debug("Krun invoked with arguments: %s" % sys.argv)
mail_recipients = config.MAIL_TO
if type(mail_recipients) is not list:
util.fatal("MAIL_TO config should be a list")
mailer = Mailer(mail_recipients, max_mails=config.MAX_MAILS)
try:
inner_main(mailer, config, args)
except util.FatalKrunError as e:
subject = "Fatal Krun Exception"
mailer.send(subject, e.args[0], bypass_limiter=True)
util.run_shell_cmd_list(config.POST_EXECUTION_CMDS)
raise e
def test_skip0010():
config = Config()
config.SKIP = ["*:SomeVM:*",
"fasta:JRubyTruffle:default-ruby",
"bench:*:*",
"bench:vm:skipvariant",
"*:*:skipvariant",
"*:*:*", # everything should be skipped due to this
]
import uuid
def rand_str():
return uuid.uuid4().hex
for i in xrange(25):
key = "%s:%s:%s" % tuple([rand_str() for x in xrange(3)])
assert config.should_skip(key)
def test_custom_dmesg_whitelist0003(monkeypatch):
"""Test a config file that uses no custom whitelist"""
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
platform = krun.platform.detect_platform(None, config)
patterns = [p.pattern for p in platform.get_dmesg_whitelist()]
assert patterns == platform.default_dmesg_whitelist()
def test_custom_dmesg_whitelist0002(monkeypatch):
"""Test a config file that replaces entirely the dmesg whitelist"""
path = os.path.join(TEST_DIR, "custom_dmesg_whitelist0002.krun")
config = Config(path)
platform = krun.platform.detect_platform(None, config)
patterns = [p.pattern for p in platform.get_dmesg_whitelist()]
assert patterns == ["^.no+", "^defaults", "^here+"]
def test_eq(self, mock_platform):
results = Results(None,
None,
results_file="krun/tests/quick_results.json.bz2")
assert results == results
assert not results == None
assert not results == \
Results(Config("krun/tests/example.krun"), mock_platform)
def test_write_new_manifest0001(mock_platform):
_setup(BLANK_EXAMPLE_MANIFEST)
config = Config(os.path.join(TEST_DIR, "example.krun"))
manifest1 = ManifestManager(config, mock_platform, new_file=True)
manifest2 = ManifestManager(
config, mock_platform) # reads the file in from the last line
assert manifest1 == manifest2
_tear_down(manifest2.path)
def test_skip0010():
config = Config()
config.SKIP = ["*:SomeVM:*",
"fasta:TruffleRuby:default-ruby",
"bench:*:*",
"bench:vm:skipvariant",
"*:*:skipvariant",
"*:*:*", # everything should be skipped due to this
]
import uuid
def rand_str():
return uuid.uuid4().hex
for i in xrange(25):
key = "%s:%s:%s" % tuple([rand_str() for x in xrange(3)])
assert config.should_skip(key)
def test_run_schedule0002(self, mock_platform, monkeypatch):
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to dry run
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_skip0009():
config = Config()
config.SKIP = ["*:SomeVM:*",
"fasta:TruffleRuby:default-ruby",
"bench:*:*",
"bench:vm:skipvariant",
"*:*:skipvariant",
]
assert config.should_skip("fasta:TruffleRuby:default-ruby")
assert not config.should_skip("fasta:TruffleRuby:default-ruby2")
assert config.should_skip("bench:lala:hihi")
assert config.should_skip("bench:lala:hihi2")
assert not config.should_skip("bench1:lala:hihi")
assert config.should_skip("bench1:lala:skipvariant")
assert config.should_skip("bench1:lala2:skipvariant")
def test_pre_and_post_exec_cmds0001(self, monkeypatch, mock_platform):
cap_cmds = []
def dummy_run_shell_cmd(cmd, failure_fatal=False, extra_env=None):
cap_cmds.append(cmd)
return "", "", 0
monkeypatch.setattr(krun.util, "run_shell_cmd", dummy_run_shell_cmd)
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
config.PRE_EXECUTION_CMDS = ["cmd1", "cmd2"]
config.POST_EXECUTION_CMDS = ["cmd3", "cmd4"]
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
assert n_reboots == 1
expect = ["cmd1", "cmd2", "cmd3", "cmd4"]
assert cap_cmds == expect
def test_run_schedule0001(self, monkeypatch, mock_platform):
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert sched.manifest.total_num_execs == 1
assert sched.manifest.num_execs_left == 0
assert n_reboots == 1
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 1 # 1 benchmark, 1 vm
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 1
for _exec in execs:
assert len(_exec) == 0 # due to dry run
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_pre_and_post_cmds0003(self, monkeypatch, mock_platform):
"""Check that the pre/post commands use a shell and don't just exec(3)"""
config = Config(os.path.join(TEST_DIR, "one_exec.krun"))
tmp_file = os.path.join(TEST_DIR, "prepost.txt")
# commands use shell syntax
config.PRE_EXECUTION_CMDS = ["echo 'pre' > %s" % tmp_file]
config.POST_EXECUTION_CMDS = ["echo 'post' >> %s" % tmp_file]
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 1
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
with open(tmp_file) as fh:
got = fh.read()
os.unlink(tmp_file)
assert got == "pre\npost\n"
def test_skip0006():
config = Config()
config.SKIP = ["*:*:*"]
assert config.should_skip("mybench:CPython:default-python")
assert config.should_skip("myotherbench:CPython:default-python")
assert config.should_skip("mybench:PyPy:default-python")
assert config.should_skip("mybench:CPython:special-python")
def test_error_flag_persists0001(self, monkeypatch, mock_platform):
"""Check a failing exec will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend exec fails
def dummy_job_run(self, mailer, dry):
measurements = self.make_empty_measurement()
return measurements, None, 'E' # measurements, instr_data, flag
monkeypatch.setattr(ExecutionJob, 'run', dummy_job_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run a (failing) execution, which will dump the results file
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_error_flag_persists0002(self, monkeypatch, mock_platform):
"""Check a changed dmesg will correctly set the error flag"""
make_reboot_raise(monkeypatch)
no_envlogs(monkeypatch)
# pretend dmesg changes
def dummy_check_for_dmesg_changes(self):
return True
monkeypatch.setattr(mock_platform, 'check_dmesg_for_changes',
dummy_check_for_dmesg_changes)
config = Config(os.path.join(TEST_DIR, "example.krun"))
krun.util.assign_platform(config, mock_platform)
emulate_first_reboot(mock_platform, config, monkeypatch)
results_path = config.results_filename()
# To start, the error flag is not set
results = Results(config, mock_platform, results_file=results_path)
assert not results.error_flag
# run an execution where the dmesg changes
sched = ExecutionScheduler(config, mock_platform.mailer, mock_platform,
dry_run=True)
try:
sched.run()
except _TestReboot:
pass
else:
assert False
# reload results and check the error flag is now set
results = Results(config, mock_platform, results_file=results_path)
assert results.error_flag
os.unlink(sched.manifest.path)
os.unlink(results_path)
def test_run_schedule0005(self, mock_platform, monkeypatch):
def dummy_execjob_run(self, mailer, dryrun=False):
return self.empty_measurements, {}, "E" # pretend jobs fail
monkeypatch.setattr(ExecutionJob, 'run', dummy_execjob_run)
config = Config(os.path.join(TEST_DIR, "example.krun"))
n_reboots, sched = run_with_captured_reboots(config, mock_platform,
monkeypatch)
assert n_reboots == 8 # 2 benchmarks, 2 vms, 2 execs
results = Results(config, mock_platform,
results_file=config.results_filename())
type_check_results(results)
assert len(results.wallclock_times) == 4 # 2 benchmarks, 2 vms
for key, execs in results.wallclock_times.iteritems():
assert len(execs) == 2
for _exec in execs:
assert len(_exec) == 0 # due to error
os.unlink(config.results_filename())
os.unlink(sched.manifest.path)
def test_log_filename0001():
path = os.path.join(TEST_DIR, "example.krun")
example_config = Config(path)
expect_path = os.path.join(TEST_DIR, "example.log")
assert example_config.log_filename(False) == expect_path
def test_read_config_from_file():
path = os.path.join(TEST_DIR, "example.krun")
config0 = Config(path)
config1 = Config(None)
config1.read_from_file(path)
assert config0 == config1
def test_check_config_consistency():
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
with open(path) as fp:
config_string = fp.read()
config.check_config_consistency(config_string, "fakefilename")
def test_skip0008():
config = Config()
config.SKIP = ["*:SomeVM:*", "fasta:TruffleRuby:default-ruby"]
assert config.should_skip("fasta:TruffleRuby:default-ruby")
class Results(object):
"""Results of a Krun benchmarking session.
Can be serialised to disk.
"""
def __init__(self, config, platform, results_file=None):
self.config = config
# Maps key to results:
# "bmark:vm:variant" -> [[e0i0, e0i1, ...], [e1i0, e1i1, ...], ...]
self.data = dict()
self.reboots = 0
# Record how long execs are taking so we can give the user a rough ETA.
# Maps "bmark:vm:variant" -> [t_0, t_1, ...]
self.eta_estimates = dict()
# error_flag is flipped when a (non-fatal) error or warning occurs.
# When Krun finishes and this flag is true, a message is printed,
# thus prompting the user to investigate.
self.error_flag = False
# Fill in attributes from the config, platform and prior results.
if self.config is not None:
self.filename = self.config.results_filename()
self.init_from_config()
if platform is not None:
self.starting_temperatures = platform.starting_temperatures
self._audit = Audit(platform.audit)
else:
self.starting_temperatures = list()
self.audit = dict()
# Import data from a Results object serialised on disk.
if results_file is not None:
self.read_from_file(results_file)
@property
def audit(self):
return self._audit
@audit.setter
def audit(self, audit_dict):
self._audit = Audit(audit_dict)
def init_from_config(self):
"""Scaffold dictionaries based on a given configuration.
"""
# Initialise dictionaries based on config information.
for vm_name, vm_info in self.config.VMS.items():
for bmark, _ in self.config.BENCHMARKS.items():
for variant in vm_info["variants"]:
key = ":".join((bmark, vm_name, variant))
self.data[key] = []
self.eta_estimates[key] = []
def read_from_file(self, results_file):
"""Initialise object from serialised file on disk.
"""
with bz2.BZ2File(results_file, "rb") as f:
results = json.loads(f.read())
self.__dict__.update(results)
# Ensure that self.audir and self.config have correct types.
self.config = Config(None)
self.config.read_from_string(results["config"])
self.audit = results["audit"]
def write_to_file(self):
"""Serialise object on disk."""
debug("Writing results out to: %s" % self.filename)
to_write = {
"config": self.config.text,
"data": self.data,
"audit": self.audit.audit,
"reboots": self.reboots,
"starting_temperatures": self.starting_temperatures,
"eta_estimates": self.eta_estimates,
"error_flag": self.error_flag,
}
with bz2.BZ2File(self.filename, "w") as f:
f.write(json.dumps(to_write,
indent=1, sort_keys=True, encoding='utf-8'))
def jobs_completed(self, key):
"""Return number of executions for which we have data for a given
benchmark / vm / variant triplet.
"""
return len(self.data[key])
def __eq__(self, other):
if not isinstance(other, self.__class__):
return False
return (self.config == other.config and
self.data == other.data and
self.audit == other.audit and
self.reboots == other.reboots and
self.starting_temperatures == other.starting_temperatures and
self.eta_estimates == other.eta_estimates and
self.error_flag == other.error_flag)
def strip_results(self, key_spec):
debug("Strip results: %s" % key_spec)
spec_elems = key_spec.split(":")
if len(spec_elems) != 3:
fatal("malformed key spec: %s" % key_spec)
new_data = self.data.copy()
removed_keys = 0
removed_execs = 0
# We have to keep track of how many executions have run successfully so
# that we can set self.reboots accordingly. It's not correct to simply
# deduct one for each execution we remove, as the reboots value is one
# higher due to the initial reboot. Bear in mind the user may strip
# several result keys in succession, so counting the completed
# executions is the only safe way.
completed_execs = 0
for key in self.data.iterkeys():
key_elems = key.split(":")
# deal with wildcards
for i in xrange(3):
if spec_elems[i] == "*":
key_elems[i] = "*"
# decide whether to remove
if key_elems == spec_elems:
removed_keys += 1
removed_execs += len(new_data[key])
new_data[key] = []
self.eta_estimates[key] = []
info("Removed results for: %s" % key)
else:
completed_execs += len(new_data[key])
self.data = new_data
# If the results were collected with reboot mode, update reboots count
if self.reboots != 0:
self.reboots = completed_execs
return removed_keys
