def _check_apm_state(self):
debug("Checking APM state is geared for high-performance")
adjust = False
out = self._get_apm_output()
lines = out.split("\n")
n_lines = len(lines)
if n_lines != 3:
fatal("Expected 3 lines of output from apm(8), got %d" % n_lines)
perf_line = lines[2].strip()
# First, the performance mode should be manual (static)
if not perf_line.startswith("Performance adjustment mode: manual"):
debug("performance mode is not manual.")
adjust = True
# Second, the CPU should be running as fast as possible
out, _, _ = run_shell_cmd(self.GET_SETPERF_CMD)
elems = out.split("=")
if len(elems) != 2 or elems[1].strip() != "100":
debug("hw.setperf is '%s' not '100'" % elems[1])
adjust = True
if adjust:
debug("adjusting performance mode")
out, _, _ = run_shell_cmd("apm -H")
self._check_apm_state() # should work this time
def test_save_power0001(self, platform):
run_shell_cmd("apm -H")
platform.save_power()
out, _, _ = run_shell_cmd("apm")
lines = out.split("\n")
line3 = lines[2].strip()
assert line3.startswith("Performance adjustment mode: auto")
def test_save_power0001(self, platform):
run_shell_cmd("apm -H")
platform.save_power()
out, _, _ = run_shell_cmd("apm")
lines = out.split("\n")
line3 = lines[2].strip()
assert line3.startswith("Performance adjustment mode: auto")
def test_run_shell_cmd():
msg = "example text"
out, err, rc = run_shell_cmd("echo " + msg)
assert out == msg
assert err == ""
assert rc == 0
msg2 = "another example"
out, err, rc = run_shell_cmd("(>&2 echo %s) && (echo %s)" % (msg2, msg))
assert out == msg
assert err == msg2
assert rc == 0
def test_run_shell_cmd():
msg = "example text"
out, err, rc = run_shell_cmd("echo " + msg)
assert out == msg
assert err == ""
assert rc == 0
msg2 = "another example"
out, err, rc = run_shell_cmd("(>&2 echo %s) && (echo %s)" % (msg2, msg))
assert out == msg
assert err == msg2
assert rc == 0
def _check_cpu_governor(self):
"""Checks the right CPU governor is in effect
Since we do not know which CPU benchmarks will be scheduled on,
we simply check them all"""
# Check CPU cores are running with the 'performance' governor
# And that the correct scaler is in use. We never want the pstate
# scaler, as it tends to cause the clock speed to fluctuate, even
# when in performance mode. Instead we use standard ACPI.
changed = False
for cpu_n in xrange(self.num_cpus):
# Check CPU governors
debug("Checking CPU governor for CPU%d" % cpu_n)
with open(LinuxPlatform.CPU_GOV_FMT % cpu_n, "r") as fh:
v = fh.read().strip()
if v != "performance":
debug("changing CPU governor for CPU %s" % cpu_n)
cmd = "%s cpufreq-set -c %d -g performance" % \
(self.change_user_cmd, cpu_n)
stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False)
changed = True
if rc != 0:
fatal("Governor for CPU%d governor: is '%s' not "
"performance'.\nKrun attempted to adjust the "
"governor using:\n '%s'\n"
"however this command failed. Is %s configured "
"and is cpufrequtils installed?"
% (cpu_n, v, cmd, self.change_user_cmd))
if changed:
self._check_cpu_governor() # just to be sure
def _raw_read_temperature_sensor(self, sensor):
# mocked in tests
# sysctl doesn't return 0 on failure.
# Luckily we can catch the error later when when the output
# looks weird.
return run_shell_cmd("sysctl %s" % sensor)[0]
def test_apm_state0001(self, platform, caplog):
run_shell_cmd("apm -C") # cool mode; forces krun to change this.
platform._check_apm_state()
assert "performance mode is not manual" in caplog.text()
# Hard to check hw.setperf, as it may well be temproarily 100
assert "adjusting performance mode" in caplog.text()
out, err, rc = run_shell_cmd("test `sysctl hw.setperf` == 'hw.setperf=100'")
assert out == err == ""
assert rc == 0
# cool mode.
# Sadly there is no way to query the current mode (e.g. -C or -H),
# othwerwise we could restore the APM state to how the user had it
# before.
run_shell_cmd("apm -C")
def test_apm_state0001(self, platform, caplog):
run_shell_cmd("apm -C") # cool mode; forces krun to change this.
platform._check_apm_state()
assert "performance mode is not manual" in caplog.text()
# Hard to check hw.setperf, as it may well be temproarily 100
assert "adjusting performance mode" in caplog.text()
out, err, rc = run_shell_cmd(
"test `sysctl hw.setperf` == 'hw.setperf=100'")
assert out == err == ""
assert rc == 0
# cool mode.
# Sadly there is no way to query the current mode (e.g. -C or -H),
# othwerwise we could restore the APM state to how the user had it
# before.
run_shell_cmd("apm -C")
def _open_kernel_config_file():
"""Open the kernel config file in /boot. Separated for mocking."""
# get kernel release string, e.g. "3.16.0-4-amd64"
stdout, _, _ = run_shell_cmd("uname -r")
kern_rel_str = stdout.strip()
config_basename = "config-%s" % kern_rel_str
config_path = os.sep + os.path.join("boot", config_basename)
return open(config_path, "r")
def test_aslr0001(self, platform, caplog):
# get current ASLR value
with open(LinuxPlatform.ASLR_FILE, "r") as fh:
old_val = fh.read().strip()
# First turn off ASLR
cmd = "%s sh -c 'echo 0 > %s'" % \
(platform.change_user_cmd, LinuxPlatform.ASLR_FILE)
run_shell_cmd(cmd)
# This should flip it to mode 2
platform._check_aslr_enabled()
with open(LinuxPlatform.ASLR_FILE, "r") as fh:
new_val = fh.read().strip()
assert new_val == '2'
assert "Adjust ASLR" in caplog.text()
# Restore old value
cmd = "%s sh -c 'echo %s > %s'" % \
(platform.change_user_cmd, old_val, LinuxPlatform.ASLR_FILE)
run_shell_cmd(cmd)
def test_aslr0001(self, platform, caplog):
# get current ASLR value
with open(LinuxPlatform.ASLR_FILE, "r") as fh:
old_val = fh.read().strip()
# First turn off ASLR
cmd = "%s sh -c 'echo 0 > %s'" % \
(platform.change_user_cmd, LinuxPlatform.ASLR_FILE)
run_shell_cmd(cmd)
# This should flip it to mode 2
platform._check_aslr_enabled()
with open(LinuxPlatform.ASLR_FILE, "r") as fh:
new_val = fh.read().strip()
assert new_val == '2'
assert "Adjust ASLR" in caplog.text()
# Restore old value
cmd = "%s sh -c 'echo %s > %s'" % \
(platform.change_user_cmd, old_val, LinuxPlatform.ASLR_FILE)
run_shell_cmd(cmd)
def _get_num_cpus(self):
err = False
# most reliable method generic to all Linux
out, _, rv = run_shell_cmd("grep -c ^processor /proc/cpuinfo")
if rv == 0:
out = out.strip()
try:
return int(out)
except ValueError:
pass
fatal("could not detect number of logical CPUs")
def _save_power(self):
"""Called when benchmarking is done, to save power"""
for cpu_n in xrange(self.num_cpus):
debug("Set CPU%d governor to 'ondemand'" % cpu_n)
cmd = "%s cpufreq-set -c %d -g ondemand" % \
(self.change_user_cmd, cpu_n)
stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False)
if rc != 0:
# Doesn't really matter if this fails, so just warn.
warn("Could not set CPU%d governor to 'ondemand' when "
"finished.\nFailing command:\n%s" % (cpu_n, cmd))
def _check_realtime_throttle_disabled(self):
"""Linux kernel gets pretty upset if you run a CPU intensive thread
under the real-time thread schedule policy. By default Linux will
artificially pre-empt such threads to give other things a chance to run
on this core. A switch will flip at runtime leaving a message in dmesg
when this comes into effect.
See the "Limiting the CPU usage of real-time and deadline processes"
section in sched(7).
We don't want "throttling" on the benchmarking cores.
From sched(7):
"Specifying [sched_rt_runtime_us] -1 makes the runtime the same as the
period; that is, no CPU time is set aside for non-real-time processes."
"""
debug("Check real-time thread throttling is off")
for itr in xrange(2):
with open(LinuxPlatform.SCHED_RT_RUNTIME_US) as fh:
val = fh.read().strip()
if val != "-1":
if itr == 0:
debug("%s is not -1, adjusting." % LinuxPlatform.SCHED_RT_RUNTIME_US)
# Needs to happen as root
args = self.change_user_args() + \
["sh", "-c",
"'echo -1 > %s'" % LinuxPlatform.SCHED_RT_RUNTIME_US]
cmd = " ".join(args)
run_shell_cmd(cmd)
else:
fatal("Could not set %s to -1" %
LinuxPlatform.SCHED_RT_RUNTIME_US)
def _check_perf_samplerate(self):
"""Attempt to minimise time spent by the Linux perf kernel profiler.
You can't disable this, so the best we can do is set the sample
rate to the minimum value of one sample per second."""
with open(LinuxPlatform.PERF_SAMPLE_RATE) as fh:
sr = int(fh.read().strip())
if sr != 1:
cmd = "%s sh -c 'echo 1 > %s'" % \
(self.change_user_cmd, LinuxPlatform.PERF_SAMPLE_RATE)
stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False)
if rc != 0:
fatal("perf profiler sample rate >1 p/s. "
"Krun was unable to adjust it.\nFailing command:\n %s"
% cmd)
def _check_aslr_disabled(self):
debug("Checking ASLR is off")
with open(self.ASLR_FILE, "r") as fh:
enabled = fh.read().strip()
if enabled == "0":
return # OK!
else:
# ASLR is off, but we can try to enable it
debug("Turning ASLR off")
cmd = "%s sh -c 'echo 0 > %s'" % \
(self.change_user_cmd, self.ASLR_FILE)
stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False)
if rc != 0:
msg = "ASLR disabled (%s, expect '0' got '%s').\n" % \
(self.ASLR_FILE, enabled)
msg += "Krun tried to turn it off, but failed."
fatal(msg)
else:
self._check_aslr_disabled() # should work this time
def test_run_shell_cmd_fatal():
cmd = "nonsensecommand"
out, err, rc = run_shell_cmd(cmd, False)
assert rc != 0
assert cmd in err
assert out == ""
def test_run_shell_cmd():
msg = "example text"
out, err, rc = run_shell_cmd("echo " + msg)
assert out == msg
assert err == ""
assert rc == 0
def _collect_dmesg_lines(self):
return run_shell_cmd("dmesg")[0].split("\n")
def collect_audit(self):
self.audit["uname"] = run_shell_cmd("uname -a")[0]
self.audit["dmesg"] = run_shell_cmd("dmesg")[0]
self.audit["krun_version"] = util.get_git_version()
self.audit["cli_args"] = sys.argv
def inner_main(mailer, config, args):
out_file = config.results_filename()
out_file_exists = os.path.exists(out_file)
if out_file_exists and not os.path.isfile(out_file):
util.fatal(
"Output file '%s' exists but is not a regular file" % out_file)
if out_file_exists and not args.resume:
util.fatal("Output file '%s' already exists. "
"Either resume the session (--resume) or "
"move the file away" % out_file)
if not out_file_exists and args.resume:
util.fatal("No results file to resume. Expected '%s'" % out_file)
if args.started_by_init and not args.reboot:
util.fatal("--started-by-init makes no sense without --reboot")
if args.started_by_init and not args.resume:
util.fatal("--started-by-init makes no sense without --resume")
if args.develop:
warn("Developer mode enabled. Results will not be reliable.")
# Initialise platform instance and assign to VM defs.
# This needs to be done early, so VM sanity checks can run.
platform = detect_platform(mailer)
if not args.develop:
debug("Checking platform preliminaries")
platform.check_preliminaries()
else:
# Needed to skip the use of certain tools and techniques.
# E.g. switching user.
warn("Not checking platform prerequisites due to developer mode")
platform.developer_mode = True
platform.collect_audit()
# If the user has asked for resume-mode, the current platform must
# be an identical machine to the current one.
error_msg = ("You have asked Krun to resume an interrupted benchmark. " +
"This is only valid if the machine you are using is " +
"identical to the one on which the last results were " +
"gathered, which is not the case.")
current = None
if args.resume:
# output file must exist, due to check above
assert(out_file_exists)
current = Results(config, platform, results_file=out_file)
from krun.audit import Audit
if not Audit(platform.audit) == current.audit:
util.fatal(error_msg)
debug("Using pre-recorded initial temperature readings")
platform.starting_temperatures = current.starting_temperatures
else:
# Touch the config file to update its mtime. This is required
# by resume-mode which uses the mtime to determine the name of
# the log file, should this benchmark be resumed.
_, _, rc = util.run_shell_cmd("touch " + args.filename)
if rc != 0:
util.fatal("Could not touch config file: " + args.filename)
info(("Wait %s secs to allow system to cool prior to "
"collecting initial temperature readings") %
config.TEMP_READ_PAUSE)
if args.develop or args.dry_run:
info("SIMULATED: time.sleep(%s)" % config.TEMP_READ_PAUSE)
else:
time.sleep(config.TEMP_READ_PAUSE)
debug("Taking fresh initial temperature readings")
platform.starting_temperatures = platform.take_temperature_readings()
# Assign platform to VM defs -- needs to happen early for sanity checks
util.assign_platform(config, platform)
sanity_checks(config, platform)
# Build job queue -- each job is an execution
sched = ExecutionScheduler(config,
mailer,
platform,
resume=args.resume,
reboot=args.reboot,
dry_run=args.dry_run,
started_by_init=args.started_by_init)
sched.build_schedule()
sched.run()
def _get_apm_output(self):
# separate for mocking
return run_shell_cmd("apm")[0]
def _get_sysctl_sensor_lines(self):
# separate for test mocking
return run_shell_cmd(self.FIND_TEMP_SENSORS_CMD)[0]
def test_run_shell_cmd():
msg = "example text"
out, err, rc = run_shell_cmd("echo " + msg)
assert out == msg
assert err == ""
assert rc == 0
def test_run_shell_cmd_fatal():
cmd = "nonsensecommand"
out, err, rc = run_shell_cmd(cmd, False)
assert rc != 0
assert cmd in err
assert out == ""
def collect_audit(self):
BasePlatform.collect_audit(self)
# Extra CPU info, some not in dmesg. E.g. CPU cache size.
self.audit["cpuinfo"] = run_shell_cmd("cat /proc/cpuinfo")[0]
def collect_audit(self):
LinuxPlatform.collect_audit(self)
self.audit["packages"] = run_shell_cmd("dpkg-query -l")[0]
self.audit["debian_version"] = run_shell_cmd("cat /etc/debian_version")[0]
def _save_power(self):
run_shell_cmd("apm -C")