def _check_apm_state(self): debug("Checking APM state is geared for high-performance") adjust = False out = self._get_apm_output() lines = out.split("\n") n_lines = len(lines) if n_lines != 3: fatal("Expected 3 lines of output from apm(8), got %d" % n_lines) perf_line = lines[2].strip() # First, the performance mode should be manual (static) if not perf_line.startswith("Performance adjustment mode: manual"): debug("performance mode is not manual.") adjust = True # Second, the CPU should be running as fast as possible out, _, _ = run_shell_cmd(self.GET_SETPERF_CMD) elems = out.split("=") if len(elems) != 2 or elems[1].strip() != "100": debug("hw.setperf is '%s' not '100'" % elems[1]) adjust = True if adjust: debug("adjusting performance mode") out, _, _ = run_shell_cmd("apm -H") self._check_apm_state() # should work this time
def test_save_power0001(self, platform): run_shell_cmd("apm -H") platform.save_power() out, _, _ = run_shell_cmd("apm") lines = out.split("\n") line3 = lines[2].strip() assert line3.startswith("Performance adjustment mode: auto")
def test_save_power0001(self, platform): run_shell_cmd("apm -H") platform.save_power() out, _, _ = run_shell_cmd("apm") lines = out.split("\n") line3 = lines[2].strip() assert line3.startswith("Performance adjustment mode: auto")
def test_run_shell_cmd(): msg = "example text" out, err, rc = run_shell_cmd("echo " + msg) assert out == msg assert err == "" assert rc == 0 msg2 = "another example" out, err, rc = run_shell_cmd("(>&2 echo %s) && (echo %s)" % (msg2, msg)) assert out == msg assert err == msg2 assert rc == 0
def test_run_shell_cmd(): msg = "example text" out, err, rc = run_shell_cmd("echo " + msg) assert out == msg assert err == "" assert rc == 0 msg2 = "another example" out, err, rc = run_shell_cmd("(>&2 echo %s) && (echo %s)" % (msg2, msg)) assert out == msg assert err == msg2 assert rc == 0
def _check_cpu_governor(self): """Checks the right CPU governor is in effect Since we do not know which CPU benchmarks will be scheduled on, we simply check them all""" # Check CPU cores are running with the 'performance' governor # And that the correct scaler is in use. We never want the pstate # scaler, as it tends to cause the clock speed to fluctuate, even # when in performance mode. Instead we use standard ACPI. changed = False for cpu_n in xrange(self.num_cpus): # Check CPU governors debug("Checking CPU governor for CPU%d" % cpu_n) with open(LinuxPlatform.CPU_GOV_FMT % cpu_n, "r") as fh: v = fh.read().strip() if v != "performance": debug("changing CPU governor for CPU %s" % cpu_n) cmd = "%s cpufreq-set -c %d -g performance" % \ (self.change_user_cmd, cpu_n) stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False) changed = True if rc != 0: fatal("Governor for CPU%d governor: is '%s' not " "performance'.\nKrun attempted to adjust the " "governor using:\n '%s'\n" "however this command failed. Is %s configured " "and is cpufrequtils installed?" % (cpu_n, v, cmd, self.change_user_cmd)) if changed: self._check_cpu_governor() # just to be sure
def _raw_read_temperature_sensor(self, sensor): # mocked in tests # sysctl doesn't return 0 on failure. # Luckily we can catch the error later when when the output # looks weird. return run_shell_cmd("sysctl %s" % sensor)[0]
def test_apm_state0001(self, platform, caplog): run_shell_cmd("apm -C") # cool mode; forces krun to change this. platform._check_apm_state() assert "performance mode is not manual" in caplog.text() # Hard to check hw.setperf, as it may well be temproarily 100 assert "adjusting performance mode" in caplog.text() out, err, rc = run_shell_cmd("test `sysctl hw.setperf` == 'hw.setperf=100'") assert out == err == "" assert rc == 0 # cool mode. # Sadly there is no way to query the current mode (e.g. -C or -H), # othwerwise we could restore the APM state to how the user had it # before. run_shell_cmd("apm -C")
def test_apm_state0001(self, platform, caplog): run_shell_cmd("apm -C") # cool mode; forces krun to change this. platform._check_apm_state() assert "performance mode is not manual" in caplog.text() # Hard to check hw.setperf, as it may well be temproarily 100 assert "adjusting performance mode" in caplog.text() out, err, rc = run_shell_cmd( "test `sysctl hw.setperf` == 'hw.setperf=100'") assert out == err == "" assert rc == 0 # cool mode. # Sadly there is no way to query the current mode (e.g. -C or -H), # othwerwise we could restore the APM state to how the user had it # before. run_shell_cmd("apm -C")
def _open_kernel_config_file(): """Open the kernel config file in /boot. Separated for mocking.""" # get kernel release string, e.g. "3.16.0-4-amd64" stdout, _, _ = run_shell_cmd("uname -r") kern_rel_str = stdout.strip() config_basename = "config-%s" % kern_rel_str config_path = os.sep + os.path.join("boot", config_basename) return open(config_path, "r")
def test_aslr0001(self, platform, caplog): # get current ASLR value with open(LinuxPlatform.ASLR_FILE, "r") as fh: old_val = fh.read().strip() # First turn off ASLR cmd = "%s sh -c 'echo 0 > %s'" % \ (platform.change_user_cmd, LinuxPlatform.ASLR_FILE) run_shell_cmd(cmd) # This should flip it to mode 2 platform._check_aslr_enabled() with open(LinuxPlatform.ASLR_FILE, "r") as fh: new_val = fh.read().strip() assert new_val == '2' assert "Adjust ASLR" in caplog.text() # Restore old value cmd = "%s sh -c 'echo %s > %s'" % \ (platform.change_user_cmd, old_val, LinuxPlatform.ASLR_FILE) run_shell_cmd(cmd)
def test_aslr0001(self, platform, caplog): # get current ASLR value with open(LinuxPlatform.ASLR_FILE, "r") as fh: old_val = fh.read().strip() # First turn off ASLR cmd = "%s sh -c 'echo 0 > %s'" % \ (platform.change_user_cmd, LinuxPlatform.ASLR_FILE) run_shell_cmd(cmd) # This should flip it to mode 2 platform._check_aslr_enabled() with open(LinuxPlatform.ASLR_FILE, "r") as fh: new_val = fh.read().strip() assert new_val == '2' assert "Adjust ASLR" in caplog.text() # Restore old value cmd = "%s sh -c 'echo %s > %s'" % \ (platform.change_user_cmd, old_val, LinuxPlatform.ASLR_FILE) run_shell_cmd(cmd)
def _get_num_cpus(self): err = False # most reliable method generic to all Linux out, _, rv = run_shell_cmd("grep -c ^processor /proc/cpuinfo") if rv == 0: out = out.strip() try: return int(out) except ValueError: pass fatal("could not detect number of logical CPUs")
def _save_power(self): """Called when benchmarking is done, to save power""" for cpu_n in xrange(self.num_cpus): debug("Set CPU%d governor to 'ondemand'" % cpu_n) cmd = "%s cpufreq-set -c %d -g ondemand" % \ (self.change_user_cmd, cpu_n) stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False) if rc != 0: # Doesn't really matter if this fails, so just warn. warn("Could not set CPU%d governor to 'ondemand' when " "finished.\nFailing command:\n%s" % (cpu_n, cmd))
def _check_realtime_throttle_disabled(self): """Linux kernel gets pretty upset if you run a CPU intensive thread under the real-time thread schedule policy. By default Linux will artificially pre-empt such threads to give other things a chance to run on this core. A switch will flip at runtime leaving a message in dmesg when this comes into effect. See the "Limiting the CPU usage of real-time and deadline processes" section in sched(7). We don't want "throttling" on the benchmarking cores. From sched(7): "Specifying [sched_rt_runtime_us] -1 makes the runtime the same as the period; that is, no CPU time is set aside for non-real-time processes." """ debug("Check real-time thread throttling is off") for itr in xrange(2): with open(LinuxPlatform.SCHED_RT_RUNTIME_US) as fh: val = fh.read().strip() if val != "-1": if itr == 0: debug("%s is not -1, adjusting." % LinuxPlatform.SCHED_RT_RUNTIME_US) # Needs to happen as root args = self.change_user_args() + \ ["sh", "-c", "'echo -1 > %s'" % LinuxPlatform.SCHED_RT_RUNTIME_US] cmd = " ".join(args) run_shell_cmd(cmd) else: fatal("Could not set %s to -1" % LinuxPlatform.SCHED_RT_RUNTIME_US)
def _check_perf_samplerate(self): """Attempt to minimise time spent by the Linux perf kernel profiler. You can't disable this, so the best we can do is set the sample rate to the minimum value of one sample per second.""" with open(LinuxPlatform.PERF_SAMPLE_RATE) as fh: sr = int(fh.read().strip()) if sr != 1: cmd = "%s sh -c 'echo 1 > %s'" % \ (self.change_user_cmd, LinuxPlatform.PERF_SAMPLE_RATE) stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False) if rc != 0: fatal("perf profiler sample rate >1 p/s. " "Krun was unable to adjust it.\nFailing command:\n %s" % cmd)
def _check_aslr_disabled(self): debug("Checking ASLR is off") with open(self.ASLR_FILE, "r") as fh: enabled = fh.read().strip() if enabled == "0": return # OK! else: # ASLR is off, but we can try to enable it debug("Turning ASLR off") cmd = "%s sh -c 'echo 0 > %s'" % \ (self.change_user_cmd, self.ASLR_FILE) stdout, stderr, rc = run_shell_cmd(cmd, failure_fatal=False) if rc != 0: msg = "ASLR disabled (%s, expect '0' got '%s').\n" % \ (self.ASLR_FILE, enabled) msg += "Krun tried to turn it off, but failed." fatal(msg) else: self._check_aslr_disabled() # should work this time
def test_run_shell_cmd_fatal(): cmd = "nonsensecommand" out, err, rc = run_shell_cmd(cmd, False) assert rc != 0 assert cmd in err assert out == ""
def test_run_shell_cmd(): msg = "example text" out, err, rc = run_shell_cmd("echo " + msg) assert out == msg assert err == "" assert rc == 0
def _collect_dmesg_lines(self): return run_shell_cmd("dmesg")[0].split("\n")
def collect_audit(self): self.audit["uname"] = run_shell_cmd("uname -a")[0] self.audit["dmesg"] = run_shell_cmd("dmesg")[0] self.audit["krun_version"] = util.get_git_version() self.audit["cli_args"] = sys.argv
def inner_main(mailer, config, args): out_file = config.results_filename() out_file_exists = os.path.exists(out_file) if out_file_exists and not os.path.isfile(out_file): util.fatal( "Output file '%s' exists but is not a regular file" % out_file) if out_file_exists and not args.resume: util.fatal("Output file '%s' already exists. " "Either resume the session (--resume) or " "move the file away" % out_file) if not out_file_exists and args.resume: util.fatal("No results file to resume. Expected '%s'" % out_file) if args.started_by_init and not args.reboot: util.fatal("--started-by-init makes no sense without --reboot") if args.started_by_init and not args.resume: util.fatal("--started-by-init makes no sense without --resume") if args.develop: warn("Developer mode enabled. Results will not be reliable.") # Initialise platform instance and assign to VM defs. # This needs to be done early, so VM sanity checks can run. platform = detect_platform(mailer) if not args.develop: debug("Checking platform preliminaries") platform.check_preliminaries() else: # Needed to skip the use of certain tools and techniques. # E.g. switching user. warn("Not checking platform prerequisites due to developer mode") platform.developer_mode = True platform.collect_audit() # If the user has asked for resume-mode, the current platform must # be an identical machine to the current one. error_msg = ("You have asked Krun to resume an interrupted benchmark. " + "This is only valid if the machine you are using is " + "identical to the one on which the last results were " + "gathered, which is not the case.") current = None if args.resume: # output file must exist, due to check above assert(out_file_exists) current = Results(config, platform, results_file=out_file) from krun.audit import Audit if not Audit(platform.audit) == current.audit: util.fatal(error_msg) debug("Using pre-recorded initial temperature readings") platform.starting_temperatures = current.starting_temperatures else: # Touch the config file to update its mtime. This is required # by resume-mode which uses the mtime to determine the name of # the log file, should this benchmark be resumed. _, _, rc = util.run_shell_cmd("touch " + args.filename) if rc != 0: util.fatal("Could not touch config file: " + args.filename) info(("Wait %s secs to allow system to cool prior to " "collecting initial temperature readings") % config.TEMP_READ_PAUSE) if args.develop or args.dry_run: info("SIMULATED: time.sleep(%s)" % config.TEMP_READ_PAUSE) else: time.sleep(config.TEMP_READ_PAUSE) debug("Taking fresh initial temperature readings") platform.starting_temperatures = platform.take_temperature_readings() # Assign platform to VM defs -- needs to happen early for sanity checks util.assign_platform(config, platform) sanity_checks(config, platform) # Build job queue -- each job is an execution sched = ExecutionScheduler(config, mailer, platform, resume=args.resume, reboot=args.reboot, dry_run=args.dry_run, started_by_init=args.started_by_init) sched.build_schedule() sched.run()
def _get_apm_output(self): # separate for mocking return run_shell_cmd("apm")[0]
def _get_sysctl_sensor_lines(self): # separate for test mocking return run_shell_cmd(self.FIND_TEMP_SENSORS_CMD)[0]
def test_run_shell_cmd(): msg = "example text" out, err, rc = run_shell_cmd("echo " + msg) assert out == msg assert err == "" assert rc == 0
def test_run_shell_cmd_fatal(): cmd = "nonsensecommand" out, err, rc = run_shell_cmd(cmd, False) assert rc != 0 assert cmd in err assert out == ""
def collect_audit(self): BasePlatform.collect_audit(self) # Extra CPU info, some not in dmesg. E.g. CPU cache size. self.audit["cpuinfo"] = run_shell_cmd("cat /proc/cpuinfo")[0]
def collect_audit(self): LinuxPlatform.collect_audit(self) self.audit["packages"] = run_shell_cmd("dpkg-query -l")[0] self.audit["debian_version"] = run_shell_cmd("cat /etc/debian_version")[0]
def _save_power(self): run_shell_cmd("apm -C")