def emulate_first_reboot(platform, config):
platform.starting_temperatures = platform.take_temperature_readings()
manifest = ManifestManager(config, platform, new_file=True)
manifest.set_starting_temperatures(platform.starting_temperatures)
results = Results(config, platform)
results.write_to_file()
return manifest
def test_write_new_manifest0001(mock_platform):
_setup(BLANK_EXAMPLE_MANIFEST)
config = Config(os.path.join(TEST_DIR, "example.krun"))
manifest1 = ManifestManager(config, mock_platform, new_file=True)
manifest2 = ManifestManager(
config, mock_platform) # reads the file in from the last line
assert manifest1 == manifest2
_tear_down(manifest2.path)
def _setup(contents):
class FakeConfig(object):
filename = os.path.join(TEST_DIR, "manifest_tests.krun")
config = FakeConfig()
with open(ManifestManager.get_filename(config), "w") as fh:
fh.write(contents)
return ManifestManager(config, MockPlatform(None, config))
def emulate_first_reboot(platform, config, monkeypatch):
no_results_instantiation_check(monkeypatch)
platform.starting_temperatures = platform.take_temperature_readings()
manifest = ManifestManager(config, platform, new_file=True)
manifest.set_starting_temperatures(platform.starting_temperatures)
results = Results(config, platform)
results.write_to_file()
return manifest
def test_write_new_manifest0002(mock_platform):
manifest_path = "example_000.manifest"
config_path = os.path.join(TEST_DIR, "more_complicated.krun")
config = Config(config_path)
manifest = ManifestManager(config, mock_platform, new_file=True)
assert manifest.total_num_execs == 90 # taking into account skips
_tear_down(manifest.path)
def _setup(contents):
class FakeConfig(object):
filename = os.path.join(TEST_DIR, "manifest_tests.krun")
config = FakeConfig()
with open(ManifestManager.get_filename(config), "w") as fh:
fh.write(contents)
return ManifestManager(config, MockPlatform(None, config))
def test_get_session_info0002():
path = os.path.join(TEST_DIR, "more_complicated.krun")
config = Config(path)
info = get_session_info(config)
# 6 benchmarks, 9 VMs, skipped 3 exact keys, and all 6 CPython keys
# Then two repetitions (process executions) of all of the above.
expect_proc_execs = (6 * 9 - 3 - 6) * 2
assert info["n_proc_execs"] == expect_proc_execs
# 2000 in-process iterations
assert info["n_in_proc_iters"] == expect_proc_execs * 2000
expect_skip_keys = [
"fasta:TruffleRuby:default-ruby",
"richards:HHVM:default-php",
"spectralnorm:TruffleRuby:default-ruby",
"binarytrees:CPython:default-python",
"richards:CPython:default-python",
"spectralnorm:CPython:default-python",
"nbody:CPython:default-python",
"fasta:CPython:default-python",
"fannkuch_redux:CPython:default-python",
]
assert info["skipped_keys"] == set(expect_skip_keys)
expect_non_skipped_keys = [
'richards:C:default-c', 'nbody:HHVM:default-php',
'binarytrees:C:default-c', 'binarytrees:PyPy:default-python',
'spectralnorm:Hotspot:default-java',
'fannkuch_redux:Graal:default-java', 'nbody:TruffleRuby:default-ruby',
'fasta:Graal:default-java', 'binarytrees:Graal:default-java',
'fasta:C:default-c', 'binarytrees:TruffleRuby:default-ruby',
'spectralnorm:HHVM:default-php', 'nbody:PyPy:default-python',
'fannkuch_redux:C:default-c',
'fannkuch_redux:TruffleRuby:default-ruby',
'fannkuch_redux:Hotspot:default-java',
'spectralnorm:PyPy:default-python', 'fasta:PyPy:default-python',
'binarytrees:Hotspot:default-java', 'nbody:C:default-c',
'richards:TruffleRuby:default-ruby', 'fasta:V8:default-javascript',
'nbody:V8:default-javascript', 'richards:V8:default-javascript',
'nbody:LuaJIT:default-lua', 'richards:Hotspot:default-java',
'fasta:LuaJIT:default-lua', 'binarytrees:LuaJIT:default-lua',
'fannkuch_redux:V8:default-javascript',
'fannkuch_redux:LuaJIT:default-lua', 'richards:Graal:default-java',
'binarytrees:V8:default-javascript', 'spectralnorm:LuaJIT:default-lua',
'spectralnorm:C:default-c', 'fannkuch_redux:HHVM:default-php',
'fannkuch_redux:PyPy:default-python', 'binarytrees:HHVM:default-php',
'fasta:HHVM:default-php', 'spectralnorm:V8:default-javascript',
'spectralnorm:Graal:default-java', 'nbody:Graal:default-java',
'richards:LuaJIT:default-lua', 'nbody:Hotspot:default-java',
'richards:PyPy:default-python', 'fasta:Hotspot:default-java'
]
assert info["non_skipped_keys"] == set(expect_non_skipped_keys)
# There should be no overlap in the used and skipped keys
assert info["skipped_keys"].intersection(info["non_skipped_keys"]) == set()
os.unlink(ManifestManager.get_filename(config))
def get_session_info(config):
"""Gets information about the session (for --info)
Overwrites any existing manifest file.
Separated from print_session_info for ease of testing"""
from krun.scheduler import ManifestManager
from krun.platform import detect_platform
platform = detect_platform(None, config)
manifest = ManifestManager(config, platform, new_file=True)
return {
"n_proc_execs": manifest.total_num_execs,
"n_in_proc_iters": manifest.get_total_in_proc_iters(config),
"skipped_keys": manifest.skipped_keys,
"non_skipped_keys": manifest.non_skipped_keys,
}
def example_manifest(mock_platform):
# setup
config = Config(os.path.join(TEST_DIR, "example.krun"))
manifest = ManifestManager(config, mock_platform, new_file=True)
yield manifest
# teardown
if os.path.exists(manifest.path):
os.unlink(manifest.path)
def test_get_session_info0001():
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
info = get_session_info(config)
assert info["n_proc_execs"] == 8
assert info["n_in_proc_iters"] == 40
assert info["skipped_keys"] == set()
expect_non_skipped_keys = set([
"dummy:Java:default-java",
"nbody:Java:default-java",
"dummy:CPython:default-python",
"nbody:CPython:default-python",
])
assert info["non_skipped_keys"] == expect_non_skipped_keys
os.unlink(ManifestManager.get_filename(config))
def test_get_session_info0001():
path = os.path.join(TEST_DIR, "example.krun")
config = Config(path)
info = get_session_info(config)
assert info["n_proc_execs"] == 8
assert info["n_in_proc_iters"] == 40
assert info["skipped_keys"] == set()
expect_non_skipped_keys = set([
"dummy:Java:default-java",
"nbody:Java:default-java",
"dummy:CPython:default-python",
"nbody:CPython:default-python",
])
assert info["non_skipped_keys"] == expect_non_skipped_keys
os.unlink(ManifestManager.get_filename(config))
def test_get_session_info0002():
path = os.path.join(TEST_DIR, "more_complicated.krun")
config = Config(path)
info = get_session_info(config)
# 6 benchmarks, 9 VMs, skipped 3 exact keys, and all 6 CPython keys
# Then two repetitions (process executions) of all of the above.
expect_proc_execs = (6 * 9 - 3 - 6) * 2
assert info["n_proc_execs"] == expect_proc_execs
# 2000 in-process iterations
assert info["n_in_proc_iters"] == expect_proc_execs * 2000
expect_skip_keys = [
"fasta:TruffleRuby:default-ruby",
"richards:HHVM:default-php",
"spectralnorm:TruffleRuby:default-ruby",
"binarytrees:CPython:default-python",
"richards:CPython:default-python",
"spectralnorm:CPython:default-python",
"nbody:CPython:default-python",
"fasta:CPython:default-python",
"fannkuch_redux:CPython:default-python",
]
assert info["skipped_keys"] == set(expect_skip_keys)
expect_non_skipped_keys = [
'richards:C:default-c',
'nbody:HHVM:default-php',
'binarytrees:C:default-c',
'binarytrees:PyPy:default-python',
'spectralnorm:Hotspot:default-java',
'fannkuch_redux:Graal:default-java',
'nbody:TruffleRuby:default-ruby',
'fasta:Graal:default-java',
'binarytrees:Graal:default-java',
'fasta:C:default-c',
'binarytrees:TruffleRuby:default-ruby',
'spectralnorm:HHVM:default-php',
'nbody:PyPy:default-python',
'fannkuch_redux:C:default-c',
'fannkuch_redux:TruffleRuby:default-ruby',
'fannkuch_redux:Hotspot:default-java',
'spectralnorm:PyPy:default-python',
'fasta:PyPy:default-python',
'binarytrees:Hotspot:default-java',
'nbody:C:default-c',
'richards:TruffleRuby:default-ruby',
'fasta:V8:default-javascript',
'nbody:V8:default-javascript',
'richards:V8:default-javascript',
'nbody:LuaJIT:default-lua',
'richards:Hotspot:default-java',
'fasta:LuaJIT:default-lua',
'binarytrees:LuaJIT:default-lua',
'fannkuch_redux:V8:default-javascript',
'fannkuch_redux:LuaJIT:default-lua',
'richards:Graal:default-java',
'binarytrees:V8:default-javascript',
'spectralnorm:LuaJIT:default-lua',
'spectralnorm:C:default-c',
'fannkuch_redux:HHVM:default-php',
'fannkuch_redux:PyPy:default-python',
'binarytrees:HHVM:default-php',
'fasta:HHVM:default-php',
'spectralnorm:V8:default-javascript',
'spectralnorm:Graal:default-java',
'nbody:Graal:default-java',
'richards:LuaJIT:default-lua',
'nbody:Hotspot:default-java',
'richards:PyPy:default-python',
'fasta:Hotspot:default-java'
]
assert info["non_skipped_keys"] == set(expect_non_skipped_keys)
# There should be no overlap in the used and skipped keys
assert info["skipped_keys"].intersection(info["non_skipped_keys"]) == set()
os.unlink(ManifestManager.get_filename(config))
def inner_main(mailer, on_first_invocation, config, args):
out_file = config.results_filename()
out_file_exists = os.path.exists(out_file)
instr_dir = util.get_instr_json_dir(config)
instr_dir_exists = os.path.exists(instr_dir)
envlog_dir = util.get_envlog_dir(config)
envlog_dir_exists = os.path.exists(envlog_dir)
if out_file_exists and not os.path.isfile(out_file):
util.fatal("Output file '%s' exists but is not a regular file" %
out_file)
if out_file_exists and on_first_invocation:
util.fatal("Output results file '%s' already exists. "
"Move the file away before running Krun." % out_file)
if instr_dir_exists and on_first_invocation:
util.fatal("Instrumentation dir '%s' exists." % instr_dir)
if envlog_dir_exists and on_first_invocation:
util.fatal("Env log dir '%s' exists." % envlog_dir)
if not out_file_exists and not on_first_invocation:
util.fatal("No results file to resume. Expected '%s'" % out_file)
# Initialise platform instance and assign to VM defs.
# This needs to be done early, so VM sanity checks can run.
platform = detect_platform(mailer, config)
platform.quick_mode = args.quick
platform.no_user_change = args.no_user_change
platform.no_tickless_check = args.no_tickless_check
platform.no_pstate_check = args.no_pstate_check
platform.hardware_reboots = args.hardware_reboots
# Create the instrumentation directory if required
if on_first_invocation:
# We only want make a dir if >=1 VM is in instrumentation mode.
for vm in config.VMS.itervalues():
if vm['vm_def'].instrument:
util.make_instr_dir(config)
break
debug("Checking platform preliminaries")
platform.check_preliminaries()
# Make a bit of noise if this is a virtualised environment
if platform.is_virtual():
warn(
"This appears to be a virtualised host. The results will be flawed. "
"Use bare-metal for reliable results!")
platform.collect_audit()
# At this point the config file is OK, and on-disk state is consistent,
# so let's daemonise (if requested).
if args.daemonise:
util.daemonise()
if not on_first_invocation:
# output file must exist, due to check above
assert (out_file_exists)
debug("Using pre-recorded initial temperature readings")
manifest = ManifestManager(config, platform)
platform_temps = {}
for sensor, tup in manifest.starting_temperatures.iteritems():
platform_temps[sensor] = tup[1]
platform.starting_temperatures = platform_temps
else:
manifest = ManifestManager(config, platform, new_file=True)
if manifest.num_execs_left == 0:
# No executions, or all skipped
fatal("Empty schedule!")
try:
info(("Wait %s secs to allow system to cool prior to "
"collecting initial temperature readings") %
config.TEMP_READ_PAUSE)
# This part is wrapped in hooks, so that if daemons or networking are
# taken down for process executions, then the initial temperature
# reading gets the same treatment.
util.run_shell_cmd_list(config.PRE_EXECUTION_CMDS, )
platform.sleep(config.TEMP_READ_PAUSE)
debug("Taking fresh initial temperature readings")
platform.starting_temperatures = platform.take_temperature_readings(
)
manifest.set_starting_temperatures(platform.starting_temperatures)
# Write out an empty results file. After the initial reboot Krun
# will expect this to exist.
Results.ok_to_instantiate = True
results = Results(config, platform)
results.write_to_file()
except:
raise
finally:
util.run_shell_cmd_list(config.POST_EXECUTION_CMDS, )
log_path = config.log_filename(resume=False)
util.log_and_mail(mailer,
debug,
"Benchmarking started",
"Benchmarking started.\nLogging to %s" % log_path,
bypass_limiter=True)
util.reboot(manifest, platform)
# Assign platform to VM defs -- needs to happen early for sanity checks
util.assign_platform(config, platform)
sanity_checks(config, platform)
# Build job queue -- each job is an execution
sched = ExecutionScheduler(config, mailer, platform, dry_run=args.dry_run)
sched.run()
def main(parser):
args = parser.parse_args()
if args.dump is not None:
if not args.filename.endswith(".json.bz2"):
usage(parser)
else:
Results.ok_to_instantiate = True
results = Results(None, None, results_file=args.filename)
text = results.dump(args.dump)
# String data read in from JSON are unicode objects. This matters
# for us as some data in the audit includes unicode characters.
# If it does, a simple print no longer suffices if the system
# locale is (e.g.) ASCII. In this case print will raise an
# exception. The correct thing to do is to encode() the unicode to
# the system locale.
print(text.encode(locale.getpreferredencoding()))
sys.exit(0)
if not args.filename.endswith(".krun"):
usage(parser)
try:
if os.stat(args.filename).st_size <= 0:
util.fatal('Krun configuration file %s is empty.' % args.filename)
except OSError:
util.fatal('Krun configuration file %s does not exist.' %
args.filename)
config = Config(args.filename)
if args.info:
# Info mode doesn't run the experiment.
# Just prints some metrics and exits.
util.print_session_info(config)
return
manifest_filename = ManifestManager.get_filename(config)
on_first_invocation = not (os.path.isfile(manifest_filename)
and os.stat(manifest_filename).st_size > 0)
log_file = config.log_filename()
if on_first_invocation and os.path.exists(log_file):
util.fatal("Log file '%s' already exists. "
"Move the file away before running Krun." % log_file)
attach_log_file(config, not on_first_invocation)
debug("Krun invoked with arguments: %s" % sys.argv)
mail_recipients = config.MAIL_TO
if type(mail_recipients) is not list:
util.fatal("MAIL_TO config should be a list")
mailer = Mailer(mail_recipients, max_mails=config.MAX_MAILS)
try:
inner_main(mailer, on_first_invocation, config, args)
except Exception as exn:
error_info = sys.exc_info()
subject = "Fatal Krun Exception"
lines = ["Fatal Krun error: %s\n" % str(error_info[1])]
for frame in traceback.format_tb(error_info[2]):
lines.append(frame)
msg = "".join(lines)
util.log_and_mail(mailer, debug, subject, msg, bypass_limiter=True)
raise exn
def inner_main(mailer, on_first_invocation, config, args):
out_file = config.results_filename()
out_file_exists = os.path.exists(out_file)
instr_dir = util.get_instr_json_dir(config)
instr_dir_exists = os.path.exists(instr_dir)
envlog_dir = util.get_envlog_dir(config)
envlog_dir_exists = os.path.exists(envlog_dir)
if out_file_exists and not os.path.isfile(out_file):
util.fatal(
"Output file '%s' exists but is not a regular file" % out_file)
if out_file_exists and on_first_invocation:
util.fatal("Output results file '%s' already exists. "
"Move the file away before running Krun." % out_file)
if instr_dir_exists and on_first_invocation:
util.fatal("Instrumentation dir '%s' exists." % instr_dir)
if envlog_dir_exists and on_first_invocation:
util.fatal("Env log dir '%s' exists." % envlog_dir)
if not out_file_exists and not on_first_invocation:
util.fatal("No results file to resume. Expected '%s'" % out_file)
# Initialise platform instance and assign to VM defs.
# This needs to be done early, so VM sanity checks can run.
platform = detect_platform(mailer, config)
platform.quick_mode = args.quick
platform.no_user_change = args.no_user_change
platform.no_tickless_check = args.no_tickless_check
platform.no_pstate_check = args.no_pstate_check
platform.hardware_reboots = args.hardware_reboots
# Create the instrumentation directory if required
if on_first_invocation:
# We only want make a dir if >=1 VM is in instrumentation mode.
for vm in config.VMS.itervalues():
if vm['vm_def'].instrument:
util.make_instr_dir(config)
break
debug("Checking platform preliminaries")
platform.check_preliminaries()
# Make a bit of noise if this is a virtualised environment
if platform.is_virtual():
warn("This appears to be a virtualised host. The results will be flawed. "
"Use bare-metal for reliable results!")
platform.collect_audit()
# At this point the config file is OK, and on-disk state is consistent,
# so let's daemonise (if requested).
if args.daemonise:
util.daemonise()
if not on_first_invocation:
# output file must exist, due to check above
assert(out_file_exists)
debug("Using pre-recorded initial temperature readings")
manifest = ManifestManager(config, platform)
platform_temps = {}
for sensor, tup in manifest.starting_temperatures.iteritems():
platform_temps[sensor] = tup[1]
platform.starting_temperatures = platform_temps
else:
manifest = ManifestManager(config, platform, new_file=True)
if manifest.num_execs_left == 0:
# No executions, or all skipped
fatal("Empty schedule!")
try:
info(("Wait %s secs to allow system to cool prior to "
"collecting initial temperature readings") %
config.TEMP_READ_PAUSE)
# This part is wrapped in hooks, so that if daemons or networking are
# taken down for process executions, then the initial temperature
# reading gets the same treatment.
util.run_shell_cmd_list(config.PRE_EXECUTION_CMDS,)
platform.sleep(config.TEMP_READ_PAUSE)
debug("Taking fresh initial temperature readings")
platform.starting_temperatures = platform.take_temperature_readings()
manifest.set_starting_temperatures(platform.starting_temperatures)
# Write out an empty results file. After the initial reboot Krun
# will expect this to exist.
Results.ok_to_instantiate = True
results = Results(config, platform)
results.write_to_file()
except:
raise
finally:
util.run_shell_cmd_list(config.POST_EXECUTION_CMDS,)
log_path = config.log_filename(resume=False)
util.log_and_mail(mailer, debug,
"Benchmarking started",
"Benchmarking started.\nLogging to %s" %
log_path, bypass_limiter=True)
util.reboot(manifest, platform)
# Assign platform to VM defs -- needs to happen early for sanity checks
util.assign_platform(config, platform)
sanity_checks(config, platform)
# Build job queue -- each job is an execution
sched = ExecutionScheduler(config,
mailer,
platform,
dry_run=args.dry_run)
sched.run()
def main(parser):
args = parser.parse_args()
if args.dump is not None:
if not args.filename.endswith(".json.bz2"):
usage(parser)
else:
Results.ok_to_instantiate = True
results = Results(None, None, results_file=args.filename)
text = results.dump(args.dump)
# String data read in from JSON are unicode objects. This matters
# for us as some data in the audit includes unicode characters.
# If it does, a simple print no longer suffices if the system
# locale is (e.g.) ASCII. In this case print will raise an
# exception. The correct thing to do is to encode() the unicode to
# the system locale.
print(text.encode(locale.getpreferredencoding()))
sys.exit(0)
if not args.filename.endswith(".krun"):
usage(parser)
try:
if os.stat(args.filename).st_size <= 0:
util.fatal('Krun configuration file %s is empty.' % args.filename)
except OSError:
util.fatal('Krun configuration file %s does not exist.' % args.filename)
config = Config(args.filename)
if args.info:
# Info mode doesn't run the experiment.
# Just prints some metrics and exits.
util.print_session_info(config)
return
manifest_filename = ManifestManager.get_filename(config)
on_first_invocation = not (os.path.isfile(manifest_filename) and
os.stat(manifest_filename).st_size > 0)
log_file = config.log_filename()
if on_first_invocation and os.path.exists(log_file):
util.fatal("Log file '%s' already exists. "
"Move the file away before running Krun." % log_file)
attach_log_file(config, not on_first_invocation)
debug("Krun invoked with arguments: %s" % sys.argv)
mail_recipients = config.MAIL_TO
if type(mail_recipients) is not list:
util.fatal("MAIL_TO config should be a list")
mailer = Mailer(mail_recipients, max_mails=config.MAX_MAILS)
try:
inner_main(mailer, on_first_invocation, config, args)
except Exception as exn:
error_info = sys.exc_info()
subject = "Fatal Krun Exception"
lines = ["Fatal Krun error: %s\n" % str(error_info[1])]
for frame in traceback.format_tb(error_info[2]):
lines.append(frame)
msg = "".join(lines)
util.log_and_mail(mailer, debug, subject, msg, bypass_limiter=True)
raise exn