def runBenchmark(self, info, benchmark, platform):
model = benchmark["model"]
tests = benchmark["tests"]
assert len(tests) == 1, (
"At this point, only one test should " +
"exist in one benchmark. However, benchmark " +
"{} doesn't.".format(benchmark["name"]))
test = tests[0]
index = test["INDEX"] if "INDEX" in test else 0
first_iteration = index == 0
last_iteration = ("repeat"
not in model) or ("repeat" in model
and index == model["repeat"] - 1)
if self.host_platform is None:
self.host_platform = getHostPlatform(self.tempdir, self.args)
program_files = {
name: info["programs"][name]["location"]
for name in info["programs"]
}
program_path = (os.path.dirname(program_files["program"])
if "program" in program_files else None)
stringmap_from_info = info[
"string_map"] if "string_map" in info else None
self._replaceStringMap(benchmark, platform, program_path,
stringmap_from_info)
# better to be before target program files separation.
# this way, in ios, the platform may not be copied to the target.
platform.preprocess(programs=program_files, benchmark=benchmark)
tgt_program_files, host_program_files = self._separatePrograms(
program_files, test.get("commands"))
tgt_program_files = platform.copyFilesToPlatform(
tgt_program_files, copy_files=first_iteration)
programs = {}
deepMerge(programs, host_program_files)
deepMerge(programs, tgt_program_files)
model_files = {
name: model["files"][name]["location"]
for name in model["files"]
}
if "converter" in model:
converter = model["converter"]
assert "name" in converter, "converter field must have a name"
assert converter[
"name"] in self.converters, "Unknown converter {}".format(
converter)
else:
converter = None
output = {}
# inject default parameters into test
if "iter" not in test:
test["iter"] = -1
# overall preprocess
if "preprocess" in model and first_iteration:
commands = model["preprocess"]["commands"]
self._runCommands(
output,
commands,
self.host_platform,
programs,
model,
None,
model_files,
None,
None,
None,
None,
-1,
converter,
)
input_files = ({
name: test["input_files"][name]["location"]
for name in test["input_files"]
} if "input_files" in test else None)
test_files = (
{name: test["files"][name]["location"]
for name in test["files"]} if "files" in test else {})
# Let's handle preprocess comamnd first,
# since we will copy all files into host
if "preprocess" in test:
# simple thing first, let's assume preprocess is self contained
# check the program to executable
if ("files" in test["preprocess"]
and "program" in test["preprocess"]["files"]):
host_program_path = test["preprocess"]["files"]["program"][
"location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["preprocess"]:
preprocess_files = {
name: test["preprocess"]["files"][name]["location"]
for name in test["preprocess"]["files"]
}
deepMerge(test_files, preprocess_files)
if "commands" in test["preprocess"]:
commands = test["preprocess"]["commands"]
elif "command" in test["preprocess"]:
commands = [test["preprocess"]["command"]]
self._runCommands(
output,
commands,
self.host_platform,
programs,
model,
test,
model_files,
input_files,
None,
None,
test_files,
-1,
converter,
)
tgt_input_files = (platform.copyFilesToPlatform(input_files)
if input_files else None)
shared_libs = None
if "shared_libs" in info:
shared_libs = platform.copyFilesToPlatform(
info["shared_libs"], copy_files=first_iteration)
tgt_model_files = platform.copyFilesToPlatform(
model_files, copy_files=first_iteration)
tgt_result_files = None
if "output_files" in test:
tgt_result_files = {
name: test["output_files"][name]["location"]
for name in test["output_files"]
}
total_num = test["iter"]
if "platform_args" in test:
platform_args = test["platform_args"]
elif "platform_args" in model:
platform_args = model["platform_args"]
else:
platform_args = {}
if "timeout" in model:
platform_args["timeout"] = model["timeout"]
if "timeout" in test:
platform_args["timeout"] = test["timeout"]
program = programs["program"] if "program" in programs else ""
if test["metric"] == "power":
platform_args["power"] = True
method = test.get("method")
platform_args["method"] = method
if method == "software":
power_util = software_power.PowerUtil(
platform, test.get("collection_time", 300))
else:
# FIXME "Monsoon" was unimportable
from utils.monsoon_power import collectPowerData
# in power metric, the output is ignored
total_num = 0
platform.killProgram(program)
if test.get("env", False):
platform_args["env"] = test["env"]
if platform.getType() == "host":
# Fix the number of threads
if not platform_args.get("env", False):
platform_args["env"] = {}
MKL_NUM_THREADS = test.get("MKL_NUM_THREADS", 1)
OMP_NUM_THREADS = test.get("OMP_NUM_THREADS", 1)
if MKL_NUM_THREADS > 0:
platform_args["env"]["MKL_NUM_THREADS"] = MKL_NUM_THREADS
if OMP_NUM_THREADS > 0:
platform_args["env"]["OMP_NUM_THREADS"] = OMP_NUM_THREADS
# Randomly select one cpu core from logic cpu #4 to #13.
cpu_core = test.get("cpu-list", random.randint(5, 14))
if isinstance(test["commands"], list) and cpu_core > 0:
test["commands"][-1] = " ".join([
"taskset", "--cpu-list",
str(cpu_core), test["commands"][-1]
])
self._runCommands(
output,
test["commands"],
platform,
programs,
model,
test,
tgt_model_files,
tgt_input_files,
tgt_result_files,
shared_libs,
test_files,
total_num,
converter,
platform_args=platform_args,
main_command=True,
)
if test["metric"] == "power":
if test.get("method") == "software":
output = power_util.collect()
else:
collection_time = (test["collection_time"]
if "collection_time" in test else 180)
voltage = float(test["voltage"]) if "voltage" in test else 4.0
output = collectPowerData(
platform.platform_hash,
collection_time,
voltage,
test["iter"],
self.args.monsoon_map,
)
platform.waitForDevice(20)
# kill the process if exists
platform.killProgram(program)
# remove the files before copying out the output files
# this will save some time in ios platform, since in ios
# all files are copied back to the host system
if len(output) > 0:
if input_files is not None:
platform.delFilesFromPlatform(tgt_input_files)
if last_iteration:
platform.delFilesFromPlatform(tgt_model_files)
platform.delFilesFromPlatform(tgt_program_files)
if shared_libs is not None:
platform.delFilesFromPlatform(shared_libs)
output_files = None
if "output_files" in test:
target_dir = os.path.join(self.tempdir, "output")
shutil.rmtree(target_dir, True)
os.makedirs(target_dir)
output_files = platform.moveFilesFromPlatform(
tgt_result_files, target_dir)
platform.postprocess()
if "postprocess" in test:
if ("files" in test["postprocess"]
and "program" in test["preprocess"]["files"]):
host_program_path = test["postprocess"]["files"]["program"][
"location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["postprocess"]:
postprocess_files = {
name: test["postprocess"]["files"][name]["location"]
for name in test["postprocess"]["files"]
}
deepMerge(test_files, postprocess_files)
commands = None
if "commands" in test["postprocess"]:
commands = test["postprocess"]["commands"]
elif "command" in test["postprocess"]:
commands = [test["postprocess"]["command"]]
self._runCommands(
output,
commands,
self.host_platform,
programs,
model,
test,
model_files,
input_files,
output_files,
None,
test_files,
-1,
converter,
)
if "postprocess" in model and last_iteration:
commands = model["postprocess"]["commands"]
self._runCommands(
output,
commands,
self.host_platform,
programs,
model,
test,
model_files,
None,
None,
None,
None,
-1,
converter,
)
# after everything is done, some of the output files may
# contain metrics that can be processed. Those files have
# field converter, and specify which convert to use to
# convert the metrics
if output_files:
for filename in output_files:
file = output_files[filename]
converter = test["output_files"][filename].get("converter")
if not converter:
continue
assert "name" in converter, "converter field must have a name"
assert (converter["name"]
in self.converters), "Unknown converter {}".format(
converter["name"])
converter_class = self.converters[converter["name"]]
args = converter.get("args")
with open(file, "r") as f:
content = f.read()
convert = converter_class()
results, _ = convert.collect(content, args)
one_output = convert.convert(results)
deepMerge(output, one_output)
return output, output_files
def runBenchmark(self, info, benchmark, platform):
model = benchmark["model"]
tests = benchmark["tests"]
assert len(tests) == 1, "At this point, only one test should " + \
"exist in one benchmark. However, benchmark " + \
"{} doesn't.".format(benchmark["name"])
test = tests[0]
index = test["INDEX"] if "INDEX" in test else 0
first_iteration = index == 0
last_iteration = (("repeat" not in model) or
("repeat" in model and index == model["repeat"] - 1))
if self.host_platform is None:
self.host_platform = getHostPlatform(self.tempdir)
program_files = {
name: info["programs"][name]["location"]
for name in info["programs"]
}
program_path = os.path.dirname(program_files["program"]) \
if "program" in program_files else None
stringmap_from_info = info[
"string_map"] if "string_map" in info else None
self._replaceStringMap(benchmark, platform, program_path,
stringmap_from_info)
# better to be before target program files separation.
# this way, in ios, the platform may not be copied to the target.
platform.preprocess(programs=program_files, benchmark=benchmark)
tgt_program_files, host_program_files = \
self._separatePrograms(program_files, test.get("commands"))
tgt_program_files = \
platform.copyFilesToPlatform(tgt_program_files,
copy_files = first_iteration)
programs = {}
deepMerge(programs, host_program_files)
deepMerge(programs, tgt_program_files)
model_files = {
name: model["files"][name]["location"]
for name in model["files"]
}
if "converter" in model:
converter = model["converter"]
assert "name" in converter, "converter field must have a name"
assert converter["name"] in self.converters, \
"Unknown converter {}".format(converter)
else:
converter = None
output = {}
# overall preprocess
if "preprocess" in model and first_iteration:
commands = model["preprocess"]["commands"]
self._runCommands(output, commands, self.host_platform, programs,
model, None, model_files, None, None, None, None,
-1, converter)
input_files = {name: test["input_files"][name]["location"]
for name in test["input_files"]} \
if "input_files" in test else None
test_files = {
name: test["files"][name]["location"]
for name in test["files"]
} if "files" in test else {}
# Let's handle preprocess comamnd first,
# since we will copy all files into host
if "preprocess" in test:
# simple thing first, let's assume preprocess is self contained
# check the program to executable
if "files" in test["preprocess"] and \
"program" in test["preprocess"]["files"]:
host_program_path = \
test["preprocess"]["files"]["program"]["location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["preprocess"]:
preprocess_files = \
{name: test["preprocess"]["files"][name]["location"]
for name in test["preprocess"]["files"]}
deepMerge(test_files, preprocess_files)
if "commands" in test["preprocess"]:
commands = test["preprocess"]["commands"]
elif "command" in test["preprocess"]:
commands = [test["preprocess"]["command"]]
self._runCommands(output, commands, self.host_platform, programs,
model, test, model_files, input_files, None,
None, test_files, -1, converter)
tgt_input_files = platform.copyFilesToPlatform(input_files) \
if input_files else None
shared_libs = None
if "shared_libs" in info:
shared_libs = \
platform.copyFilesToPlatform(info["shared_libs"],
copy_files = first_iteration)
tgt_model_files = \
platform.copyFilesToPlatform(model_files,
copy_files = first_iteration)
tgt_result_files = None
if "output_files" in test:
tgt_result_files = {
name: test["output_files"][name]["location"]
for name in test["output_files"]
}
total_num = test["iter"]
if "platform_args" in test:
platform_args = test["platform_args"]
elif "platform_args" in model:
platform_args = model["platform_args"]
else:
platform_args = {}
if 'timeout' in model:
platform_args['timeout'] = model['timeout']
if 'timeout' in test:
platform_args['timeout'] = test['timeout']
program = programs["program"] if "program" in programs else ""
if test["metric"] == "power":
platform_args["power"] = True
# in power metric, the output is ignored
total_num = 0
platform.killProgram(program)
if test.get("env", False):
platform_args["env"] = test["env"]
self._runCommands(output,
test["commands"],
platform,
programs,
model,
test,
tgt_model_files,
tgt_input_files,
tgt_result_files,
shared_libs,
test_files,
total_num,
converter,
platform_args=platform_args)
if test["metric"] == "power":
collection_time = test["collection_time"] \
if "collection_time" in test else 180
voltage = float(test["voltage"]) if "voltage" in test else 4.0
from utils.monsoon_power import collectPowerData
output = collectPowerData(platform.platform_hash, collection_time,
voltage, test["iter"])
platform.waitForDevice(20)
# kill the process if exists
platform.killProgram(program)
# remove the files before copying out the output files
# this will save some time in ios platform, since in ios
# all files are copied back to the host system
if len(output) > 0:
if input_files is not None:
platform.delFilesFromPlatform(input_files)
if last_iteration:
platform.delFilesFromPlatform(tgt_model_files)
platform.delFilesFromPlatform(tgt_program_files)
if shared_libs is not None:
platform.delFilesFromPlatform(shared_libs)
output_files = None
if "output_files" in test:
target_dir = os.path.join(self.tempdir, "output")
shutil.rmtree(target_dir, True)
os.makedirs(target_dir)
output_files = \
platform.moveFilesFromPlatform(tgt_result_files, target_dir)
if "postprocess" in test:
if "files" in test["postprocess"] and \
"program" in test["preprocess"]["files"]:
host_program_path = \
test["postprocess"]["files"]["program"]["location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["postprocess"]:
postprocess_files = \
{name: test["postprocess"]["files"][name]["location"]
for name in test["postprocess"]["files"]}
deepMerge(test_files, postprocess_files)
commands = None
if "commands" in test["postprocess"]:
commands = test["postprocess"]["commands"]
elif "command" in test["postprocess"]:
commands = [test["postprocess"]["command"]]
self._runCommands(output, commands, self.host_platform, programs,
model, test, model_files, input_files,
output_files, None, test_files, -1, converter)
if "postprocess" in model and last_iteration:
commands = model["postprocess"]["commands"]
self._runCommands(output, commands, self.host_platform, programs,
model, test, model_files, None, None, None, None,
-1, converter)
# after everything is done, some of the output files may
# contain metrics that can be processed. Those files have
# field converter, and specify which convert to use to
# convert the metrics
if output_files:
for filename in output_files:
file = output_files[filename]
converter = \
test["output_files"][filename].get("converter")
if not converter:
continue
assert "name" in converter, "converter field must have a name"
assert converter["name"] in self.converters, \
"Unknown converter {}".format(converter["name"])
converter_class = self.converters[converter["name"]]
args = converter.get("args")
with open(file, "r") as f:
content = f.read()
convert = converter_class()
results, _ = convert.collect(content, args)
one_output = convert.convert(results)
deepMerge(output, one_output)
return output, output_files
def runBenchmark(self, info, benchmark, platform):
model = benchmark["model"]
tests = benchmark["tests"]
assert len(tests) == 1, "At this point, only one test should " + \
"exist in one benchmark. However, benchmark " + \
"{} doesn't.".format(benchmark["name"])
test = tests[0]
index = test["INDEX"] if "INDEX" in test else 0
first_iteration = index == 0
last_iteration = (("repeat" not in model) or
("repeat" in model and index == model["repeat"] - 1))
if self.host_platform is None:
self.host_platform = getHostPlatform(self.tempdir)
self._replaceStringMap(benchmark, platform)
program_files = {name: info["programs"][name]["location"]
for name in info["programs"]}
# better to be before target program files separation.
# this way, in ios, the platform may not be copied to the target.
platform.preprocess(programs=program_files)
tgt_program_files, host_program_files = \
self._separatePrograms(program_files, test["commands"])
# we need to copy programs in all iterations, because this is
# how we get the absolute path of the programs in the target platform
# may consider optimize this later that only copying for the first
# iteration
programs = platform.copyFilesToPlatform(tgt_program_files)
deepMerge(programs, host_program_files)
model_files = {name: model["files"][name]["location"]
for name in model["files"]}
if "converter" in model:
converter_name = model["converter"]
assert converter_name in self.converters, \
"Unknown converter {}".format(converter_name)
converter = self.converters[converter_name]
else:
converter = None
log_output = {"log_output": True}
output = {}
# overall preprocess
if "preprocess" in model and first_iteration:
commands = model["preprocess"]["commands"]
self._runCommands(output, commands, self.host_platform, programs,
model, None, model_files, None, None, None,
None, -1, log_output, converter)
input_files = {name: test["input_files"][name]["location"]
for name in test["input_files"]} \
if "input_files" in test else None
test_files = {name: test["files"][name]["location"]
for name in test["files"]} if "files" in test else {}
# Let's handle preprocess comamnd first,
# since we will copy all files into host
if "preprocess" in test:
# simple thing first, let's assume preprocess is self contained
# check the program to executable
if "files" in test["preprocess"] and \
"program" in test["preprocess"]["files"]:
host_program_path = \
test["preprocess"]["files"]["program"]["location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["preprocess"]:
preprocess_files = \
{name: test["preprocess"]["files"][name]["location"]
for name in test["preprocess"]["files"]}
deepMerge(test_files, preprocess_files)
if "commands" in test["preprocess"]:
commands = test["preprocess"]["commands"]
elif "command" in test["preprocess"]:
commands = [test["preprocess"]["command"]]
self._runCommands(output, commands, self.host_platform, programs,
model,
test, model_files, input_files, None, None,
test_files, -1, log_output, converter)
tgt_input_files = platform.copyFilesToPlatform(input_files) \
if input_files else None
shared_libs = None
if "shared_libs" in info:
shared_libs = platform.copyFilesToPlatform(info["shared_libs"])
# We need to copy the model files in every iteration, because this
# is how we get the absolute path in the target platform,
# will optimize that later.
tgt_model_files = platform.copyFilesToPlatform(model_files)
tgt_result_files = None
if "output_files" in test:
tgt_result_files = {name: test["output_files"][name]["location"]
for name in test["output_files"]}
total_num = test["iter"]
if "platform_args" in test:
platform_args = test["platform_args"]
elif "platform_args" in model:
platform_args = model["platform_args"]
else:
platform_args = {}
if sys.version_info > (3, 0):
if 'timeout' in model:
platform_args['timeout'] = model['timeout']
if 'timeout' in test:
platform_args['timeout'] = test['timeout']
program = programs["program"] if "program" in programs else ""
if test["metric"] == "power":
platform_args["power"] = True
# in power metric, the output is ignored
total_num = 0
platform.killProgram(program)
if test.get("log_output", False):
platform_args["log_output"] = True
self._runCommands(output, test["commands"], platform, programs, model,
test, tgt_model_files, tgt_input_files,
tgt_result_files, shared_libs, test_files,
total_num, platform_args, converter)
if test["metric"] == "power":
collection_time = test["collection_time"] \
if "collection_time" in test else 180
voltage = float(test["voltage"]) if "voltage" in test else 4.0
from utils.monsoon_power import collectPowerData
output = collectPowerData(platform.platform_hash,
collection_time, voltage, test["iter"])
platform.waitForDevice(20)
# kill the process if exists
platform.killProgram(program)
# remove the files before copying out the output files
# this will save some time in ios platform, since in ios
# all files are copied back to the host system
if len(output) > 0:
platform.delFilesFromPlatform(tgt_model_files)
platform.delFilesFromPlatform(program)
if shared_libs is not None:
platform.delFilesFromPlatform(shared_libs)
if input_files is not None:
platform.delFilesFromPlatform(input_files)
output_files = None
if "output_files" in test:
target_dir = os.path.join(self.tempdir, "output")
shutil.rmtree(target_dir, True)
os.makedirs(target_dir)
output_files = \
platform.moveFilesFromPlatform(tgt_result_files, target_dir)
if "postprocess" in test:
if "files" in test["postprocess"] and \
"program" in test["preprocess"]["files"]:
host_program_path = \
test["postprocess"]["files"]["program"]["location"]
os.chmod(host_program_path, 0o777)
# will deprecate in the future
if "files" in test["postprocess"]:
postprocess_files = \
{name: test["postprocess"]["files"][name]["location"]
for name in test["postprocess"]["files"]}
deepMerge(test_files, postprocess_files)
commands = test["postprocess"]["commands"]
self._runCommands(output, commands, self.host_platform, programs,
model, test, model_files, input_files,
output_files, None, test_files, -1, log_output,
converter)
if "postprocess" in model and last_iteration:
commands = model["postprocess"]["commands"]
self._runCommands(output, commands, self.host_platform, programs,
model, test, model_files, None, None, None, None,
-1, log_output, converter)
return output, output_files