def parse_benchmark(self, cmd_res):
logger.info("==== {} ====".format(self.parse_benchmark.__name__))
output_lines = cmd_res
logger.debug(output_lines)
framework_name = self.config["framework_name"]
benchmark = dict()
if framework_name == "tnn" or framework_name == "mnn":
if framework_name == "tnn":
bench_res_keyword = "time cost"
elif framework_name == "mnn":
bench_res_keyword = "max ="
output_lines = filter( # noqa
lambda line: bench_res_keyword in line, output_lines)
output_lines = list(output_lines)
assert len(output_lines) == 1
line = output_lines[0].split()
logger.debug(line)
line = "".join(line)
logger.debug(line)
benchmark["min"] = pattern_match(line, "min=", "ms", False)
benchmark["max"] = pattern_match(line, "max=", "ms", False)
benchmark["avg"] = pattern_match(line, "avg=", "ms", False)
benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False)
elif framework_name == "ncnn":
is_no_vulkan = list(
filter(lambda line: "no vulkan device" in line, output_lines))
is_no_vulkan = len(is_no_vulkan) > 0
if is_no_vulkan:
benchmark["min"] = 0.0
benchmark["max"] = 0.0
benchmark["avg"] = 0.0
benchmark["std_dev"] = 0.0
else:
output_lines = filter(lambda line: "min = " in line,
output_lines) # noqa
output_lines = list(output_lines)
logger.debug("output_lines:\n{}".format(output_lines))
assert len(output_lines) == 1
line = output_lines[0]
line = line.split()
logger.debug(line)
line = "".join(line) + "END"
benchmark["min"] = pattern_match(line, "min=", "max", False)
benchmark["max"] = pattern_match(line, "max=", "avg", False)
benchmark["avg"] = pattern_match(line, "avg=", "std_dev",
False) # noqa
benchmark["std_dev"] = pattern_match(line, "std_dev=", "END",
False) # noqa
else:
logger.fatal("Unsupported framework {}".format( # noqa
self.config["framework_name"] # noqa
) # noqa
)
exit(1)
logger.info("benchmark:{}".format(benchmark))
assert len(benchmark) != 0
return benchmark
def run_cmd(cmd, wait_interval_sec=5, max_timeout_sec=100):
cmd_type = "CMD"
logger.info("{}> {}".format(cmd_type, cmd))
subp = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
subp.wait(max_timeout_sec)
subp_status = -1
duration_sec = 0
while 1:
if subp.poll() == 0:
logger.debug("{} finished".format(cmd_type))
subp_status = int(subp.poll())
break
elif subp.poll() is None:
logger.debug("{} duration += {} second".format(
cmd_type, wait_interval_sec) # noqa
) # noqa
time.sleep(wait_interval_sec)
duration_sec += wait_interval_sec
if duration_sec > max_timeout_sec:
logger.error(
"{} duration {} second timeout with max_timeout_sec {}".
format( # noqa
cmd_type, duration_sec, max_timeout_sec))
subp.kill()
break
else:
subp_status = subp.poll()
logger.fatal(
"exited with abnormal subprocess status: {}".format( # noqa
subp_status) # noqa
)
if subp_status == 139:
break
else:
exit(1)
logger.debug("{} consume {} seconds to finish".format(
cmd_type, duration_sec) # noqa
) # noqa
cmd_res = None
if subp_status == 0 or subp_status == 139:
cmd_res = "".join(subp.communicate())
logger.debug("cmd_res:{}".format(subp.communicate()))
cmd_res = cmd_res.split("\n")
cmd_res = filter(lambda r: r != "", cmd_res)
cmd_res = list(cmd_res)
return cmd_res
def prepare_models_for_devices(self):
logger.info("==== {} ====".format(
self.prepare_models_for_devices.__name__)) # noqa
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
model_dict = self.config["model_dict"]
device_serials = list(device_dict.keys())
model_names = list(model_dict.keys())
logger.debug(model_dict)
cmds = list()
for didx in range(len(device_serials)):
device_serial = device_serials[didx]
mkdir_cmd = "adb -s {} shell mkdir -p {}".format(
device_serial, device_work_dir)
cmds.append(mkdir_cmd)
for midx in range(len(model_names)):
model_name = model_names[midx]
model_proto = model_dict[model_name]
if (self.config["framework_name"] == "ncnn"
or self.config["framework_name"] == "tnn"):
model_param = model_proto.replace("tnnproto", "tnnmodel")
elif (self.config["framework_name"] == "mnn"
or self.config["framework_name"] == "tflite"):
model_param = None
else:
logger.fatal("Unsupported framework {}".format( # noqa
self.config["framework_name"] # noqa
) # noqa
)
exit(1)
push_proto_cmd = "adb -s {} push {} {}".format(
device_serial,
model_proto,
"/".join([device_work_dir,
os.path.basename(model_proto)]), # noqa
)
push_param_cmd = "adb -s {} push {} {}".format(
device_serial, model_param, device_work_dir)
push_param_cmd = (
"echo" if model_param is None else push_param_cmd) # noqa
cmds.extend([push_proto_cmd, push_param_cmd])
run_cmds(cmds)
return 0
def parse_benchmark(self, cmd_res):
logger.info("==== {} ====".format(self.parse_benchmark.__name__))
output_lines = cmd_res
logger.debug(output_lines)
framework_name = self.config["framework_name"]
benchmark = dict()
benchmark["min"] = 0.0
benchmark["max"] = 0.0
benchmark["avg"] = 0.0
benchmark["std_dev"] = 0.0
if cmd_res is None:
return benchmark
elif (framework_name == "tnn" or framework_name == "mnn"
or framework_name == "tflite"):
if framework_name == "tnn" or framework_name == "tflite":
bench_res_keyword = "time cost"
elif framework_name == "mnn":
bench_res_keyword = "max ="
output_lines_str = "".join(output_lines)
output_lines = filter( # noqa
lambda line: bench_res_keyword in line, output_lines)
if (framework_name == "mnn"
and "Floating point exception" in output_lines_str):
return benchmark
elif (framework_name == "tflite" and
'Error: dlopen failed: library "libhexagon_interface.so" not found' # noqa
in output_lines_str):
return benchmark
output_lines = list(output_lines)
assert len(output_lines) == 1
line = output_lines[0].split()
logger.debug(line)
line = "".join(line)
logger.debug(line)
benchmark["min"] = pattern_match(line, "min=", "ms", False)
benchmark["max"] = pattern_match(line, "max=", "ms", False)
benchmark["avg"] = pattern_match(line, "avg=", "ms", False)
benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False)
elif framework_name == "ncnn":
is_no_vulkan = list(
filter(
lambda line: "no vulkan device" in line or
'"libvulkan.so" not found' in line,
output_lines,
))
is_no_vulkan = len(is_no_vulkan) > 0
if is_no_vulkan:
return benchmark
else:
output_lines = filter(lambda line: "min = " in line,
output_lines) # noqa
output_lines = list(output_lines)
logger.debug("output_lines:\n{}".format(output_lines))
assert len(output_lines) == 1
line = output_lines[0]
line = line.split()
logger.debug(line)
line = "".join(line) + "END"
benchmark["min"] = pattern_match(line, "min=", "max", False)
benchmark["max"] = pattern_match(line, "max=", "avg", False)
benchmark["avg"] = pattern_match(line, "avg=", "std_dev",
False) # noqa
benchmark["std_dev"] = pattern_match(line, "std_dev=", "END",
False) # noqa
else:
logger.fatal("Unsupported framework {}".format( # noqa
self.config["framework_name"] # noqa
) # noqa
)
exit(1)
logger.info("benchmark:{}".format(benchmark))
assert len(benchmark) != 0
return benchmark
def run_bench_for_single_thread_func(
self,
device_serial,
device_idx=-1,
thread_idx=-1,
thread_num=-1,
framework_name="",
):
logger.info( # noqa
"==== {}, thread_idx(from0):{}/{} ====".format(
self.run_bench_for_single_thread_func.__name__,
thread_idx,
thread_num, # noqa
))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
cur_device_dict = device_dict[device_serial]
logger.info("cur_device_dict:{}".format(cur_device_dict))
model_dict = self.config["model_dict"]
model_names = list(model_dict.keys())
platforms = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
# note(ysh329): this bench_dict is for single thread about device
bench_dict = dict()
bench_dict[device_serial] = dict()
cpu_backend_num = list(
map(self.config["is_cpu_backend"], support_backend)).count(True)
bench_case_num = (
len(platforms) * len(model_names) * sum([
len(self.config["cpu_thread_num"]) * cpu_backend_num +
len(set(support_backend) - set(["ARM", "ARM_XNNPACK"])) # noqa
]))
logger.info("len(platform):{}".format(len(platforms)))
logger.info("len(model_names):{}".format(len(model_names)))
logger.info(
'len(self.config["cpu_thread_num"]) * cpu_backend_num:{}'.format(
len(self.config["cpu_thread_num"]) * cpu_backend_num))
logger.info("support_backend:{}".format(support_backend))
logger.info(
'len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")):{}'.
format( # noqa
len(set(support_backend) - set("ARM") - set("ARM_XNNPACK"))))
logger.info("bench_case_num:{}".format(bench_case_num))
logger.info(
'len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0:{}'
.format( # noqa
len(self.config["cpu_thread_num"])
if "CPU" in support_backend else 0 # noqa
))
logger.info('len(set(support_backend) - set("ARM")):{}'.format(
len(set(support_backend) - set("ARM"))))
bench_case_idx = 0
# platform: armv7/armv8/...
for pidx in range(len(platforms)):
platform = platforms[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join([
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]), # noqa
])
bench_dict[device_serial][platform] = dict()
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
# model: mobilenetv1/v2/...
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join([
device_work_dir,
os.path.basename(model_dict[model_name]),
] # noqa
)
logger.debug("midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir))
bench_dict[device_serial][platform][model_name] = dict(
) # noqa
# backend: cpu/gpu/xpu/...
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
logger.debug("bidx:{}, backend:{}".format(bidx,
backend)) # noqa
bench_dict[device_serial][platform][model_name][
backend] = dict() # noqa
# thread: 1/2/4/...
for tidx in range(
len(self.config["cpu_thread_num"]) if self.
config["is_cpu_backend"](backend) else 1 # noqa
):
bench_case_idx += 1
logger.info(
"\n\nframework_name:{}, device_idx(from1):{}/{}, bench_case_idx(from1):{}/{}," # noqa
" enable_multi_threads:{}, thread_idx(from0):{}/{}"
.format( # noqa
# noqa
self.engine_name(), # noqa
device_idx + 1,
len(device_dict), # noqa
bench_case_idx,
bench_case_num, # noqa
self.config["enable_multi_threads"], # noqa
thread_idx,
thread_num, # noqa
) # noqa
)
cpu_thread_num = (
self.config["cpu_thread_num"][tidx]
if self.config["is_cpu_backend"](backend) else 1
) # noqa
bench_dict[device_serial][platform][
model_name][ # noqa
backend][cpu_thread_num] = dict() # noqa
#######################
# bench case start
#######################
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_type":
self.config["framework_name"], # noqa
"model_dir":
model_dir,
"backend":
backend,
"repeats":
self.config["repeats"](backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"bind_cpu_idx":
device_dict[device_serial][ # noqa
"bind_cpu_idx"],
})
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial,
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode":
self.config["power_mode_id"], # noqa
"gpu_device": backend,
})
elif self.config["framework_name"] == "mnn":
# '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa
# '{forwardtype} {thread_num} {precision}"'
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"forwardtype": backend,
# power_mode: big_core default
})
elif self.config["framework_name"] == "tflite":
# {device_benchmark_bin} --graph={model_dir}
# --output_prefix={model_name}
# --num_runs={repeats} --warmup_runs={warmup}
# --num_threads={thread_num} {backend}
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir":
model_dir,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"backend":
self.config["support_backend_cmd_id"]
( # noqa
backend),
# power_mode(TODO): no bind deafault? need explore deeply # noqa
})
print(bench_cmd)
else:
logger.fatal("Unsupported framework {}".format(
self.config["framework_name"]))
exit(1)
#################################
# run benchmark
#################################
run_times_sum = (
self.config["repeats"](backend) +
self.config["warmup"] # noqa
) # noqa
max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER *
run_times_sum) # noqa
cmd_res = run_cmd(
bench_cmd,
wait_interval_sec=3,
max_timeout_sec=max_wait_sec,
) # noqa
perf_dict = self.parse_benchmark(cmd_res)
#################################
# summarize benchmark info
#################################
bench_record = {
"soc_code":
device_dict[device_serial]["soc_code"], ## noqa
"product":
device_dict[device_serial][ # noqa
"product"], # noqa
"serial_num":
device_serial,
"platform":
platform,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend),
"warmup":
self.config["warmup"],
"avg":
perf_dict["avg"],
"max":
perf_dict["max"],
"min":
perf_dict["min"],
"std_dev":
perf_dict["std_dev"],
"backend":
backend,
"cpu_thread_num":
cpu_thread_num,
"power_mode":
self.config["power_mode"],
"bind_cpu_idx":
device_dict[device_serial]["bind_cpu_idx"], # noqa
"cpu_max_freqs":
device_dict[device_serial][ # noqa
"cpu_max_freqs"],
"battery_level":
device_dict[device_serial][ # noqa
"battery_level"],
"system_version":
device_dict[device_serial][ # noqa
"system_version"],
"cmd":
bench_cmd,
"imei":
device_dict[device_serial]["imei"],
}
bench_dict[device_serial][platform][
model_name][ # noqa
backend][cpu_thread_num] = bench_record # noqa
logger.info(bench_record)
return bench_dict
def generate_benchmark_summary(self, bench_dict, is_print_summary=True):
logger.info(
"==== {} ====".format(self.generate_benchmark_summary.__name__)
) # noqa
summary_header = [
"framework",
"branch",
"commit_id",
"model_name",
"platform",
"soc",
"product",
"power_mode",
"backend",
"cpu_thread_num",
"avg",
"max",
"min",
"battery_level",
"system_version",
"repeats",
"warmup",
"imei",
]
summary_header_str = ",".join(summary_header)
summary = [summary_header_str]
model_names = list(bench_dict.keys())
model_names.sort()
model_num = len(model_names)
for midx in range(model_num):
model_name = model_names[midx]
bench_records = bench_dict[model_name]
logger.info("midx:{}/{},{}".format(midx + 1, model_num, model_name)) # noqa
for ridx in range(len(bench_records)):
record_dict = bench_records[ridx]
logger.info(record_dict)
if self.config["framework_name"] == "tnn":
record = [
self.config["framework_name"],
self.config["framework_repo_branch"],
self.config["framework_repo_commit_id"],
record_dict["model_name"],
record_dict["platform"],
record_dict["soc"],
record_dict["product"],
record_dict["power_mode"],
record_dict["backend"],
record_dict["cpu_thread_num"],
record_dict["avg"],
record_dict["max"],
record_dict["min"],
record_dict["battery_level"],
record_dict["system_version"],
record_dict["repeats"],
record_dict["warmup"],
record_dict["imei"],
]
elif self.config["framework_name"] == "ncnn":
backend_id_to_str_dict = self.config[
"backend_id_to_str_dict"
] # noqa
record = [
self.config["framework_name"],
self.config["framework_repo_branch"],
self.config["framework_repo_commit_id"],
record_dict["model_name"],
record_dict["platform"],
record_dict["soc"],
record_dict["product"],
record_dict["power_mode"],
backend_id_to_str_dict[record_dict["backend"]],
record_dict["cpu_thread_num"],
record_dict["avg"],
record_dict["max"],
record_dict["min"],
record_dict["battery_level"],
record_dict["system_version"],
record_dict["repeats"],
record_dict["warmup"],
record_dict["imei"],
]
else:
logger.fatal(
"Unsupported framework name:{}".format(
self.config["framework_name"]
)
)
exit(1)
record_str = ",".join(map(str, record))
if True:
logger.info(record_str)
summary.append(record_str)
if is_print_summary:
summary_str = "\n".join(summary)
logger.info("\n" + summary_str)
return summary
def run_bench(self):
logger.info("==== {} ====".format(self.run_bench.__name__))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
model_dict = self.config["model_dict"]
device_serials = list(device_dict.keys())
model_names = list(model_dict.keys())
benchmark_platform = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
bench_dict = dict()
bench_case_idx = 0
for didx in range(len(device_serials)):
device_serial_num = device_serials[didx]
logger.debug("didx:{}, serial_num:{}".format(
didx, device_serial_num) # noqa
)
bench_dict[device_serial_num] = dict()
for pidx in range(len(benchmark_platform)):
platform = benchmark_platform[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join([
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]), # noqa
])
bench_dict[device_serial_num][platform] = dict()
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join([
device_work_dir,
os.path.basename(model_dict[model_name]),
] # noqa
)
logger.debug("midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir))
bench_dict[device_serial_num][platform][model_name] = dict(
) # noqa
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
is_cpu = self.config["is_cpu_backend"]
logger.debug("bidx:{}, backend:{}".format(
bidx, backend)) # noqa
bench_dict[device_serial_num][platform][model_name][
backend] = dict() # noqa
for tidx in range(
len(self.config["cpu_thread_num"]
) if is_cpu(backend) else 1 # noqa
):
bench_case_idx += 1
logger.info("\n\nbench_case_idx(from 1):{}".format(
bench_case_idx) # noqa
)
cpu_thread_num = self.config["cpu_thread_num"][
tidx] # noqa
bench_dict[device_serial_num][platform][
model_name][ # noqa
backend][cpu_thread_num] = dict() # noqa
#######################
# bench start
#######################
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_type":
self.config["framework_name"], # noqa
"model_dir":
model_dir,
"backend":
backend,
"repeats":
self.config["repeats"](
backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"bind_cpu_idx":
device_dict[device_serial_num][ # noqa
"bind_cpu_idx"],
})
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode":
self.config["power_mode_id"], # noqa
"gpu_device": backend,
})
elif self.config["framework_name"] == "mnn":
# '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa
# '{forwardtype} {thread_num} {precision}"'
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"forwardtype": backend,
# power_mode: big_core default
})
else:
logger.fatal("Unsupported framework {}".format(
self.config["framework_name"]))
exit(1)
cmd_res = run_cmd(
bench_cmd,
wait_interval_sec=3 # noqa
) # noqa
perf_dict = self.parse_benchmark(cmd_res)
#################################
# summarize benchmark info
#################################
bench_record = {
"soc":
device_dict[device_serial_num]["soc"],
"product":
device_dict[device_serial_num][ # noqa
"product"], # noqa
"serial_num":
device_serial_num,
"platform":
platform,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend),
"warmup":
self.config["warmup"],
"avg":
perf_dict["avg"],
"max":
perf_dict["max"],
"min":
perf_dict["min"],
"std_dev":
perf_dict["std_dev"],
"backend":
backend,
"cpu_thread_num":
cpu_thread_num,
"power_mode":
self.config["power_mode"],
"bind_cpu_idx":
device_dict[device_serial_num]["bind_cpu_idx"],
"cpu_max_freqs":
device_dict[device_serial_num][ # noqa
"cpu_max_freqs"],
"battery_level":
device_dict[device_serial_num][ # noqa
"battery_level"],
"system_version":
device_dict[device_serial_num][ # noqa
"system_version"],
"cmd":
bench_cmd,
"imei":
device_dict[device_serial_num]["imei"],
}
bench_dict[device_serial_num][platform][
model_name][ # noqa
backend][
cpu_thread_num] = bench_record # noqa
# bench_dict[model_name].append(bench_record)
logger.info(bench_record)
return bench_dict
def gen_bench_cmds(self):
logger.info("==== {} ====".format(self.gen_bench_cmds.__name__))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
model_dict = self.config["model_dict"]
device_serials = list(device_dict.keys())
model_names = list(model_dict.keys())
benchmark_platform = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
bench_dict = dict()
bench_case_idx = 0
for didx in range(len(device_serials)):
device_serial_num = device_serials[didx]
logger.debug("didx:{}, serial_num:{}".format(
didx, device_serial_num) # noqa
)
bench_dict[device_serial_num] = dict()
bench_cmds = []
for pidx in range(len(benchmark_platform)):
platform = benchmark_platform[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join([
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]), # noqa
])
bench_dict[device_serial_num][platform] = dict()
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join([
device_work_dir,
os.path.basename(model_dict[model_name]),
] # noqa
)
logger.debug("midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir))
bench_dict[device_serial_num][platform][model_name] = dict(
) # noqa
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
is_cpu = self.config["is_cpu_backend"]
logger.debug("bidx:{}, backend:{}".format(
bidx, backend)) # noqa
bench_dict[device_serial_num][platform][model_name][
backend] = dict() # noqa
for tidx in range(
len(self.config["cpu_thread_num"]
) if is_cpu(backend) else 1 # noqa
):
bench_case_idx += 1
logger.info("\n\nbench_case_idx(from 1):{}".format(
bench_case_idx) # noqa
)
cpu_thread_num = self.config["cpu_thread_num"][
tidx] # noqa
bench_dict[device_serial_num][platform][
model_name][ # noqa
backend][cpu_thread_num] = dict() # noqa
#######################
# bench start
#######################
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_type":
self.config["framework_name"], # noqa
"model_dir":
model_dir,
"backend":
backend,
"repeats":
self.config["repeats"](
backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"bind_cpu_idx":
device_dict[device_serial_num][ # noqa
"bind_cpu_idx"],
})
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode":
self.config["power_mode_id"], # noqa
"gpu_device": backend,
})
elif self.config["framework_name"] == "mnn":
# '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa
# '{forwardtype} {thread_num} {precision}"'
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"forwardtype": backend,
# power_mode: big_core default
})
else:
logger.fatal("Unsupported framework {}".format(
self.config["framework_name"]))
continue
bench_cmds.append(bench_cmd)
bench_dict[device_serial_num]["bench_cmds"] = bench_cmds
return bench_dict
