def test_tflite_engine(self):
from core.global_config import create_config
framework_name = "tflite"
config_dict = create_config(framework_name)
config_dict["work_dir"] = os.getcwd() + "/../tflite"
tflite = Engine(config_dict)
tflite.set_config("benchmark_platform", ["android-armv7"]) # noqa
tflite.set_config(
"support_backend",
["ARM", "ARM_XNNPACK", "GPU_CL_GL", "DSP_HEXAGON"], # noqa
) # ARM，ARM_XNNPACK, GPU_CL_GL, DSP_HEXAGON
tflite.set_config("cpu_thread_num", [1, 2, 4]) # [1, 2, 4]
model_dict = tflite.prepare_models()
device_dict = tflite.prepare_devices()
if len(device_dict) == 0:
logger.error("no device found")
return 0
config_dict = tflite.set_config("model_dict", model_dict)
config_dict = tflite.set_config("device_dict", device_dict)
tflite.prepare_models_for_devices()
tflite.prepare_benchmark_assets_for_devices()
bench_dict = tflite.run_bench()
summary_list = tflite.generate_benchmark_summary(bench_dict)
summary_str = "\n".join(summary_list)
logger.info("summary_str:\n{}".format(summary_str))
tflite.write_list_to_file(summary_list)
return 0
def test_mnn_engine(self):
from core.global_config import create_config
framework_name = "mnn"
config_dict = create_config(framework_name)
config_dict["work_dir"] = os.getcwd() + "/../mnn"
mnn = Engine(config_dict)
mnn.set_config("benchmark_platform",
["android-armv8", "android-armv7"]) # noqa
mnn.set_config(
"support_backend",
["3", "6", "7"]) # 0->CPU，1->Metal，3->OpenCL，6->OpenGL，7->Vulkan
mnn.set_config("cpu_thread_num", [1, 2, 4]) # [1, 2, 4]
mnn.config["warmup"] = 2
model_dict = mnn.prepare_models()
device_dict = mnn.prepare_devices()
if len(device_dict) == 0:
logger.error("no device found")
return 0
config_dict = mnn.set_config("model_dict", model_dict)
config_dict = mnn.set_config("device_dict", device_dict)
mnn.prepare_models_for_devices()
mnn.prepare_benchmark_assets_for_devices()
bench_dict = mnn.run_bench()
summary_list = mnn.generate_benchmark_summary(bench_dict)
summary_str = "\n".join(summary_list)
logger.info("summary_str:\n{}".format(summary_str))
mnn.write_list_to_file(summary_list)
return 0
def run_cmd(cmd, wait_interval_sec=5, max_timeout_sec=100):
cmd_type = "CMD"
logger.info("{}> {}".format(cmd_type, cmd))
subp = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
subp.wait(max_timeout_sec)
subp_status = -1
duration_sec = 0
while 1:
if subp.poll() == 0:
logger.debug("{} finished".format(cmd_type))
subp_status = int(subp.poll())
break
elif subp.poll() is None:
logger.debug("{} duration += {} second".format(
cmd_type, wait_interval_sec) # noqa
) # noqa
time.sleep(wait_interval_sec)
duration_sec += wait_interval_sec
if duration_sec > max_timeout_sec:
logger.error(
"{} duration {} second timeout with max_timeout_sec {}".
format( # noqa
cmd_type, duration_sec, max_timeout_sec))
subp.kill()
break
else:
subp_status = subp.poll()
logger.fatal(
"exited with abnormal subprocess status: {}".format( # noqa
subp_status) # noqa
)
if subp_status == 139:
break
else:
exit(1)
logger.debug("{} consume {} seconds to finish".format(
cmd_type, duration_sec) # noqa
) # noqa
cmd_res = None
if subp_status == 0 or subp_status == 139:
cmd_res = "".join(subp.communicate())
logger.debug("cmd_res:{}".format(subp.communicate()))
cmd_res = cmd_res.split("\n")
cmd_res = filter(lambda r: r != "", cmd_res)
cmd_res = list(cmd_res)
return cmd_res
def run_bench_multi_threads(self):
logger.info("==== {} ====".format(
self.run_bench_multi_threads.__name__) # noqa
)
device_serials = list(self.config["device_dict"].keys())
device_threads = dict()
for didx in range(len(device_serials)):
thread_idx = didx
ser = device_serials[didx]
logger.info(
"create device(thread)_idx(from1):{}/{}, serial_num:{}".format(
didx + 1, len(device_serials), ser) # noqa
)
device_threads[ser] = MyThread(
func=self.run_bench_for_single_thread_func,
func_args_tuple=(
ser, # noqa
didx, # noqa
thread_idx, # noqa
len(device_serials), # noqa
self.config["framework_name"],
), # noqa
device_serial=ser,
thread_idx=thread_idx,
thread_num=len(device_serials),
framework_name=self.config["framework_name"],
)
assert len(device_threads) == len(device_serials)
for tidx in range(len(device_threads)):
ser = device_serials[tidx]
device_threads[ser].start()
for tidx in range(len(device_threads)):
ser = device_serials[tidx]
device_threads[ser].join()
bench_dict = dict()
for tidx in range(len(device_threads)):
ser = device_serials[tidx]
res = device_threads[ser].get_result()
if res is None:
# TODO(ysh329): need watch out for failed device
# when benchmark some framework
logger.error("device {} result is None,"
" skipped and continue".format(ser) # noqa
)
continue
bench_dict[ser] = res[ser]
return bench_dict
def test_ncnn_engine(self):
from core.global_config import create_config
framework_name = "ncnn"
config_dict = create_config(framework_name)
config_dict["work_dir"] = os.getcwd() + "/../ncnn"
ncnn = Engine(config_dict)
ncnn.set_config(
"benchmark_platform", ["android-armv8", "android-armv7"] # noqa
)
ncnn.set_config("support_backend", ["0"]) # -1: cpu, 0: gpu # noqa
ncnn.set_config("cpu_thread_num", [1, 2, 4])
ncnn.config["repeats"] = (
lambda backend: 100 if backend == "VULKAN" else 1000
) # noqa
ncnn.config["warmup"] = 2
model_dict = ncnn.prepare_models()
device_dict = ncnn.prepare_devices()
if len(device_dict) == 0:
logger.error("no device found")
return 0
config_dict = ncnn.set_config("model_dict", model_dict)
config_dict = ncnn.set_config("device_dict", device_dict)
ncnn.prepare_models_for_devices()
ncnn.prepare_benchmark_assets_for_devices()
bench_dict = ncnn.run_bench()
summary_list = ncnn.generate_benchmark_summary(bench_dict)
ncnn.write_list_to_file(summary_list)
summary_str = "\n".join(summary_list)
logger.info("summary_str:\n{}".format(summary_str))
return 0
