def parse_benchmark(self, cmd_res): logger.info("==== {} ====".format(self.parse_benchmark.__name__)) output_lines = cmd_res logger.debug(output_lines) framework_name = self.config["framework_name"] benchmark = dict() if framework_name == "tnn" or framework_name == "mnn": if framework_name == "tnn": bench_res_keyword = "time cost" elif framework_name == "mnn": bench_res_keyword = "max =" output_lines = filter( # noqa lambda line: bench_res_keyword in line, output_lines) output_lines = list(output_lines) assert len(output_lines) == 1 line = output_lines[0].split() logger.debug(line) line = "".join(line) logger.debug(line) benchmark["min"] = pattern_match(line, "min=", "ms", False) benchmark["max"] = pattern_match(line, "max=", "ms", False) benchmark["avg"] = pattern_match(line, "avg=", "ms", False) benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False) elif framework_name == "ncnn": is_no_vulkan = list( filter(lambda line: "no vulkan device" in line, output_lines)) is_no_vulkan = len(is_no_vulkan) > 0 if is_no_vulkan: benchmark["min"] = 0.0 benchmark["max"] = 0.0 benchmark["avg"] = 0.0 benchmark["std_dev"] = 0.0 else: output_lines = filter(lambda line: "min = " in line, output_lines) # noqa output_lines = list(output_lines) logger.debug("output_lines:\n{}".format(output_lines)) assert len(output_lines) == 1 line = output_lines[0] line = line.split() logger.debug(line) line = "".join(line) + "END" benchmark["min"] = pattern_match(line, "min=", "max", False) benchmark["max"] = pattern_match(line, "max=", "avg", False) benchmark["avg"] = pattern_match(line, "avg=", "std_dev", False) # noqa benchmark["std_dev"] = pattern_match(line, "std_dev=", "END", False) # noqa else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) logger.info("benchmark:{}".format(benchmark)) assert len(benchmark) != 0 return benchmark
def run_cmd(cmd, wait_interval_sec=5, max_timeout_sec=100): cmd_type = "CMD" logger.info("{}> {}".format(cmd_type, cmd)) subp = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", ) subp.wait(max_timeout_sec) subp_status = -1 duration_sec = 0 while 1: if subp.poll() == 0: logger.debug("{} finished".format(cmd_type)) subp_status = int(subp.poll()) break elif subp.poll() is None: logger.debug("{} duration += {} second".format( cmd_type, wait_interval_sec) # noqa ) # noqa time.sleep(wait_interval_sec) duration_sec += wait_interval_sec if duration_sec > max_timeout_sec: logger.error( "{} duration {} second timeout with max_timeout_sec {}". format( # noqa cmd_type, duration_sec, max_timeout_sec)) subp.kill() break else: subp_status = subp.poll() logger.fatal( "exited with abnormal subprocess status: {}".format( # noqa subp_status) # noqa ) if subp_status == 139: break else: exit(1) logger.debug("{} consume {} seconds to finish".format( cmd_type, duration_sec) # noqa ) # noqa cmd_res = None if subp_status == 0 or subp_status == 139: cmd_res = "".join(subp.communicate()) logger.debug("cmd_res:{}".format(subp.communicate())) cmd_res = cmd_res.split("\n") cmd_res = filter(lambda r: r != "", cmd_res) cmd_res = list(cmd_res) return cmd_res
def prepare_models_for_devices(self): logger.info("==== {} ====".format( self.prepare_models_for_devices.__name__)) # noqa device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) logger.debug(model_dict) cmds = list() for didx in range(len(device_serials)): device_serial = device_serials[didx] mkdir_cmd = "adb -s {} shell mkdir -p {}".format( device_serial, device_work_dir) cmds.append(mkdir_cmd) for midx in range(len(model_names)): model_name = model_names[midx] model_proto = model_dict[model_name] if (self.config["framework_name"] == "ncnn" or self.config["framework_name"] == "tnn"): model_param = model_proto.replace("tnnproto", "tnnmodel") elif (self.config["framework_name"] == "mnn" or self.config["framework_name"] == "tflite"): model_param = None else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) push_proto_cmd = "adb -s {} push {} {}".format( device_serial, model_proto, "/".join([device_work_dir, os.path.basename(model_proto)]), # noqa ) push_param_cmd = "adb -s {} push {} {}".format( device_serial, model_param, device_work_dir) push_param_cmd = ( "echo" if model_param is None else push_param_cmd) # noqa cmds.extend([push_proto_cmd, push_param_cmd]) run_cmds(cmds) return 0
def parse_benchmark(self, cmd_res): logger.info("==== {} ====".format(self.parse_benchmark.__name__)) output_lines = cmd_res logger.debug(output_lines) framework_name = self.config["framework_name"] benchmark = dict() benchmark["min"] = 0.0 benchmark["max"] = 0.0 benchmark["avg"] = 0.0 benchmark["std_dev"] = 0.0 if cmd_res is None: return benchmark elif (framework_name == "tnn" or framework_name == "mnn" or framework_name == "tflite"): if framework_name == "tnn" or framework_name == "tflite": bench_res_keyword = "time cost" elif framework_name == "mnn": bench_res_keyword = "max =" output_lines_str = "".join(output_lines) output_lines = filter( # noqa lambda line: bench_res_keyword in line, output_lines) if (framework_name == "mnn" and "Floating point exception" in output_lines_str): return benchmark elif (framework_name == "tflite" and 'Error: dlopen failed: library "libhexagon_interface.so" not found' # noqa in output_lines_str): return benchmark output_lines = list(output_lines) assert len(output_lines) == 1 line = output_lines[0].split() logger.debug(line) line = "".join(line) logger.debug(line) benchmark["min"] = pattern_match(line, "min=", "ms", False) benchmark["max"] = pattern_match(line, "max=", "ms", False) benchmark["avg"] = pattern_match(line, "avg=", "ms", False) benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False) elif framework_name == "ncnn": is_no_vulkan = list( filter( lambda line: "no vulkan device" in line or '"libvulkan.so" not found' in line, output_lines, )) is_no_vulkan = len(is_no_vulkan) > 0 if is_no_vulkan: return benchmark else: output_lines = filter(lambda line: "min = " in line, output_lines) # noqa output_lines = list(output_lines) logger.debug("output_lines:\n{}".format(output_lines)) assert len(output_lines) == 1 line = output_lines[0] line = line.split() logger.debug(line) line = "".join(line) + "END" benchmark["min"] = pattern_match(line, "min=", "max", False) benchmark["max"] = pattern_match(line, "max=", "avg", False) benchmark["avg"] = pattern_match(line, "avg=", "std_dev", False) # noqa benchmark["std_dev"] = pattern_match(line, "std_dev=", "END", False) # noqa else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) logger.info("benchmark:{}".format(benchmark)) assert len(benchmark) != 0 return benchmark
def run_bench_for_single_thread_func( self, device_serial, device_idx=-1, thread_idx=-1, thread_num=-1, framework_name="", ): logger.info( # noqa "==== {}, thread_idx(from0):{}/{} ====".format( self.run_bench_for_single_thread_func.__name__, thread_idx, thread_num, # noqa )) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] cur_device_dict = device_dict[device_serial] logger.info("cur_device_dict:{}".format(cur_device_dict)) model_dict = self.config["model_dict"] model_names = list(model_dict.keys()) platforms = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] # note(ysh329): this bench_dict is for single thread about device bench_dict = dict() bench_dict[device_serial] = dict() cpu_backend_num = list( map(self.config["is_cpu_backend"], support_backend)).count(True) bench_case_num = ( len(platforms) * len(model_names) * sum([ len(self.config["cpu_thread_num"]) * cpu_backend_num + len(set(support_backend) - set(["ARM", "ARM_XNNPACK"])) # noqa ])) logger.info("len(platform):{}".format(len(platforms))) logger.info("len(model_names):{}".format(len(model_names))) logger.info( 'len(self.config["cpu_thread_num"]) * cpu_backend_num:{}'.format( len(self.config["cpu_thread_num"]) * cpu_backend_num)) logger.info("support_backend:{}".format(support_backend)) logger.info( 'len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")):{}'. format( # noqa len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")))) logger.info("bench_case_num:{}".format(bench_case_num)) logger.info( 'len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0:{}' .format( # noqa len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0 # noqa )) logger.info('len(set(support_backend) - set("ARM")):{}'.format( len(set(support_backend) - set("ARM")))) bench_case_idx = 0 # platform: armv7/armv8/... for pidx in range(len(platforms)): platform = platforms[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) # model: mobilenetv1/v2/... for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial][platform][model_name] = dict( ) # noqa # backend: cpu/gpu/xpu/... for bidx in range(len(support_backend)): backend = support_backend[bidx] logger.debug("bidx:{}, backend:{}".format(bidx, backend)) # noqa bench_dict[device_serial][platform][model_name][ backend] = dict() # noqa # thread: 1/2/4/... for tidx in range( len(self.config["cpu_thread_num"]) if self. config["is_cpu_backend"](backend) else 1 # noqa ): bench_case_idx += 1 logger.info( "\n\nframework_name:{}, device_idx(from1):{}/{}, bench_case_idx(from1):{}/{}," # noqa " enable_multi_threads:{}, thread_idx(from0):{}/{}" .format( # noqa # noqa self.engine_name(), # noqa device_idx + 1, len(device_dict), # noqa bench_case_idx, bench_case_num, # noqa self.config["enable_multi_threads"], # noqa thread_idx, thread_num, # noqa ) # noqa ) cpu_thread_num = ( self.config["cpu_thread_num"][tidx] if self.config["is_cpu_backend"](backend) else 1 ) # noqa bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench case start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) elif self.config["framework_name"] == "tflite": # {device_benchmark_bin} --graph={model_dir} # --output_prefix={model_name} # --num_runs={repeats} --warmup_runs={warmup} # --num_threads={thread_num} {backend} bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "model_name": model_name, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "backend": self.config["support_backend_cmd_id"] ( # noqa backend), # power_mode(TODO): no bind deafault? need explore deeply # noqa }) print(bench_cmd) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) exit(1) ################################# # run benchmark ################################# run_times_sum = ( self.config["repeats"](backend) + self.config["warmup"] # noqa ) # noqa max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER * run_times_sum) # noqa cmd_res = run_cmd( bench_cmd, wait_interval_sec=3, max_timeout_sec=max_wait_sec, ) # noqa perf_dict = self.parse_benchmark(cmd_res) ################################# # summarize benchmark info ################################# bench_record = { "soc_code": device_dict[device_serial]["soc_code"], ## noqa "product": device_dict[device_serial][ # noqa "product"], # noqa "serial_num": device_serial, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "std_dev": perf_dict["std_dev"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial]["bind_cpu_idx"], # noqa "cpu_max_freqs": device_dict[device_serial][ # noqa "cpu_max_freqs"], "battery_level": device_dict[device_serial][ # noqa "battery_level"], "system_version": device_dict[device_serial][ # noqa "system_version"], "cmd": bench_cmd, "imei": device_dict[device_serial]["imei"], } bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = bench_record # noqa logger.info(bench_record) return bench_dict
def generate_benchmark_summary(self, bench_dict, is_print_summary=True): logger.info( "==== {} ====".format(self.generate_benchmark_summary.__name__) ) # noqa summary_header = [ "framework", "branch", "commit_id", "model_name", "platform", "soc", "product", "power_mode", "backend", "cpu_thread_num", "avg", "max", "min", "battery_level", "system_version", "repeats", "warmup", "imei", ] summary_header_str = ",".join(summary_header) summary = [summary_header_str] model_names = list(bench_dict.keys()) model_names.sort() model_num = len(model_names) for midx in range(model_num): model_name = model_names[midx] bench_records = bench_dict[model_name] logger.info("midx:{}/{},{}".format(midx + 1, model_num, model_name)) # noqa for ridx in range(len(bench_records)): record_dict = bench_records[ridx] logger.info(record_dict) if self.config["framework_name"] == "tnn": record = [ self.config["framework_name"], self.config["framework_repo_branch"], self.config["framework_repo_commit_id"], record_dict["model_name"], record_dict["platform"], record_dict["soc"], record_dict["product"], record_dict["power_mode"], record_dict["backend"], record_dict["cpu_thread_num"], record_dict["avg"], record_dict["max"], record_dict["min"], record_dict["battery_level"], record_dict["system_version"], record_dict["repeats"], record_dict["warmup"], record_dict["imei"], ] elif self.config["framework_name"] == "ncnn": backend_id_to_str_dict = self.config[ "backend_id_to_str_dict" ] # noqa record = [ self.config["framework_name"], self.config["framework_repo_branch"], self.config["framework_repo_commit_id"], record_dict["model_name"], record_dict["platform"], record_dict["soc"], record_dict["product"], record_dict["power_mode"], backend_id_to_str_dict[record_dict["backend"]], record_dict["cpu_thread_num"], record_dict["avg"], record_dict["max"], record_dict["min"], record_dict["battery_level"], record_dict["system_version"], record_dict["repeats"], record_dict["warmup"], record_dict["imei"], ] else: logger.fatal( "Unsupported framework name:{}".format( self.config["framework_name"] ) ) exit(1) record_str = ",".join(map(str, record)) if True: logger.info(record_str) summary.append(record_str) if is_print_summary: summary_str = "\n".join(summary) logger.info("\n" + summary_str) return summary
def run_bench(self): logger.info("==== {} ====".format(self.run_bench.__name__)) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) benchmark_platform = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] bench_dict = dict() bench_case_idx = 0 for didx in range(len(device_serials)): device_serial_num = device_serials[didx] logger.debug("didx:{}, serial_num:{}".format( didx, device_serial_num) # noqa ) bench_dict[device_serial_num] = dict() for pidx in range(len(benchmark_platform)): platform = benchmark_platform[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial_num][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial_num][platform][model_name] = dict( ) # noqa for bidx in range(len(support_backend)): backend = support_backend[bidx] is_cpu = self.config["is_cpu_backend"] logger.debug("bidx:{}, backend:{}".format( bidx, backend)) # noqa bench_dict[device_serial_num][platform][model_name][ backend] = dict() # noqa for tidx in range( len(self.config["cpu_thread_num"] ) if is_cpu(backend) else 1 # noqa ): bench_case_idx += 1 logger.info("\n\nbench_case_idx(from 1):{}".format( bench_case_idx) # noqa ) cpu_thread_num = self.config["cpu_thread_num"][ tidx] # noqa bench_dict[device_serial_num][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial_num][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) exit(1) cmd_res = run_cmd( bench_cmd, wait_interval_sec=3 # noqa ) # noqa perf_dict = self.parse_benchmark(cmd_res) ################################# # summarize benchmark info ################################# bench_record = { "soc": device_dict[device_serial_num]["soc"], "product": device_dict[device_serial_num][ # noqa "product"], # noqa "serial_num": device_serial_num, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "std_dev": perf_dict["std_dev"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial_num]["bind_cpu_idx"], "cpu_max_freqs": device_dict[device_serial_num][ # noqa "cpu_max_freqs"], "battery_level": device_dict[device_serial_num][ # noqa "battery_level"], "system_version": device_dict[device_serial_num][ # noqa "system_version"], "cmd": bench_cmd, "imei": device_dict[device_serial_num]["imei"], } bench_dict[device_serial_num][platform][ model_name][ # noqa backend][ cpu_thread_num] = bench_record # noqa # bench_dict[model_name].append(bench_record) logger.info(bench_record) return bench_dict
def gen_bench_cmds(self): logger.info("==== {} ====".format(self.gen_bench_cmds.__name__)) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) benchmark_platform = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] bench_dict = dict() bench_case_idx = 0 for didx in range(len(device_serials)): device_serial_num = device_serials[didx] logger.debug("didx:{}, serial_num:{}".format( didx, device_serial_num) # noqa ) bench_dict[device_serial_num] = dict() bench_cmds = [] for pidx in range(len(benchmark_platform)): platform = benchmark_platform[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial_num][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial_num][platform][model_name] = dict( ) # noqa for bidx in range(len(support_backend)): backend = support_backend[bidx] is_cpu = self.config["is_cpu_backend"] logger.debug("bidx:{}, backend:{}".format( bidx, backend)) # noqa bench_dict[device_serial_num][platform][model_name][ backend] = dict() # noqa for tidx in range( len(self.config["cpu_thread_num"] ) if is_cpu(backend) else 1 # noqa ): bench_case_idx += 1 logger.info("\n\nbench_case_idx(from 1):{}".format( bench_case_idx) # noqa ) cpu_thread_num = self.config["cpu_thread_num"][ tidx] # noqa bench_dict[device_serial_num][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial_num][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) continue bench_cmds.append(bench_cmd) bench_dict[device_serial_num]["bench_cmds"] = bench_cmds return bench_dict