def write_list_to_file(self, bench_list, out_file_dir=None, suffix=".bench.csv"): # noqa framework_name = self.config["framework_name"] if out_file_dir is None: time_stamp_human = time.strftime( "%Y%m%d-%H%M%S", time.localtime() # noqa ) # noqa benchmark_platform = "".join(self.config["benchmark_platform"]) framework_repo_branch = self.config["framework_repo_branch"] framework_repo_commit_id = self.config["framework_repo_commit_id"] model_version = str(1) # noqa work_dir = self.config["work_dir"] # noqa out_file_dir = "-".join([ framework_name, framework_repo_branch, framework_repo_commit_id, benchmark_platform, time_stamp_human, ]) out_file_dir += suffix logger.info("bench for {} 's out_file_dir:{}".format( # noqa framework_name, out_file_dir) # noqa ) bench_str = "\n".join(bench_list) with open(out_file_dir, "w") as f: f.writelines(bench_str) logger.info("write {} benchmark result to {}".format( # noqa framework_name, out_file_dir) # noqa )
def test_tflite_engine(self): from core.global_config import create_config framework_name = "tflite" config_dict = create_config(framework_name) config_dict["work_dir"] = os.getcwd() + "/../tflite" tflite = Engine(config_dict) tflite.set_config("benchmark_platform", ["android-armv7"]) # noqa tflite.set_config( "support_backend", ["ARM", "ARM_XNNPACK", "GPU_CL_GL", "DSP_HEXAGON"], # noqa ) # ARM,ARM_XNNPACK, GPU_CL_GL, DSP_HEXAGON tflite.set_config("cpu_thread_num", [1, 2, 4]) # [1, 2, 4] model_dict = tflite.prepare_models() device_dict = tflite.prepare_devices() if len(device_dict) == 0: logger.error("no device found") return 0 config_dict = tflite.set_config("model_dict", model_dict) config_dict = tflite.set_config("device_dict", device_dict) tflite.prepare_models_for_devices() tflite.prepare_benchmark_assets_for_devices() bench_dict = tflite.run_bench() summary_list = tflite.generate_benchmark_summary(bench_dict) summary_str = "\n".join(summary_list) logger.info("summary_str:\n{}".format(summary_str)) tflite.write_list_to_file(summary_list) return 0
def prepare_models_for_devices(self): logger.info( "==== {} ====".format(self.prepare_models_for_devices.__name__) ) # noqa device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) cmds = list() for didx in range(len(device_serials)): device_serial = device_serials[didx] mkdir_cmd = "adb -s {} shell mkdir -p {}".format( device_serial, device_work_dir ) cmds.append(mkdir_cmd) for midx in range(len(model_names)): model_name = model_names[midx] model_proto = model_dict[model_name] model_param = model_proto.replace("tnnproto", "tnnmodel") push_proto_cmd = "adb -s {} push {} {}".format( device_serial, model_proto, device_work_dir ) push_param_cmd = "adb -s {} push {} {}".format( device_serial, model_param, device_work_dir ) cmds.extend([push_proto_cmd, push_param_cmd]) logger.debug([push_proto_cmd, push_param_cmd]) run_cmds(cmds) return 0
def test_run_cmd(self): cmds = ["ls", "pwd", "adb devices"] for cidx in range(len(cmds)): cmd = cmds[cidx] h = run_cmd(cmd) logger.info("cmd_idx:{}, cmd:{}".format(cidx, cmd)) logger.info(h)
def test_mnn_engine(self): from core.global_config import create_config framework_name = "mnn" config_dict = create_config(framework_name) config_dict["work_dir"] = os.getcwd() + "/../mnn" mnn = Engine(config_dict) mnn.set_config("benchmark_platform", ["android-armv8", "android-armv7"]) # noqa mnn.set_config( "support_backend", ["3", "6", "7"]) # 0->CPU,1->Metal,3->OpenCL,6->OpenGL,7->Vulkan mnn.set_config("cpu_thread_num", [1, 2, 4]) # [1, 2, 4] mnn.config["warmup"] = 2 model_dict = mnn.prepare_models() device_dict = mnn.prepare_devices() if len(device_dict) == 0: logger.error("no device found") return 0 config_dict = mnn.set_config("model_dict", model_dict) config_dict = mnn.set_config("device_dict", device_dict) mnn.prepare_models_for_devices() mnn.prepare_benchmark_assets_for_devices() bench_dict = mnn.run_bench() summary_list = mnn.generate_benchmark_summary(bench_dict) summary_str = "\n".join(summary_list) logger.info("summary_str:\n{}".format(summary_str)) mnn.write_list_to_file(summary_list) return 0
def test_run_cmds(self): cmds = ["ls", "ls -l", "adb devices"] cmd_handls = run_cmds(cmds) for cidx in range(len(cmds)): cmd = cmds[cidx] h = cmd_handls[cmd] logger.info("cmd_idx:{}, cmd:{}".format(cidx, cmd)) logger.info(h)
def run_bench(self): logger.info("==== {} ====".format(self.run_bench.__name__)) logger.info("enable_multi_threads:{}".format( # noqa self.config["enable_multi_threads"])) if self.config["enable_multi_threads"]: return self.run_bench_multi_threads() else: # use single thread return self.run_bench_single_thread()
def parse_benchmark(self, cmd_res): logger.info("==== {} ====".format(self.parse_benchmark.__name__)) output_lines = cmd_res logger.debug(output_lines) framework_name = self.config["framework_name"] benchmark = dict() if framework_name == "tnn" or framework_name == "mnn": if framework_name == "tnn": bench_res_keyword = "time cost" elif framework_name == "mnn": bench_res_keyword = "max =" output_lines = filter( # noqa lambda line: bench_res_keyword in line, output_lines) output_lines = list(output_lines) assert len(output_lines) == 1 line = output_lines[0].split() logger.debug(line) line = "".join(line) logger.debug(line) benchmark["min"] = pattern_match(line, "min=", "ms", False) benchmark["max"] = pattern_match(line, "max=", "ms", False) benchmark["avg"] = pattern_match(line, "avg=", "ms", False) benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False) elif framework_name == "ncnn": is_no_vulkan = list( filter(lambda line: "no vulkan device" in line, output_lines)) is_no_vulkan = len(is_no_vulkan) > 0 if is_no_vulkan: benchmark["min"] = 0.0 benchmark["max"] = 0.0 benchmark["avg"] = 0.0 benchmark["std_dev"] = 0.0 else: output_lines = filter(lambda line: "min = " in line, output_lines) # noqa output_lines = list(output_lines) logger.debug("output_lines:\n{}".format(output_lines)) assert len(output_lines) == 1 line = output_lines[0] line = line.split() logger.debug(line) line = "".join(line) + "END" benchmark["min"] = pattern_match(line, "min=", "max", False) benchmark["max"] = pattern_match(line, "max=", "avg", False) benchmark["avg"] = pattern_match(line, "avg=", "std_dev", False) # noqa benchmark["std_dev"] = pattern_match(line, "std_dev=", "END", False) # noqa else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) logger.info("benchmark:{}".format(benchmark)) assert len(benchmark) != 0 return benchmark
def setUp(self): import sys print(sys.argv) logger.info("{} {}".format( self.__class__.__name__, sys._getframe().f_code.co_name # noqa ) # noqa )
def run_bench_v2(self, bench_dict): logger.info("==== {} ====".format(self.run_bench_v2.__name__)) device_serials = list(bench_dict.keys()) for didx in range(len(device_serials)): ser = device_serials[didx] cmds = bench_dict[ser] logger.info("didx: {}, ser: {}, cmds: {}".format( # noqa didx, ser, len(cmds)) # noqa )
def get_cpu_max_freqs(serial_num): check_cpu_num_cmd = "adb -s {} shell cat /proc/cpuinfo | grep processor".format( # noqa serial_num ) cmd_res = run_cmd(check_cpu_num_cmd) cpu_num = len(cmd_res) check_cpu_max_freq_cmd_pattern = ( "adb -s {} shell cat " "/sys/devices/system/cpu/cpu{}/cpufreq/cpuinfo_max_freq" # noqa ) cmds = map( lambda cpu_idx: check_cpu_max_freq_cmd_pattern.format( # noqa serial_num, cpu_idx ), range(cpu_num), ) cmds = list(cmds) try: cmd_res_list = run_cmds(cmds) logger.info(cmd_res_list[cmds[0]]) cpu_max_freqs = map(lambda cmd_key: cmd_res_list[cmd_key], cmds) except IndexError: logger.warn( "cat: /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq:" # noqa " Permission denied" # noqa ) logger.warn("replacing scaling_max_freq with cpuinfo_max_freq") cmds = map(lambda c: c.replace("cpuinfo", "scaling"), cmds) cmd_res_list = run_cmds(cmds) # logger.warn(cmd_res_list[cmds[0]].strip()) cpu_max_freqs = map( lambda cmd_key: cmd_res_list[cmd_key].strip(), cmds # noqa ) # noqa cpu_max_freqs = list(cpu_max_freqs) cpu_max_freqs = cpu_max_freqs[:cpu_num] # get str from list cpu_max_freqs = list(map(lambda l: l[0], cpu_max_freqs)) logger.debug( "cpu_max_freqs:{}, cpu_num:{}".format(cpu_max_freqs, cpu_num) # noqa ) # noqa is_valid_freq = ( lambda freq_str: True if "No such file or director" not in freq_str else False # noqa ) cpu_max_freqs_ghz = map( lambda freq: float(freq) / 1e6 if is_valid_freq(freq) else None, # noqa cpu_max_freqs, # noqa ) logger.debug(cpu_max_freqs_ghz) cpu_max_freqs_ghz = list(cpu_max_freqs_ghz) return cpu_max_freqs_ghz
def run_cmd(cmd, wait_interval_sec=5, max_timeout_sec=100): cmd_type = "CMD" logger.info("{}> {}".format(cmd_type, cmd)) subp = subprocess.Popen( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", ) subp.wait(max_timeout_sec) subp_status = -1 duration_sec = 0 while 1: if subp.poll() == 0: logger.debug("{} finished".format(cmd_type)) subp_status = int(subp.poll()) break elif subp.poll() is None: logger.debug("{} duration += {} second".format( cmd_type, wait_interval_sec) # noqa ) # noqa time.sleep(wait_interval_sec) duration_sec += wait_interval_sec if duration_sec > max_timeout_sec: logger.error( "{} duration {} second timeout with max_timeout_sec {}". format( # noqa cmd_type, duration_sec, max_timeout_sec)) subp.kill() break else: subp_status = subp.poll() logger.fatal( "exited with abnormal subprocess status: {}".format( # noqa subp_status) # noqa ) if subp_status == 139: break else: exit(1) logger.debug("{} consume {} seconds to finish".format( cmd_type, duration_sec) # noqa ) # noqa cmd_res = None if subp_status == 0 or subp_status == 139: cmd_res = "".join(subp.communicate()) logger.debug("cmd_res:{}".format(subp.communicate())) cmd_res = cmd_res.split("\n") cmd_res = filter(lambda r: r != "", cmd_res) cmd_res = list(cmd_res) return cmd_res
def run_bench_multi_threads(self): logger.info("==== {} ====".format( self.run_bench_multi_threads.__name__) # noqa ) device_serials = list(self.config["device_dict"].keys()) device_threads = dict() for didx in range(len(device_serials)): thread_idx = didx ser = device_serials[didx] logger.info( "create device(thread)_idx(from1):{}/{}, serial_num:{}".format( didx + 1, len(device_serials), ser) # noqa ) device_threads[ser] = MyThread( func=self.run_bench_for_single_thread_func, func_args_tuple=( ser, # noqa didx, # noqa thread_idx, # noqa len(device_serials), # noqa self.config["framework_name"], ), # noqa device_serial=ser, thread_idx=thread_idx, thread_num=len(device_serials), framework_name=self.config["framework_name"], ) assert len(device_threads) == len(device_serials) for tidx in range(len(device_threads)): ser = device_serials[tidx] device_threads[ser].start() for tidx in range(len(device_threads)): ser = device_serials[tidx] device_threads[ser].join() bench_dict = dict() for tidx in range(len(device_threads)): ser = device_serials[tidx] res = device_threads[ser].get_result() if res is None: # TODO(ysh329): need watch out for failed device # when benchmark some framework logger.error("device {} result is None," " skipped and continue".format(ser) # noqa ) continue bench_dict[ser] = res[ser] return bench_dict
def test_pattern_match(self): text = "abcdefgh" contain_a_b_list = [False, True] ptn_a = "b" ptn_b = "g" for contain_a_b in contain_a_b_list: res = pattern_match(text, ptn_a, ptn_b, contain_a_b) logger.info( "text:{}, ptn_a:{}, ptn_b:{}, contain_a_b:{}".format( text, ptn_a, ptn_b, contain_a_b ) ) logger.info("{} res:{}".format(pattern_match.__name__, res))
def prepare_models_for_devices(self): logger.info("==== {} ====".format( self.prepare_models_for_devices.__name__)) # noqa device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) logger.debug(model_dict) cmds = list() for didx in range(len(device_serials)): device_serial = device_serials[didx] mkdir_cmd = "adb -s {} shell mkdir -p {}".format( device_serial, device_work_dir) cmds.append(mkdir_cmd) for midx in range(len(model_names)): model_name = model_names[midx] model_proto = model_dict[model_name] if (self.config["framework_name"] == "ncnn" or self.config["framework_name"] == "tnn"): model_param = model_proto.replace("tnnproto", "tnnmodel") elif (self.config["framework_name"] == "mnn" or self.config["framework_name"] == "tflite"): model_param = None else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) push_proto_cmd = "adb -s {} push {} {}".format( device_serial, model_proto, "/".join([device_work_dir, os.path.basename(model_proto)]), # noqa ) push_param_cmd = "adb -s {} push {} {}".format( device_serial, model_param, device_work_dir) push_param_cmd = ( "echo" if model_param is None else push_param_cmd) # noqa cmds.extend([push_proto_cmd, push_param_cmd]) run_cmds(cmds) return 0
def run_bench_single_thread(self): logger.info("==== {} ====".format( self.run_bench_single_thread.__name__) # noqa ) device_dict = self.config["device_dict"] device_serials = list(device_dict.keys()) bench_dict = dict() for didx in range(len(device_serials)): device_serial_num = device_serials[didx] logger.debug("didx:{}, serial_num:{}".format( didx, device_serial_num) # noqa ) bench_dict[ device_serial_num] = self.run_bench_for_single_thread_func( # noqa device_serial_num, device_idx=didx)[device_serial_num] return bench_dict
def get_adb_devices(is_print_status=False): device_dict = dict() adb_device_cmd = "adb devices" res = run_cmd(adb_device_cmd) devices = res[1:] devices = list(map(lambda devi: devi.split("\t"), devices)) logger.info(devices) for didx in range(len(devices)): dev = devices[didx] if len(dev) == 2: # sublist: (serial_num, status) serial_num = dev[0] status = dev[1].strip() device_dict[serial_num] = status if is_print_status: logger.info("device_idx:{}, serial_num:{}, status:{}".format( didx, serial_num, status)) return device_dict
def test_threads(self): import sys sys.path.append("..") from utils.device import get_adb_devices config = dict() config["benchmark_platform"] = ["android-armv7", "android-armv8"] config["model_names"] = ["caffe_mobilenetv1", "caffe_mobilenetv2"] config["support_backend"] = ["ARM"] device_serials = list(map(lambda t: t, get_adb_devices())) logger.info(device_serials) device_threads = dict() for didx in range(len(device_serials)): thread_idx = didx ser = device_serials[didx] logger.info("didx(from1):{}/{}".format(didx, len(device_serials))) device_threads[ser] = MyThread( func=run_bench_for_test, func_args_tuple=(config, ser, thread_idx), device_serial=ser, thread_idx=thread_idx, ) assert len(device_serials) == len(device_threads) for didx in range(len(device_threads)): ser = device_serials[didx] device_threads[ser].start() for didx in range(len(device_threads)): ser = device_serials[didx] device_threads[ser].join() bench_dict = dict() for didx in range(len(device_serials)): ser = device_serials[didx] t = device_threads[ser] t_bench_dict = t.get_result() bench_dict[ser] = t_bench_dict[ser] return bench_dict
def test_get_file_name(self): test_cases = [ { "text": "https://github.com/Tencent/ncnn.git", "contain_suffix": True, "expect": "ncnn.git", }, { "text": "https://github.com/Tencent/ncnn.git", "contain_suffix": False, "expect": "ncnn", }, { "text": "/Users/code/abc.def.gh", "contain_suffix": True, "expect": "abc.def.gh", }, { "text": "/Users/code/abc.def.gh", "contain_suffix": False, "expect": "abc.def", }, ] test_cases_num = len(test_cases) for tidx in range(test_cases_num): test_case = test_cases[tidx] logger.info( "tidx:{}/{}, text:{}, contain_suffix:{}, expect:{}".format( tidx + 1, test_cases_num, test_case["text"], test_case["contain_suffix"], test_case["expect"], ) ) res = get_file_name(test_case["text"], test_case["contain_suffix"]) self.assertEqual(res, test_case["expect"])
def test_ncnn_engine(self): from core.global_config import create_config framework_name = "ncnn" config_dict = create_config(framework_name) config_dict["work_dir"] = os.getcwd() + "/../ncnn" ncnn = Engine(config_dict) ncnn.set_config( "benchmark_platform", ["android-armv8", "android-armv7"] # noqa ) ncnn.set_config("support_backend", ["0"]) # -1: cpu, 0: gpu # noqa ncnn.set_config("cpu_thread_num", [1, 2, 4]) ncnn.config["repeats"] = ( lambda backend: 100 if backend == "VULKAN" else 1000 ) # noqa ncnn.config["warmup"] = 2 model_dict = ncnn.prepare_models() device_dict = ncnn.prepare_devices() if len(device_dict) == 0: logger.error("no device found") return 0 config_dict = ncnn.set_config("model_dict", model_dict) config_dict = ncnn.set_config("device_dict", device_dict) ncnn.prepare_models_for_devices() ncnn.prepare_benchmark_assets_for_devices() bench_dict = ncnn.run_bench() summary_list = ncnn.generate_benchmark_summary(bench_dict) ncnn.write_list_to_file(summary_list) summary_str = "\n".join(summary_list) logger.info("summary_str:\n{}".format(summary_str)) return 0
def prepare_engine(self): logger.info("==== {} ====".format(self.prepare_engine.__name__)) lookup_branch_name_cmd = "cd {}; git branch".format( self.config["framework_name"]) branch = run_cmd(lookup_branch_name_cmd)[0].replace("* ", "") self.config["framework_repo_branch"] = branch logger.info("framework_repo_branch:{}".format( self.config["framework_repo_branch"] # noqa ) # noqa ) lookup_commit_id_cmd = "cd {}; git rev-parse --short HEAD".format( self.config["framework_name"]) commit_id = run_cmd(lookup_commit_id_cmd)[0] self.config["framework_repo_commit_id"] = commit_id logger.info("framework_repo_commit_id:{}".format( self.config["framework_repo_commit_id"]))
def prepare_devices(self): logger.info("==== {} ====".format(self.prepare_devices.__name__)) device_status_dict = get_adb_devices(True) serial_num_list = list(device_status_dict.keys()) logger.debug(serial_num_list) device_dict = dict() for sidx in range(len(serial_num_list)): ser = serial_num_list[sidx] device_status = device_status_dict[ser] if device_status != "device": logger.info("device {} status is {}, skipped".format( # noqa ser, device_status) # noqa ) continue device_dict[ser] = dict() device_dict[ser]["status"] = device_status device_dict[ser]["cpu_max_freqs"] = get_cpu_max_freqs(ser) # noqa cpu_max_freqs = get_cpu_max_freqs(ser) cpu_valid_freqs = list( filter(lambda freq: freq is not None, cpu_max_freqs)) # noqa big_cores_idx = get_target_freq_idx(max(cpu_valid_freqs), ser, cpu_max_freqs) big_cores_idx_str = ",".join(big_cores_idx) little_cores_idx = get_target_freq_idx(min(cpu_valid_freqs), ser, cpu_max_freqs) little_cores_idx_str = ",".join(little_cores_idx) device_dict[ser]["big_cores_idx"] = big_cores_idx_str device_dict[ser]["little_cores_idx"] = little_cores_idx_str # noqa if self.config["power_mode"] == "big_cores": device_dict[ser]["bind_cpu_idx"] = big_cores_idx_str # noqa elif self.config["power_mode"] == "little_cores": device_dict[ser]["bind_cpu_idx"] = little_cores_idx_str # noqa elif self.config["power_mode"] == "no_bind": device_dict[ser]["bind_cpu_idx"] = ",".join( map(str, range(len(cpu_max_freqs)))) else: logger.info("Unsupported power_mode:{}".format( self.config["power_mode"]) # noqa ) exit(1) # battery level device_dict[ser]["battery_level"] = get_battery_level(ser) # noqa # system version device_dict[ser]["system_version"] = get_system_version( ser) # noqa # imie device_dict[ser]["imei"] = get_imei(ser) # noqa # ro.board.platform, ro.board.chiptype, ro.board.hardware device_dict[ser]["soc_code"] = get_soc_code(ser) # noqa # soc info device_dict[ser]["soc_info"] = get_soc_info_from_soc_code( device_dict[ser]["soc_code"]) # product device_dict[ser]["product"] = get_product(ser) # noqa logger.debug(device_dict) logger.info("len(device_dict):{}".format(len(device_dict))) return device_dict
def tearDown(self): logger.info("{} {}".format( self.__class__.__name__, sys._getframe().f_code.co_name # noqa ) # noqa )
def generate_benchmark_summary(self, bench_dict, is_print_summary=True): logger.info( "==== {} ====".format(self.generate_benchmark_summary.__name__) ) # noqa summary_header = [ "framework", "branch", "commit_id", "model_name", "platform", "soc", "product", "power_mode", "backend", "cpu_thread_num", "avg", "max", "min", "battery_level", "system_version", "repeats", "warmup", "imei", ] summary_header_str = ",".join(summary_header) summary = [summary_header_str] model_names = list(bench_dict.keys()) model_names.sort() model_num = len(model_names) for midx in range(model_num): model_name = model_names[midx] bench_records = bench_dict[model_name] logger.info("midx:{}/{},{}".format(midx + 1, model_num, model_name)) # noqa for ridx in range(len(bench_records)): record_dict = bench_records[ridx] logger.info(record_dict) if self.config["framework_name"] == "tnn": record = [ self.config["framework_name"], self.config["framework_repo_branch"], self.config["framework_repo_commit_id"], record_dict["model_name"], record_dict["platform"], record_dict["soc"], record_dict["product"], record_dict["power_mode"], record_dict["backend"], record_dict["cpu_thread_num"], record_dict["avg"], record_dict["max"], record_dict["min"], record_dict["battery_level"], record_dict["system_version"], record_dict["repeats"], record_dict["warmup"], record_dict["imei"], ] elif self.config["framework_name"] == "ncnn": backend_id_to_str_dict = self.config[ "backend_id_to_str_dict" ] # noqa record = [ self.config["framework_name"], self.config["framework_repo_branch"], self.config["framework_repo_commit_id"], record_dict["model_name"], record_dict["platform"], record_dict["soc"], record_dict["product"], record_dict["power_mode"], backend_id_to_str_dict[record_dict["backend"]], record_dict["cpu_thread_num"], record_dict["avg"], record_dict["max"], record_dict["min"], record_dict["battery_level"], record_dict["system_version"], record_dict["repeats"], record_dict["warmup"], record_dict["imei"], ] else: logger.fatal( "Unsupported framework name:{}".format( self.config["framework_name"] ) ) exit(1) record_str = ",".join(map(str, record)) if True: logger.info(record_str) summary.append(record_str) if is_print_summary: summary_str = "\n".join(summary) logger.info("\n" + summary_str) return summary
def generate_benchmark_summary(self, bench_dict, is_print_summary=True): logger.info("==== {} ====".format( self.generate_benchmark_summary.__name__)) # noqa summary_header = [ "framework", "branch", "commit_id", "model_name", "platform", "soc_code", "soc_name", "cpu", "gpu", "npu", "product", "power_mode", "backend", "cpu_thread_num", "avg", "max", "min", "std_dev", "battery_level", "system_version", "repeats", "warmup", "imei", ] summary_header_str = ",".join(summary_header) summary = [summary_header_str] # bench_dict[device_serial_num][platform][model_name][support_backend][cpu_thread_num] = bench_record # noqa device_serials = list(bench_dict.keys()) for didx in range(len(device_serials)): serial = device_serials[didx] soc_info_dict = self.config["device_dict"][serial]["soc_info"] platforms = list(bench_dict[serial].keys()) for pidx in range(len(platforms)): platform = platforms[pidx] model_names = list(bench_dict[serial][platform].keys()) for midx in range(len(model_names)): model_name = model_names[midx] backends = list(bench_dict[serial][platform] [model_name].keys()) # noqa for bidx in range(len(backends)): backend = backends[bidx] cpu_thread_nums = list( bench_dict[serial][platform][model_name] [backend].keys() # noqa ) # noqa for tidx in range(len(cpu_thread_nums)): cpu_thread_num = cpu_thread_nums[tidx] # noqa # get record record_dict = bench_dict[serial][platform][ model_name][ # noqa backend][cpu_thread_num] # noqa logger.debug("record_dict:{}".format(record_dict)) record = [ self.config["framework_name"], self.config["framework_repo_branch"], self.config["framework_repo_commit_id"], record_dict["model_name"], record_dict["platform"], record_dict["soc_code"], soc_info_dict["name"], soc_info_dict["cpu"], soc_info_dict["gpu"], soc_info_dict["npu/apu/xpu/dsp"], record_dict["product"], record_dict["power_mode"], self.config["support_backend_id"]( record_dict["backend"]), # noqa record_dict["cpu_thread_num"], record_dict["avg"], record_dict["max"], record_dict["min"], record_dict["std_dev"], record_dict["battery_level"], record_dict["system_version"], record_dict["repeats"], record_dict["warmup"], record_dict["imei"], ] record_str = ",".join(map(str, record)) if True: logger.info(record_str) summary.append(record_str) if is_print_summary: summary_str = "\n".join(summary) logger.info("\n" + summary_str) return summary
def parse_benchmark(self, cmd_res): logger.info("==== {} ====".format(self.parse_benchmark.__name__)) output_lines = cmd_res logger.debug(output_lines) framework_name = self.config["framework_name"] benchmark = dict() benchmark["min"] = 0.0 benchmark["max"] = 0.0 benchmark["avg"] = 0.0 benchmark["std_dev"] = 0.0 if cmd_res is None: return benchmark elif (framework_name == "tnn" or framework_name == "mnn" or framework_name == "tflite"): if framework_name == "tnn" or framework_name == "tflite": bench_res_keyword = "time cost" elif framework_name == "mnn": bench_res_keyword = "max =" output_lines_str = "".join(output_lines) output_lines = filter( # noqa lambda line: bench_res_keyword in line, output_lines) if (framework_name == "mnn" and "Floating point exception" in output_lines_str): return benchmark elif (framework_name == "tflite" and 'Error: dlopen failed: library "libhexagon_interface.so" not found' # noqa in output_lines_str): return benchmark output_lines = list(output_lines) assert len(output_lines) == 1 line = output_lines[0].split() logger.debug(line) line = "".join(line) logger.debug(line) benchmark["min"] = pattern_match(line, "min=", "ms", False) benchmark["max"] = pattern_match(line, "max=", "ms", False) benchmark["avg"] = pattern_match(line, "avg=", "ms", False) benchmark["std_dev"] = pattern_match(line, "std_dev=", "ms", False) elif framework_name == "ncnn": is_no_vulkan = list( filter( lambda line: "no vulkan device" in line or '"libvulkan.so" not found' in line, output_lines, )) is_no_vulkan = len(is_no_vulkan) > 0 if is_no_vulkan: return benchmark else: output_lines = filter(lambda line: "min = " in line, output_lines) # noqa output_lines = list(output_lines) logger.debug("output_lines:\n{}".format(output_lines)) assert len(output_lines) == 1 line = output_lines[0] line = line.split() logger.debug(line) line = "".join(line) + "END" benchmark["min"] = pattern_match(line, "min=", "max", False) benchmark["max"] = pattern_match(line, "max=", "avg", False) benchmark["avg"] = pattern_match(line, "avg=", "std_dev", False) # noqa benchmark["std_dev"] = pattern_match(line, "std_dev=", "END", False) # noqa else: logger.fatal("Unsupported framework {}".format( # noqa self.config["framework_name"] # noqa ) # noqa ) exit(1) logger.info("benchmark:{}".format(benchmark)) assert len(benchmark) != 0 return benchmark
def prepare_models(self): logger.info("==== {} ====".format(self.prepare_models.__name__)) cmds = list() framework_name = self.config["framework_name"] model_repo = self.config["model_repo"] model_type_keyword = self.config["model_type_keyword"] repo_name = get_file_name(model_repo, False) logger.info("repo_name:{}".format(repo_name)) logger.debug("os.path.exists:{}".format(os.path.exists(repo_name))) if os.path.exists(repo_name): clone_models_cmd = "ls {}".format(repo_name) else: clone_models_cmd = "git clone {}".format(model_repo) model_repo_version_cmd = ( 'cd ./{}-models/; git log --pretty=format:"SHA-1:%h date:%ad" ' '--date=format:"%y-%m-%d" -n1 #--shortstat -n1'.format( framework_name) # noqa ) model_repo_version_extra_cmd = ( "cd ./{}-models/; " 'git log --pretty=format:"SHA-1:%h - author:%an date:%ad ' 'note:%s" --date=format:"%y-%m-%d %H:%M:%S" -n1'.format( framework_name) # noqa ) model_repo_branch_cmd = "cd ./{}-models/; git branch | sed 's/\* //g'".format( # noqa framework_name) lookup_models_path_cmd = "realpath ./{}-models/*{}*".format( framework_name, model_type_keyword) cmds.extend([ clone_models_cmd, model_repo_version_cmd, model_repo_version_extra_cmd, model_repo_branch_cmd, lookup_models_path_cmd, ]) cmds_res = run_cmds(cmds) # TODO(ysh329): add model_repo_version to config # self.config["model_repo_branch"] = # cmds_res[model_repo_branch_cmd][0] # self.config["model_repo_version"] = # cmds_res[model_repo_version_cmd].readalines()[0] # self.config["model_repo_version_extra"] = # cmds_res[model_repo_version_extra_cmd].readalines()[0] models_dir = list( map(lambda path: path.strip(), cmds_res[lookup_models_path_cmd]) # noqa ) model_dict = dict() for midx in range(len(models_dir)): model_dir = models_dir[midx] logger.debug("{} {}".format(midx, model_dir)) file_type = model_dir.split(".")[-1] model_name = (model_dir.split("/")[-1].replace( "." + file_type, "").replace(file_type, "")) logger.debug("model_name:{}, file_type:{}".format( model_name, file_type)) # noqa model_dict[model_name] = model_dir logger.debug(models_dir) logger.debug(model_dict) return model_dict
def run_bench_for_single_thread_func( self, device_serial, device_idx=-1, thread_idx=-1, thread_num=-1, framework_name="", ): logger.info( # noqa "==== {}, thread_idx(from0):{}/{} ====".format( self.run_bench_for_single_thread_func.__name__, thread_idx, thread_num, # noqa )) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] cur_device_dict = device_dict[device_serial] logger.info("cur_device_dict:{}".format(cur_device_dict)) model_dict = self.config["model_dict"] model_names = list(model_dict.keys()) platforms = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] # note(ysh329): this bench_dict is for single thread about device bench_dict = dict() bench_dict[device_serial] = dict() cpu_backend_num = list( map(self.config["is_cpu_backend"], support_backend)).count(True) bench_case_num = ( len(platforms) * len(model_names) * sum([ len(self.config["cpu_thread_num"]) * cpu_backend_num + len(set(support_backend) - set(["ARM", "ARM_XNNPACK"])) # noqa ])) logger.info("len(platform):{}".format(len(platforms))) logger.info("len(model_names):{}".format(len(model_names))) logger.info( 'len(self.config["cpu_thread_num"]) * cpu_backend_num:{}'.format( len(self.config["cpu_thread_num"]) * cpu_backend_num)) logger.info("support_backend:{}".format(support_backend)) logger.info( 'len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")):{}'. format( # noqa len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")))) logger.info("bench_case_num:{}".format(bench_case_num)) logger.info( 'len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0:{}' .format( # noqa len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0 # noqa )) logger.info('len(set(support_backend) - set("ARM")):{}'.format( len(set(support_backend) - set("ARM")))) bench_case_idx = 0 # platform: armv7/armv8/... for pidx in range(len(platforms)): platform = platforms[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) # model: mobilenetv1/v2/... for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial][platform][model_name] = dict( ) # noqa # backend: cpu/gpu/xpu/... for bidx in range(len(support_backend)): backend = support_backend[bidx] logger.debug("bidx:{}, backend:{}".format(bidx, backend)) # noqa bench_dict[device_serial][platform][model_name][ backend] = dict() # noqa # thread: 1/2/4/... for tidx in range( len(self.config["cpu_thread_num"]) if self. config["is_cpu_backend"](backend) else 1 # noqa ): bench_case_idx += 1 logger.info( "\n\nframework_name:{}, device_idx(from1):{}/{}, bench_case_idx(from1):{}/{}," # noqa " enable_multi_threads:{}, thread_idx(from0):{}/{}" .format( # noqa # noqa self.engine_name(), # noqa device_idx + 1, len(device_dict), # noqa bench_case_idx, bench_case_num, # noqa self.config["enable_multi_threads"], # noqa thread_idx, thread_num, # noqa ) # noqa ) cpu_thread_num = ( self.config["cpu_thread_num"][tidx] if self.config["is_cpu_backend"](backend) else 1 ) # noqa bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench case start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) elif self.config["framework_name"] == "tflite": # {device_benchmark_bin} --graph={model_dir} # --output_prefix={model_name} # --num_runs={repeats} --warmup_runs={warmup} # --num_threads={thread_num} {backend} bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "model_name": model_name, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "backend": self.config["support_backend_cmd_id"] ( # noqa backend), # power_mode(TODO): no bind deafault? need explore deeply # noqa }) print(bench_cmd) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) exit(1) ################################# # run benchmark ################################# run_times_sum = ( self.config["repeats"](backend) + self.config["warmup"] # noqa ) # noqa max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER * run_times_sum) # noqa cmd_res = run_cmd( bench_cmd, wait_interval_sec=3, max_timeout_sec=max_wait_sec, ) # noqa perf_dict = self.parse_benchmark(cmd_res) ################################# # summarize benchmark info ################################# bench_record = { "soc_code": device_dict[device_serial]["soc_code"], ## noqa "product": device_dict[device_serial][ # noqa "product"], # noqa "serial_num": device_serial, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "std_dev": perf_dict["std_dev"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial]["bind_cpu_idx"], # noqa "cpu_max_freqs": device_dict[device_serial][ # noqa "cpu_max_freqs"], "battery_level": device_dict[device_serial][ # noqa "battery_level"], "system_version": device_dict[device_serial][ # noqa "system_version"], "cmd": bench_cmd, "imei": device_dict[device_serial]["imei"], } bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = bench_record # noqa logger.info(bench_record) return bench_dict
def test_main(self): device_dict = get_adb_devices(True) serials = device_dict.keys() serials = list(serials) for sidx in range(len(serials)): ser = serials[sidx] status = device_dict[ser] # for each cpu's max freq cpus_max_freq = get_cpu_max_freqs(ser) cpus_max_freq = list( filter(lambda freq: freq is not None, cpus_max_freq)) # noqa max_freq = max(cpus_max_freq) min_freq = min(cpus_max_freq) max_freq_cluster_idx = get_target_freq_idx(max_freq, ser, cpus_max_freq) # noqa min_freq_cluster_idx = get_target_freq_idx(min_freq, ser, cpus_max_freq) # noqa battery_level = get_battery_level(ser) system_version = get_system_version(ser) imei = get_imei(ser) soc_code = get_soc_code(ser) product = get_product(ser) soc_dict = get_soc_info_from_soc_code(soc_code) logger.info("sidx:{}, ser:{}, status:{}".format(sidx, ser, status)) logger.info("cpus_max_freq:{}".format(cpus_max_freq)) logger.info("max_freq:{}".format(max_freq)) logger.info("max_freq_cluster_idx:{}".format(max_freq_cluster_idx)) logger.info("min_freq:{}".format(min_freq)) logger.info("min_freq_cluster_idx:{}".format(min_freq_cluster_idx)) logger.info("battery_level:{}".format(battery_level)) logger.info("system_version:{}".format(system_version)) logger.info("imei:{}".format(imei)) logger.info("soc_code:{}".format(soc_code)) logger.info("product:{}".format(product)) logger.info("soc_dict:{}".format(soc_dict))
def prepare_benchmark_assets_for_devices(self): logger.info("==== {} ====".format( self.prepare_benchmark_assets_for_devices.__name__) # noqa ) benchmark_platform = self.config["benchmark_platform"] device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] device_serials = list(device_dict.keys()) cmds = list() for didx in range(len(device_serials)): device_serial = device_serials[didx] for pidx in range(len(benchmark_platform)): platform = benchmark_platform[pidx] device_work_dir_platform = device_work_dir + "/" + platform # benchmark assets benchmark_bin = self.config[platform]["benchmark_bin"] benchmark_lib = self.config[platform]["shared_lib"] benchmark_libs = (benchmark_lib if isinstance( benchmark_lib, list) else [benchmark_lib]) # create cmds rmdir_cmd = "adb -s {} shell rm -rf {}".format( device_serial, device_work_dir_platform) mkdir_cmd = "adb -s {} shell mkdir -p {}".format( device_serial, device_work_dir_platform) # lib benchmark_lib_device_paths = map( lambda lib: device_work_dir_platform + "/" + os.path. basename(lib), # noqa benchmark_libs, ) benchmark_lib_device_paths = list(benchmark_lib_device_paths) logger.debug("benchmark_lib_device_paths:{}".format( # noqa benchmark_lib_device_paths # noqa ) # noqa ) push_shared_lib_cmds = map( lambda lib, lib_device: "adb -s {} push {} {}". format( # noqa device_serial, lib, lib_device), # noqa benchmark_libs, benchmark_lib_device_paths, ) push_shared_lib_cmds = list(push_shared_lib_cmds) push_shared_lib_cmds = (["echo"] if benchmark_lib is None else push_shared_lib_cmds) logger.debug("push_shared_lib_cmds:{}".format( # noqa push_shared_lib_cmds) # noqa ) logger.debug("len(push_shared_lib_cmds):{}".format( # noqa len(push_shared_lib_cmds) # noqa ) # noqa ) # bin benchmark_bin_device_path = ( device_work_dir_platform + "/" + os.path.basename(benchmark_bin) # noqa ) push_benchmark_bin_cmd = "adb -s {} push {} {}".format( device_serial, benchmark_bin, benchmark_bin_device_path) chmod_x_bin_cmd = "adb -s {} shell chmod +x {}".format( device_serial, benchmark_bin_device_path) cmds.extend([rmdir_cmd, mkdir_cmd]) cmds.extend(push_shared_lib_cmds) cmds.extend([push_benchmark_bin_cmd, chmod_x_bin_cmd]) logger.info(cmds) run_cmds(cmds) return 0