def remove(args): factory = PackageFactory() if re.match(r"^https?:\/\/", args.repository): package = factory.generate_from_url(args.repository) else: package = factory.generate_from_name(args.repository) name = package.get_name() scanners = config.load()['scanners'] if name in scanners: factory = ScannerFactory() package = factory.generate_from_package(package) config.delete('scanners', package.get_name()) signatures = config.load()['signatures'] if name in signatures: factory = SignatureFactory() package = factory.generate_from_package(package) config.delete('signatures', package.get_name()) cmd = f"rm -rf ./{package.get_path()}" _cmd.run_cmd(cmd) cmd = f"rm -r ./{package.parent_path}/{package.auther}" _cmd.run_cmd(cmd)
def install(args): factory = PackageFactory() if re.match(r'^https?:\/\/', args.repository): package = factory.generate_from_url(args.repository) else: package = factory.generate_from_name(args.repository) # clone repository has_faild = os.system(f'git clone {package.url} {package.name}') if has_faild: print("[err] Error occues around clone.") sys.exit(1) # check config file if os.path.isfile(f'{package.name}/signature.yml'): factory = SignatureFactory() opt_func = signature_opt elif os.path.isfile(f'{package.name}/scanner.yml'): factory = ScannerFactory() opt_func = scanner_opt else: print('[err] config file not found.') sys.exit(1) package = factory.generate_from_package(package) # move folder cmd = f'mkdir {package.parent_path}/{package.auther}' _cmd.run_cmd(cmd) has_failed = os.rename( f'{package.name}', f'{factory.parent_path}/{package.auther}/{package.name}') if has_failed: print("[err] Error occues around clone.") sys.exit(1) path = package.get_config_path() with open(path) as f: package.config = yaml.load(f) has_failed = opt_func(package) if has_faild: print('Abroad') return None return package
def prepare_engine(self): logger.info("==== {} ====".format(self.prepare_engine.__name__)) lookup_branch_name_cmd = "cd {}; git branch".format( self.config["framework_name"]) branch = run_cmd(lookup_branch_name_cmd)[0].replace("* ", "") self.config["framework_repo_branch"] = branch logger.info("framework_repo_branch:{}".format( self.config["framework_repo_branch"] # noqa ) # noqa ) lookup_commit_id_cmd = "cd {}; git rev-parse --short HEAD".format( self.config["framework_name"]) commit_id = run_cmd(lookup_commit_id_cmd)[0] self.config["framework_repo_commit_id"] = commit_id logger.info("framework_repo_commit_id:{}".format( self.config["framework_repo_commit_id"]))
def get_system_version(serial_num): lookup_sys_ver_cmd = "adb -s {} shell getprop ro.build.version.release".format( # noqa serial_num ) sys_ver = run_cmd(lookup_sys_ver_cmd)[0] logger.debug("system_version:{}".format(sys_ver)) return sys_ver
def get_product(serial_num): lookup_product_cmd = "adb -s {} shell getprop | grep 'ro.product.model'".format( # noqa serial_num # noqa ) # noqa product = run_cmd(lookup_product_cmd)[0] product = product.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa logger.debug(product) return product
def get_soc_code(serial_num): lookup_soc_code_cmd = ( "adb -s {} shell getprop | " # noqa "grep 'ro.board.platform'".format(serial_num) # noqa ) soc_code = run_cmd(lookup_soc_code_cmd)[0] soc_code = soc_code.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa logger.debug(soc_code) return soc_code
def get_cpu_max_freqs(serial_num): check_cpu_num_cmd = "adb -s {} shell cat /proc/cpuinfo | grep processor".format( # noqa serial_num ) cmd_res = run_cmd(check_cpu_num_cmd) cpu_num = len(cmd_res) check_cpu_max_freq_cmd_pattern = ( "adb -s {} shell cat " "/sys/devices/system/cpu/cpu{}/cpufreq/cpuinfo_max_freq" # noqa ) cmds = map( lambda cpu_idx: check_cpu_max_freq_cmd_pattern.format( # noqa serial_num, cpu_idx ), range(cpu_num), ) cmds = list(cmds) try: cmd_res_list = run_cmds(cmds) logger.info(cmd_res_list[cmds[0]]) cpu_max_freqs = map(lambda cmd_key: cmd_res_list[cmd_key], cmds) except IndexError: logger.warn( "cat: /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq:" # noqa " Permission denied" # noqa ) logger.warn("replacing scaling_max_freq with cpuinfo_max_freq") cmds = map(lambda c: c.replace("cpuinfo", "scaling"), cmds) cmd_res_list = run_cmds(cmds) # logger.warn(cmd_res_list[cmds[0]].strip()) cpu_max_freqs = map( lambda cmd_key: cmd_res_list[cmd_key].strip(), cmds # noqa ) # noqa cpu_max_freqs = list(cpu_max_freqs) cpu_max_freqs = cpu_max_freqs[:cpu_num] # get str from list cpu_max_freqs = list(map(lambda l: l[0], cpu_max_freqs)) logger.debug( "cpu_max_freqs:{}, cpu_num:{}".format(cpu_max_freqs, cpu_num) # noqa ) # noqa is_valid_freq = ( lambda freq_str: True if "No such file or director" not in freq_str else False # noqa ) cpu_max_freqs_ghz = map( lambda freq: float(freq) / 1e6 if is_valid_freq(freq) else None, # noqa cpu_max_freqs, # noqa ) logger.debug(cpu_max_freqs_ghz) cpu_max_freqs_ghz = list(cpu_max_freqs_ghz) return cpu_max_freqs_ghz
def generate_from_directory(self, auther, name): if os.path.isdir(f'{self.parent_path}/{auther}/{name}'): cmd = f'''cd {self.parent_path}/{auther}/{name}/ \\ && git config --get remote.origin.url ''' url = _cmd.run_cmd(cmd, subprocess=True, output=False) else: url = f'{_setting.url}{auther}/{name}' return self.generate(url=url, name=name, auther=auther)
def get_imei(serial_num): lookup_imei_cmd = "adb -s {} shell service call iphonesubinfo 1".format( # noqa serial_num) imei_list = run_cmd(lookup_imei_cmd) imei_list = list(filter(lambda l: "." in l, imei_list)) assert 0 < len(imei_list) imei_list = list( map(lambda l: pattern_match(l, "'", "'", False), imei_list)) # noqa logger.debug("imei_list:{}".format(imei_list)) imei_list = map(lambda l: l.replace(".", ""), imei_list) imei = "".join(imei_list) imei = imei.replace(" ", "") logger.debug("imei:{}".format(imei)) return imei
def get_adb_devices(is_print_status=False): device_dict = dict() adb_device_cmd = "adb devices" res = run_cmd(adb_device_cmd) devices = res[1:] devices = list(map(lambda devi: devi.split("\t"), devices)) logger.info(devices) for didx in range(len(devices)): dev = devices[didx] if len(dev) == 2: # sublist: (serial_num, status) serial_num = dev[0] status = dev[1].strip() device_dict[serial_num] = status if is_print_status: logger.info("device_idx:{}, serial_num:{}, status:{}".format( didx, serial_num, status)) return device_dict
def run_bench_for_test(config, serial, thread_idx): bench_dict = dict() bench_dict[serial] = dict() logger.info("start thread_idx: {} at {}".format(thread_idx, ctime())) logger.info("do nothing") logger.info("end thread_idx: {} at {}".format(thread_idx, ctime())) models = config["model_names"] platforms = config["benchmark_platform"] backends = config["support_backend"] logger.info("models:{}".format(models)) logger.info("platforms:{}".format(platforms)) logger.info("backends:{}".format(backends)) for pidx in range(len(platforms)): platform = platforms[pidx] bench_dict[serial][platform] = dict() for midx in range(len(models)): model = models[midx] bench_dict[serial][platform][model] = dict() for bidx in range(len(backends)): backend = backends[bidx] bench_dict[serial][platform][model][backend] = dict() threads = [1, 2, 4] if backend == "ARM" else [1] for tidx in range(len(threads)): thread = threads[tidx] bench_dict[serial][platform][model][backend][thread] = [] # create cmd concats = map(str, [platform, model, backend, thread]) concat = "-".join(concats) cmd = "adb -s {} shell echo '{}'".format(serial, concat) from utils.cmd import run_cmd bench_record = run_cmd(cmd) bench_dict[serial][platform][model][backend][ thread] = bench_record # noqa logger.info(bench_record) return bench_dict
def run_bench_for_single_thread_func( self, device_serial, device_idx=-1, thread_idx=-1, thread_num=-1, framework_name="", ): logger.info( # noqa "==== {}, thread_idx(from0):{}/{} ====".format( self.run_bench_for_single_thread_func.__name__, thread_idx, thread_num, # noqa )) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] cur_device_dict = device_dict[device_serial] logger.info("cur_device_dict:{}".format(cur_device_dict)) model_dict = self.config["model_dict"] model_names = list(model_dict.keys()) platforms = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] # note(ysh329): this bench_dict is for single thread about device bench_dict = dict() bench_dict[device_serial] = dict() cpu_backend_num = list( map(self.config["is_cpu_backend"], support_backend)).count(True) bench_case_num = ( len(platforms) * len(model_names) * sum([ len(self.config["cpu_thread_num"]) * cpu_backend_num + len(set(support_backend) - set(["ARM", "ARM_XNNPACK"])) # noqa ])) logger.info("len(platform):{}".format(len(platforms))) logger.info("len(model_names):{}".format(len(model_names))) logger.info( 'len(self.config["cpu_thread_num"]) * cpu_backend_num:{}'.format( len(self.config["cpu_thread_num"]) * cpu_backend_num)) logger.info("support_backend:{}".format(support_backend)) logger.info( 'len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")):{}'. format( # noqa len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")))) logger.info("bench_case_num:{}".format(bench_case_num)) logger.info( 'len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0:{}' .format( # noqa len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0 # noqa )) logger.info('len(set(support_backend) - set("ARM")):{}'.format( len(set(support_backend) - set("ARM")))) bench_case_idx = 0 # platform: armv7/armv8/... for pidx in range(len(platforms)): platform = platforms[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) # model: mobilenetv1/v2/... for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial][platform][model_name] = dict( ) # noqa # backend: cpu/gpu/xpu/... for bidx in range(len(support_backend)): backend = support_backend[bidx] logger.debug("bidx:{}, backend:{}".format(bidx, backend)) # noqa bench_dict[device_serial][platform][model_name][ backend] = dict() # noqa # thread: 1/2/4/... for tidx in range( len(self.config["cpu_thread_num"]) if self. config["is_cpu_backend"](backend) else 1 # noqa ): bench_case_idx += 1 logger.info( "\n\nframework_name:{}, device_idx(from1):{}/{}, bench_case_idx(from1):{}/{}," # noqa " enable_multi_threads:{}, thread_idx(from0):{}/{}" .format( # noqa # noqa self.engine_name(), # noqa device_idx + 1, len(device_dict), # noqa bench_case_idx, bench_case_num, # noqa self.config["enable_multi_threads"], # noqa thread_idx, thread_num, # noqa ) # noqa ) cpu_thread_num = ( self.config["cpu_thread_num"][tidx] if self.config["is_cpu_backend"](backend) else 1 ) # noqa bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench case start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) elif self.config["framework_name"] == "tflite": # {device_benchmark_bin} --graph={model_dir} # --output_prefix={model_name} # --num_runs={repeats} --warmup_runs={warmup} # --num_threads={thread_num} {backend} bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "model_name": model_name, "repeats": self.config["repeats"](backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "backend": self.config["support_backend_cmd_id"] ( # noqa backend), # power_mode(TODO): no bind deafault? need explore deeply # noqa }) print(bench_cmd) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) exit(1) ################################# # run benchmark ################################# run_times_sum = ( self.config["repeats"](backend) + self.config["warmup"] # noqa ) # noqa max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER * run_times_sum) # noqa cmd_res = run_cmd( bench_cmd, wait_interval_sec=3, max_timeout_sec=max_wait_sec, ) # noqa perf_dict = self.parse_benchmark(cmd_res) ################################# # summarize benchmark info ################################# bench_record = { "soc_code": device_dict[device_serial]["soc_code"], ## noqa "product": device_dict[device_serial][ # noqa "product"], # noqa "serial_num": device_serial, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "std_dev": perf_dict["std_dev"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial]["bind_cpu_idx"], # noqa "cpu_max_freqs": device_dict[device_serial][ # noqa "cpu_max_freqs"], "battery_level": device_dict[device_serial][ # noqa "battery_level"], "system_version": device_dict[device_serial][ # noqa "system_version"], "cmd": bench_cmd, "imei": device_dict[device_serial]["imei"], } bench_dict[device_serial][platform][ model_name][ # noqa backend][cpu_thread_num] = bench_record # noqa logger.info(bench_record) return bench_dict
def run_bench(self): logger.info("==== {} ====".format(self.run_bench.__name__)) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) benchmark_platform = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] bench_dict = dict() bench_case_idx = 0 for didx in range(len(device_serials)): device_serial_num = device_serials[didx] logger.debug( "didx:{}, serial_num:{}".format(didx, device_serial_num) # noqa ) for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join( [device_work_dir, os.path.basename(model_dict[model_name])] # noqa ) logger.debug( "midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir ) ) bench_dict[model_name] = [] for pidx in range(len(benchmark_platform)): platform = benchmark_platform[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join( [ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"] ), # noqa ] ) logger.debug("pidx:{}, platform:{}".format(pidx, platform)) for bidx in range(len(support_backend)): backend = support_backend[bidx] is_cpu = self.config["is_cpu_backend"] logger.debug( "bidx:{}, backend:{}".format(bidx, backend) ) # noqa for tidx in range( len(self.config["cpu_thread_num"]) if is_cpu(backend) else 1 # noqa ): bench_case_idx += 1 logger.info( "\n\nbench_case_idx(from 1):{}".format( bench_case_idx ) # noqa ) cpu_thread_num = self.config["cpu_thread_num"][tidx] # noqa if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config[ "framework_name" ], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"]( backend ), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[ device_serial_num ][ # noqa "bind_cpu_idx" ], } ) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend ), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config[ "power_mode_id" ], # noqa "gpu_device": backend, } ) cmd_handle = run_cmd( bench_cmd, wait_interval_sec=3 # noqa ) # noqa perf_dict = self.parse_benchmark(cmd_handle) # summarize benchmark info bench_record = { "soc": device_dict[device_serial_num]["soc"], "product": device_dict[device_serial_num][ # noqa "product" ], # noqa "serial_num": device_serial_num, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial_num][ "bind_cpu_idx" ], "cpu_max_freqs": device_dict[device_serial_num][ # noqa "cpu_max_freqs" ], "battery_level": device_dict[device_serial_num][ # noqa "battery_level" ], "system_version": device_dict[ device_serial_num ][ # noqa "system_version" ], "cmd": bench_cmd, "imei": device_dict[device_serial_num]["imei"], } bench_dict[model_name].append(bench_record) logger.info(bench_record) return bench_dict
def prepare_devices(self): logger.info("==== {} ====".format(self.prepare_devices.__name__)) device_status_dict = get_adb_devices(True) serial_num_list = list(device_status_dict.keys()) logger.debug(serial_num_list) device_dict = dict() for sidx in range(len(serial_num_list)): device_serial_num = serial_num_list[sidx] device_status = device_status_dict[device_serial_num] if device_status != "device": logger.info( "device {} status is {}, skipped".format( device_serial_num, device_status ) ) continue device_dict[device_serial_num] = dict() device_dict[device_serial_num]["status"] = device_status device_dict[device_serial_num]["cpu_max_freqs"] = get_cpu_max_freqs( # noqa device_serial_num ) cpu_max_freqs = get_cpu_max_freqs(device_serial_num) cpu_valid_freqs = list( filter(lambda freq: freq is not None, cpu_max_freqs) ) # noqa big_cores_idx = get_target_freq_idx( max(cpu_valid_freqs), device_serial_num, cpu_max_freqs ) big_cores_idx_str = ",".join(big_cores_idx) little_cores_idx = get_target_freq_idx( min(cpu_valid_freqs), device_serial_num, cpu_max_freqs ) little_cores_idx_str = ",".join(little_cores_idx) device_dict[device_serial_num]["big_cores_idx"] = big_cores_idx_str device_dict[device_serial_num][ "little_cores_idx" ] = little_cores_idx_str # noqa if self.config["power_mode"] == "big_cores": device_dict[device_serial_num][ "bind_cpu_idx" ] = big_cores_idx_str # noqa elif self.config["power_mode"] == "little_cores": device_dict[device_serial_num][ "bind_cpu_idx" ] = little_cores_idx_str # noqa elif self.config["power_mode"] == "no_bind": device_dict[device_serial_num]["bind_cpu_idx"] = ",".join( map(str, range(len(cpu_max_freqs))) ) else: logger.info( "Unsupported power_mode:{}".format( self.config["power_mode"] ) # noqa ) exit(1) # battery level device_dict[device_serial_num]["battery_level"] = get_battery_level( # noqa device_serial_num ) # system version device_dict[device_serial_num][ "system_version" ] = get_system_version( # noqa device_serial_num ) # imie device_dict[device_serial_num]["imei"] = get_imei(device_serial_num) # noqa # ro.board.platform, ro.board.chiptype, ro.board.hardware device_soc_cmd = ( "adb -s {} shell getprop |" " grep 'ro.board.platform'".format(device_serial_num) ) cmd_handls = run_cmds([device_soc_cmd]) soc = cmd_handls[device_soc_cmd][0] soc = soc.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa device_dict[device_serial_num]["soc"] = soc logger.debug(soc) # product device_product_cmd = ( "adb -s {} shell getprop | " "grep 'ro.product.model'".format(device_serial_num) ) cmd_handle = run_cmd(device_product_cmd) product = cmd_handle[0] product = ( product.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa ) # noqa device_dict[device_serial_num]["product"] = product logger.debug(product) logger.debug(device_dict) logger.info("len(device_dict):{}".format(len(device_dict))) return device_dict
def run_bench(self): logger.info("==== {} ====".format(self.run_bench.__name__)) device_work_dir = self.config["device_work_dir"] device_dict = self.config["device_dict"] model_dict = self.config["model_dict"] device_serials = list(device_dict.keys()) model_names = list(model_dict.keys()) benchmark_platform = self.config["benchmark_platform"] support_backend = self.config["support_backend"] bench_cmd_pattern = self.config["bench_cmd_pattern"] bench_dict = dict() bench_case_idx = 0 for didx in range(len(device_serials)): device_serial_num = device_serials[didx] logger.debug("didx:{}, serial_num:{}".format( didx, device_serial_num) # noqa ) bench_dict[device_serial_num] = dict() for pidx in range(len(benchmark_platform)): platform = benchmark_platform[pidx] device_work_dir_platform = device_work_dir + "/" + platform # noqa device_benchmark_bin = "/".join([ device_work_dir_platform, os.path.basename( self.config[platform]["benchmark_bin"]), # noqa ]) bench_dict[device_serial_num][platform] = dict() logger.debug("pidx:{}, platform:{}".format(pidx, platform)) for midx in range(len(model_names)): model_name = model_names[midx] model_dir = "/".join([ device_work_dir, os.path.basename(model_dict[model_name]), ] # noqa ) logger.debug("midx:{}, model_name:{}, model_dir:{}".format( midx, model_name, model_dir)) bench_dict[device_serial_num][platform][model_name] = dict( ) # noqa for bidx in range(len(support_backend)): backend = support_backend[bidx] is_cpu = self.config["is_cpu_backend"] logger.debug("bidx:{}, backend:{}".format( bidx, backend)) # noqa bench_dict[device_serial_num][platform][model_name][ backend] = dict() # noqa for tidx in range( len(self.config["cpu_thread_num"] ) if is_cpu(backend) else 1 # noqa ): bench_case_idx += 1 logger.info("\n\nbench_case_idx(from 1):{}".format( bench_case_idx) # noqa ) cpu_thread_num = self.config["cpu_thread_num"][ tidx] # noqa bench_dict[device_serial_num][platform][ model_name][ # noqa backend][cpu_thread_num] = dict() # noqa ####################### # bench start ####################### if self.config["framework_name"] == "tnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_type": self.config["framework_name"], # noqa "model_dir": model_dir, "backend": backend, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "bind_cpu_idx": device_dict[device_serial_num][ # noqa "bind_cpu_idx"], }) elif self.config["framework_name"] == "ncnn": bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "power_mode": self.config["power_mode_id"], # noqa "gpu_device": backend, }) elif self.config["framework_name"] == "mnn": # '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa # '{forwardtype} {thread_num} {precision}"' bench_cmd = bench_cmd_pattern.format( **{ "serial_num": device_serial_num, "device_work_dir": device_work_dir_platform, # noqa "device_benchmark_bin": device_benchmark_bin, # noqa "model_dir": model_dir, "repeats": self.config["repeats"]( backend), # noqa "warmup": self.config["warmup"], "thread_num": cpu_thread_num, "forwardtype": backend, # power_mode: big_core default }) else: logger.fatal("Unsupported framework {}".format( self.config["framework_name"])) exit(1) ################################# # run benchmark ################################# run_times_sum = ( self.config["repeats"](backend) + self.config["warmup"] # noqa ) # noqa max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER * run_times_sum) # noqa cmd_res = run_cmd( bench_cmd, wait_interval_sec=3, max_timeout_sec=max_wait_sec, ) # noqa perf_dict = self.parse_benchmark(cmd_res) ################################# # summarize benchmark info ################################# bench_record = { "soc_code": device_dict[device_serial_num] ["soc_code"], ## noqa "product": device_dict[device_serial_num][ # noqa "product"], # noqa "serial_num": device_serial_num, "platform": platform, "model_name": model_name, "repeats": self.config["repeats"](backend), "warmup": self.config["warmup"], "avg": perf_dict["avg"], "max": perf_dict["max"], "min": perf_dict["min"], "std_dev": perf_dict["std_dev"], "backend": backend, "cpu_thread_num": cpu_thread_num, "power_mode": self.config["power_mode"], "bind_cpu_idx": device_dict[device_serial_num]["bind_cpu_idx"], "cpu_max_freqs": device_dict[device_serial_num][ # noqa "cpu_max_freqs"], "battery_level": device_dict[device_serial_num][ # noqa "battery_level"], "system_version": device_dict[device_serial_num][ # noqa "system_version"], "cmd": bench_cmd, "imei": device_dict[device_serial_num]["imei"], } bench_dict[device_serial_num][platform][ model_name][ # noqa backend][ cpu_thread_num] = bench_record # noqa # bench_dict[model_name].append(bench_record) logger.info(bench_record) return bench_dict