def remove(args):
factory = PackageFactory()
if re.match(r"^https?:\/\/", args.repository):
package = factory.generate_from_url(args.repository)
else:
package = factory.generate_from_name(args.repository)
name = package.get_name()
scanners = config.load()['scanners']
if name in scanners:
factory = ScannerFactory()
package = factory.generate_from_package(package)
config.delete('scanners', package.get_name())
signatures = config.load()['signatures']
if name in signatures:
factory = SignatureFactory()
package = factory.generate_from_package(package)
config.delete('signatures', package.get_name())
cmd = f"rm -rf ./{package.get_path()}"
_cmd.run_cmd(cmd)
cmd = f"rm -r ./{package.parent_path}/{package.auther}"
_cmd.run_cmd(cmd)
def install(args):
factory = PackageFactory()
if re.match(r'^https?:\/\/', args.repository):
package = factory.generate_from_url(args.repository)
else:
package = factory.generate_from_name(args.repository)
# clone repository
has_faild = os.system(f'git clone {package.url} {package.name}')
if has_faild:
print("[err] Error occues around clone.")
sys.exit(1)
# check config file
if os.path.isfile(f'{package.name}/signature.yml'):
factory = SignatureFactory()
opt_func = signature_opt
elif os.path.isfile(f'{package.name}/scanner.yml'):
factory = ScannerFactory()
opt_func = scanner_opt
else:
print('[err] config file not found.')
sys.exit(1)
package = factory.generate_from_package(package)
# move folder
cmd = f'mkdir {package.parent_path}/{package.auther}'
_cmd.run_cmd(cmd)
has_failed = os.rename(
f'{package.name}',
f'{factory.parent_path}/{package.auther}/{package.name}')
if has_failed:
print("[err] Error occues around clone.")
sys.exit(1)
path = package.get_config_path()
with open(path) as f:
package.config = yaml.load(f)
has_failed = opt_func(package)
if has_faild:
print('Abroad')
return None
return package
def prepare_engine(self):
logger.info("==== {} ====".format(self.prepare_engine.__name__))
lookup_branch_name_cmd = "cd {}; git branch".format(
self.config["framework_name"])
branch = run_cmd(lookup_branch_name_cmd)[0].replace("* ", "")
self.config["framework_repo_branch"] = branch
logger.info("framework_repo_branch:{}".format(
self.config["framework_repo_branch"] # noqa
) # noqa
)
lookup_commit_id_cmd = "cd {}; git rev-parse --short HEAD".format(
self.config["framework_name"])
commit_id = run_cmd(lookup_commit_id_cmd)[0]
self.config["framework_repo_commit_id"] = commit_id
logger.info("framework_repo_commit_id:{}".format(
self.config["framework_repo_commit_id"]))
def get_system_version(serial_num):
lookup_sys_ver_cmd = "adb -s {} shell getprop ro.build.version.release".format( # noqa
serial_num
)
sys_ver = run_cmd(lookup_sys_ver_cmd)[0]
logger.debug("system_version:{}".format(sys_ver))
return sys_ver
def get_product(serial_num):
lookup_product_cmd = "adb -s {} shell getprop | grep 'ro.product.model'".format( # noqa
serial_num # noqa
) # noqa
product = run_cmd(lookup_product_cmd)[0]
product = product.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa
logger.debug(product)
return product
def get_soc_code(serial_num):
lookup_soc_code_cmd = (
"adb -s {} shell getprop | " # noqa
"grep 'ro.board.platform'".format(serial_num) # noqa
)
soc_code = run_cmd(lookup_soc_code_cmd)[0]
soc_code = soc_code.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa
logger.debug(soc_code)
return soc_code
def get_cpu_max_freqs(serial_num):
check_cpu_num_cmd = "adb -s {} shell cat /proc/cpuinfo | grep processor".format( # noqa
serial_num
)
cmd_res = run_cmd(check_cpu_num_cmd)
cpu_num = len(cmd_res)
check_cpu_max_freq_cmd_pattern = (
"adb -s {} shell cat "
"/sys/devices/system/cpu/cpu{}/cpufreq/cpuinfo_max_freq" # noqa
)
cmds = map(
lambda cpu_idx: check_cpu_max_freq_cmd_pattern.format( # noqa
serial_num, cpu_idx
),
range(cpu_num),
)
cmds = list(cmds)
try:
cmd_res_list = run_cmds(cmds)
logger.info(cmd_res_list[cmds[0]])
cpu_max_freqs = map(lambda cmd_key: cmd_res_list[cmd_key], cmds)
except IndexError:
logger.warn(
"cat: /sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq:" # noqa
" Permission denied" # noqa
)
logger.warn("replacing scaling_max_freq with cpuinfo_max_freq")
cmds = map(lambda c: c.replace("cpuinfo", "scaling"), cmds)
cmd_res_list = run_cmds(cmds)
# logger.warn(cmd_res_list[cmds[0]].strip())
cpu_max_freqs = map(
lambda cmd_key: cmd_res_list[cmd_key].strip(), cmds # noqa
) # noqa
cpu_max_freqs = list(cpu_max_freqs)
cpu_max_freqs = cpu_max_freqs[:cpu_num]
# get str from list
cpu_max_freqs = list(map(lambda l: l[0], cpu_max_freqs))
logger.debug(
"cpu_max_freqs:{}, cpu_num:{}".format(cpu_max_freqs, cpu_num) # noqa
) # noqa
is_valid_freq = (
lambda freq_str: True
if "No such file or director" not in freq_str
else False # noqa
)
cpu_max_freqs_ghz = map(
lambda freq: float(freq) / 1e6 if is_valid_freq(freq) else None, # noqa
cpu_max_freqs, # noqa
)
logger.debug(cpu_max_freqs_ghz)
cpu_max_freqs_ghz = list(cpu_max_freqs_ghz)
return cpu_max_freqs_ghz
def generate_from_directory(self, auther, name):
if os.path.isdir(f'{self.parent_path}/{auther}/{name}'):
cmd = f'''cd {self.parent_path}/{auther}/{name}/ \\
&& git config --get remote.origin.url
'''
url = _cmd.run_cmd(cmd, subprocess=True, output=False)
else:
url = f'{_setting.url}{auther}/{name}'
return self.generate(url=url, name=name, auther=auther)
def get_imei(serial_num):
lookup_imei_cmd = "adb -s {} shell service call iphonesubinfo 1".format( # noqa
serial_num)
imei_list = run_cmd(lookup_imei_cmd)
imei_list = list(filter(lambda l: "." in l, imei_list))
assert 0 < len(imei_list)
imei_list = list(
map(lambda l: pattern_match(l, "'", "'", False), imei_list)) # noqa
logger.debug("imei_list:{}".format(imei_list))
imei_list = map(lambda l: l.replace(".", ""), imei_list)
imei = "".join(imei_list)
imei = imei.replace(" ", "")
logger.debug("imei:{}".format(imei))
return imei
def get_adb_devices(is_print_status=False):
device_dict = dict()
adb_device_cmd = "adb devices"
res = run_cmd(adb_device_cmd)
devices = res[1:]
devices = list(map(lambda devi: devi.split("\t"), devices))
logger.info(devices)
for didx in range(len(devices)):
dev = devices[didx]
if len(dev) == 2: # sublist: (serial_num, status)
serial_num = dev[0]
status = dev[1].strip()
device_dict[serial_num] = status
if is_print_status:
logger.info("device_idx:{}, serial_num:{}, status:{}".format(
didx, serial_num, status))
return device_dict
def run_bench_for_test(config, serial, thread_idx):
bench_dict = dict()
bench_dict[serial] = dict()
logger.info("start thread_idx: {} at {}".format(thread_idx, ctime()))
logger.info("do nothing")
logger.info("end thread_idx: {} at {}".format(thread_idx, ctime()))
models = config["model_names"]
platforms = config["benchmark_platform"]
backends = config["support_backend"]
logger.info("models:{}".format(models))
logger.info("platforms:{}".format(platforms))
logger.info("backends:{}".format(backends))
for pidx in range(len(platforms)):
platform = platforms[pidx]
bench_dict[serial][platform] = dict()
for midx in range(len(models)):
model = models[midx]
bench_dict[serial][platform][model] = dict()
for bidx in range(len(backends)):
backend = backends[bidx]
bench_dict[serial][platform][model][backend] = dict()
threads = [1, 2, 4] if backend == "ARM" else [1]
for tidx in range(len(threads)):
thread = threads[tidx]
bench_dict[serial][platform][model][backend][thread] = []
# create cmd
concats = map(str, [platform, model, backend, thread])
concat = "-".join(concats)
cmd = "adb -s {} shell echo '{}'".format(serial, concat)
from utils.cmd import run_cmd
bench_record = run_cmd(cmd)
bench_dict[serial][platform][model][backend][
thread] = bench_record # noqa
logger.info(bench_record)
return bench_dict
def run_bench_for_single_thread_func(
self,
device_serial,
device_idx=-1,
thread_idx=-1,
thread_num=-1,
framework_name="",
):
logger.info( # noqa
"==== {}, thread_idx(from0):{}/{} ====".format(
self.run_bench_for_single_thread_func.__name__,
thread_idx,
thread_num, # noqa
))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
cur_device_dict = device_dict[device_serial]
logger.info("cur_device_dict:{}".format(cur_device_dict))
model_dict = self.config["model_dict"]
model_names = list(model_dict.keys())
platforms = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
# note(ysh329): this bench_dict is for single thread about device
bench_dict = dict()
bench_dict[device_serial] = dict()
cpu_backend_num = list(
map(self.config["is_cpu_backend"], support_backend)).count(True)
bench_case_num = (
len(platforms) * len(model_names) * sum([
len(self.config["cpu_thread_num"]) * cpu_backend_num +
len(set(support_backend) - set(["ARM", "ARM_XNNPACK"])) # noqa
]))
logger.info("len(platform):{}".format(len(platforms)))
logger.info("len(model_names):{}".format(len(model_names)))
logger.info(
'len(self.config["cpu_thread_num"]) * cpu_backend_num:{}'.format(
len(self.config["cpu_thread_num"]) * cpu_backend_num))
logger.info("support_backend:{}".format(support_backend))
logger.info(
'len(set(support_backend) - set("ARM") - set("ARM_XNNPACK")):{}'.
format( # noqa
len(set(support_backend) - set("ARM") - set("ARM_XNNPACK"))))
logger.info("bench_case_num:{}".format(bench_case_num))
logger.info(
'len(self.config["cpu_thread_num"]) if "CPU" in support_backend else 0:{}'
.format( # noqa
len(self.config["cpu_thread_num"])
if "CPU" in support_backend else 0 # noqa
))
logger.info('len(set(support_backend) - set("ARM")):{}'.format(
len(set(support_backend) - set("ARM"))))
bench_case_idx = 0
# platform: armv7/armv8/...
for pidx in range(len(platforms)):
platform = platforms[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join([
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]), # noqa
])
bench_dict[device_serial][platform] = dict()
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
# model: mobilenetv1/v2/...
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join([
device_work_dir,
os.path.basename(model_dict[model_name]),
] # noqa
)
logger.debug("midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir))
bench_dict[device_serial][platform][model_name] = dict(
) # noqa
# backend: cpu/gpu/xpu/...
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
logger.debug("bidx:{}, backend:{}".format(bidx,
backend)) # noqa
bench_dict[device_serial][platform][model_name][
backend] = dict() # noqa
# thread: 1/2/4/...
for tidx in range(
len(self.config["cpu_thread_num"]) if self.
config["is_cpu_backend"](backend) else 1 # noqa
):
bench_case_idx += 1
logger.info(
"\n\nframework_name:{}, device_idx(from1):{}/{}, bench_case_idx(from1):{}/{}," # noqa
" enable_multi_threads:{}, thread_idx(from0):{}/{}"
.format( # noqa
# noqa
self.engine_name(), # noqa
device_idx + 1,
len(device_dict), # noqa
bench_case_idx,
bench_case_num, # noqa
self.config["enable_multi_threads"], # noqa
thread_idx,
thread_num, # noqa
) # noqa
)
cpu_thread_num = (
self.config["cpu_thread_num"][tidx]
if self.config["is_cpu_backend"](backend) else 1
) # noqa
bench_dict[device_serial][platform][
model_name][ # noqa
backend][cpu_thread_num] = dict() # noqa
#######################
# bench case start
#######################
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_type":
self.config["framework_name"], # noqa
"model_dir":
model_dir,
"backend":
backend,
"repeats":
self.config["repeats"](backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"bind_cpu_idx":
device_dict[device_serial][ # noqa
"bind_cpu_idx"],
})
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial,
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode":
self.config["power_mode_id"], # noqa
"gpu_device": backend,
})
elif self.config["framework_name"] == "mnn":
# '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa
# '{forwardtype} {thread_num} {precision}"'
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"forwardtype": backend,
# power_mode: big_core default
})
elif self.config["framework_name"] == "tflite":
# {device_benchmark_bin} --graph={model_dir}
# --output_prefix={model_name}
# --num_runs={repeats} --warmup_runs={warmup}
# --num_threads={thread_num} {backend}
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir":
model_dir,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"backend":
self.config["support_backend_cmd_id"]
( # noqa
backend),
# power_mode(TODO): no bind deafault? need explore deeply # noqa
})
print(bench_cmd)
else:
logger.fatal("Unsupported framework {}".format(
self.config["framework_name"]))
exit(1)
#################################
# run benchmark
#################################
run_times_sum = (
self.config["repeats"](backend) +
self.config["warmup"] # noqa
) # noqa
max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER *
run_times_sum) # noqa
cmd_res = run_cmd(
bench_cmd,
wait_interval_sec=3,
max_timeout_sec=max_wait_sec,
) # noqa
perf_dict = self.parse_benchmark(cmd_res)
#################################
# summarize benchmark info
#################################
bench_record = {
"soc_code":
device_dict[device_serial]["soc_code"], ## noqa
"product":
device_dict[device_serial][ # noqa
"product"], # noqa
"serial_num":
device_serial,
"platform":
platform,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend),
"warmup":
self.config["warmup"],
"avg":
perf_dict["avg"],
"max":
perf_dict["max"],
"min":
perf_dict["min"],
"std_dev":
perf_dict["std_dev"],
"backend":
backend,
"cpu_thread_num":
cpu_thread_num,
"power_mode":
self.config["power_mode"],
"bind_cpu_idx":
device_dict[device_serial]["bind_cpu_idx"], # noqa
"cpu_max_freqs":
device_dict[device_serial][ # noqa
"cpu_max_freqs"],
"battery_level":
device_dict[device_serial][ # noqa
"battery_level"],
"system_version":
device_dict[device_serial][ # noqa
"system_version"],
"cmd":
bench_cmd,
"imei":
device_dict[device_serial]["imei"],
}
bench_dict[device_serial][platform][
model_name][ # noqa
backend][cpu_thread_num] = bench_record # noqa
logger.info(bench_record)
return bench_dict
def run_bench(self):
logger.info("==== {} ====".format(self.run_bench.__name__))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
model_dict = self.config["model_dict"]
device_serials = list(device_dict.keys())
model_names = list(model_dict.keys())
benchmark_platform = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
bench_dict = dict()
bench_case_idx = 0
for didx in range(len(device_serials)):
device_serial_num = device_serials[didx]
logger.debug(
"didx:{}, serial_num:{}".format(didx, device_serial_num) # noqa
)
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join(
[device_work_dir, os.path.basename(model_dict[model_name])] # noqa
)
logger.debug(
"midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir
)
)
bench_dict[model_name] = []
for pidx in range(len(benchmark_platform)):
platform = benchmark_platform[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join(
[
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]
), # noqa
]
)
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
is_cpu = self.config["is_cpu_backend"]
logger.debug(
"bidx:{}, backend:{}".format(bidx, backend)
) # noqa
for tidx in range(
len(self.config["cpu_thread_num"])
if is_cpu(backend)
else 1 # noqa
):
bench_case_idx += 1
logger.info(
"\n\nbench_case_idx(from 1):{}".format(
bench_case_idx
) # noqa
)
cpu_thread_num = self.config["cpu_thread_num"][tidx] # noqa
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_work_dir": device_work_dir_platform, # noqa
"device_benchmark_bin": device_benchmark_bin, # noqa
"model_type": self.config[
"framework_name"
], # noqa
"model_dir": model_dir,
"backend": backend,
"repeats": self.config["repeats"](
backend
), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"bind_cpu_idx": device_dict[
device_serial_num
][ # noqa
"bind_cpu_idx"
],
}
)
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_benchmark_bin": device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend
), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode": self.config[
"power_mode_id"
], # noqa
"gpu_device": backend,
}
)
cmd_handle = run_cmd(
bench_cmd, wait_interval_sec=3 # noqa
) # noqa
perf_dict = self.parse_benchmark(cmd_handle)
# summarize benchmark info
bench_record = {
"soc": device_dict[device_serial_num]["soc"],
"product": device_dict[device_serial_num][ # noqa
"product"
], # noqa
"serial_num": device_serial_num,
"platform": platform,
"model_name": model_name,
"repeats": self.config["repeats"](backend),
"warmup": self.config["warmup"],
"avg": perf_dict["avg"],
"max": perf_dict["max"],
"min": perf_dict["min"],
"backend": backend,
"cpu_thread_num": cpu_thread_num,
"power_mode": self.config["power_mode"],
"bind_cpu_idx": device_dict[device_serial_num][
"bind_cpu_idx"
],
"cpu_max_freqs": device_dict[device_serial_num][ # noqa
"cpu_max_freqs"
],
"battery_level": device_dict[device_serial_num][ # noqa
"battery_level"
],
"system_version": device_dict[
device_serial_num
][ # noqa
"system_version"
],
"cmd": bench_cmd,
"imei": device_dict[device_serial_num]["imei"],
}
bench_dict[model_name].append(bench_record)
logger.info(bench_record)
return bench_dict
def prepare_devices(self):
logger.info("==== {} ====".format(self.prepare_devices.__name__))
device_status_dict = get_adb_devices(True)
serial_num_list = list(device_status_dict.keys())
logger.debug(serial_num_list)
device_dict = dict()
for sidx in range(len(serial_num_list)):
device_serial_num = serial_num_list[sidx]
device_status = device_status_dict[device_serial_num]
if device_status != "device":
logger.info(
"device {} status is {}, skipped".format(
device_serial_num, device_status
)
)
continue
device_dict[device_serial_num] = dict()
device_dict[device_serial_num]["status"] = device_status
device_dict[device_serial_num]["cpu_max_freqs"] = get_cpu_max_freqs( # noqa
device_serial_num
)
cpu_max_freqs = get_cpu_max_freqs(device_serial_num)
cpu_valid_freqs = list(
filter(lambda freq: freq is not None, cpu_max_freqs)
) # noqa
big_cores_idx = get_target_freq_idx(
max(cpu_valid_freqs), device_serial_num, cpu_max_freqs
)
big_cores_idx_str = ",".join(big_cores_idx)
little_cores_idx = get_target_freq_idx(
min(cpu_valid_freqs), device_serial_num, cpu_max_freqs
)
little_cores_idx_str = ",".join(little_cores_idx)
device_dict[device_serial_num]["big_cores_idx"] = big_cores_idx_str
device_dict[device_serial_num][
"little_cores_idx"
] = little_cores_idx_str # noqa
if self.config["power_mode"] == "big_cores":
device_dict[device_serial_num][
"bind_cpu_idx"
] = big_cores_idx_str # noqa
elif self.config["power_mode"] == "little_cores":
device_dict[device_serial_num][
"bind_cpu_idx"
] = little_cores_idx_str # noqa
elif self.config["power_mode"] == "no_bind":
device_dict[device_serial_num]["bind_cpu_idx"] = ",".join(
map(str, range(len(cpu_max_freqs)))
)
else:
logger.info(
"Unsupported power_mode:{}".format(
self.config["power_mode"]
) # noqa
)
exit(1)
# battery level
device_dict[device_serial_num]["battery_level"] = get_battery_level( # noqa
device_serial_num
)
# system version
device_dict[device_serial_num][
"system_version"
] = get_system_version( # noqa
device_serial_num
)
# imie
device_dict[device_serial_num]["imei"] = get_imei(device_serial_num) # noqa
# ro.board.platform, ro.board.chiptype, ro.board.hardware
device_soc_cmd = (
"adb -s {} shell getprop |"
" grep 'ro.board.platform'".format(device_serial_num)
)
cmd_handls = run_cmds([device_soc_cmd])
soc = cmd_handls[device_soc_cmd][0]
soc = soc.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa
device_dict[device_serial_num]["soc"] = soc
logger.debug(soc)
# product
device_product_cmd = (
"adb -s {} shell getprop | "
"grep 'ro.product.model'".format(device_serial_num)
)
cmd_handle = run_cmd(device_product_cmd)
product = cmd_handle[0]
product = (
product.split(": ")[1].strip().replace("[", "").replace("]", "") # noqa
) # noqa
device_dict[device_serial_num]["product"] = product
logger.debug(product)
logger.debug(device_dict)
logger.info("len(device_dict):{}".format(len(device_dict)))
return device_dict
def run_bench(self):
logger.info("==== {} ====".format(self.run_bench.__name__))
device_work_dir = self.config["device_work_dir"]
device_dict = self.config["device_dict"]
model_dict = self.config["model_dict"]
device_serials = list(device_dict.keys())
model_names = list(model_dict.keys())
benchmark_platform = self.config["benchmark_platform"]
support_backend = self.config["support_backend"]
bench_cmd_pattern = self.config["bench_cmd_pattern"]
bench_dict = dict()
bench_case_idx = 0
for didx in range(len(device_serials)):
device_serial_num = device_serials[didx]
logger.debug("didx:{}, serial_num:{}".format(
didx, device_serial_num) # noqa
)
bench_dict[device_serial_num] = dict()
for pidx in range(len(benchmark_platform)):
platform = benchmark_platform[pidx]
device_work_dir_platform = device_work_dir + "/" + platform # noqa
device_benchmark_bin = "/".join([
device_work_dir_platform,
os.path.basename(
self.config[platform]["benchmark_bin"]), # noqa
])
bench_dict[device_serial_num][platform] = dict()
logger.debug("pidx:{}, platform:{}".format(pidx, platform))
for midx in range(len(model_names)):
model_name = model_names[midx]
model_dir = "/".join([
device_work_dir,
os.path.basename(model_dict[model_name]),
] # noqa
)
logger.debug("midx:{}, model_name:{}, model_dir:{}".format(
midx, model_name, model_dir))
bench_dict[device_serial_num][platform][model_name] = dict(
) # noqa
for bidx in range(len(support_backend)):
backend = support_backend[bidx]
is_cpu = self.config["is_cpu_backend"]
logger.debug("bidx:{}, backend:{}".format(
bidx, backend)) # noqa
bench_dict[device_serial_num][platform][model_name][
backend] = dict() # noqa
for tidx in range(
len(self.config["cpu_thread_num"]
) if is_cpu(backend) else 1 # noqa
):
bench_case_idx += 1
logger.info("\n\nbench_case_idx(from 1):{}".format(
bench_case_idx) # noqa
)
cpu_thread_num = self.config["cpu_thread_num"][
tidx] # noqa
bench_dict[device_serial_num][platform][
model_name][ # noqa
backend][cpu_thread_num] = dict() # noqa
#######################
# bench start
#######################
if self.config["framework_name"] == "tnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num":
device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_type":
self.config["framework_name"], # noqa
"model_dir":
model_dir,
"backend":
backend,
"repeats":
self.config["repeats"](
backend), # noqa
"warmup":
self.config["warmup"],
"thread_num":
cpu_thread_num,
"bind_cpu_idx":
device_dict[device_serial_num][ # noqa
"bind_cpu_idx"],
})
elif self.config["framework_name"] == "ncnn":
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"power_mode":
self.config["power_mode_id"], # noqa
"gpu_device": backend,
})
elif self.config["framework_name"] == "mnn":
# '{device_benchmark_bin} {model_dir} {repeats} {warmup}' # noqa
# '{forwardtype} {thread_num} {precision}"'
bench_cmd = bench_cmd_pattern.format(
**{
"serial_num": device_serial_num,
"device_work_dir":
device_work_dir_platform, # noqa
"device_benchmark_bin":
device_benchmark_bin, # noqa
"model_dir": model_dir,
"repeats": self.config["repeats"](
backend), # noqa
"warmup": self.config["warmup"],
"thread_num": cpu_thread_num,
"forwardtype": backend,
# power_mode: big_core default
})
else:
logger.fatal("Unsupported framework {}".format(
self.config["framework_name"]))
exit(1)
#################################
# run benchmark
#################################
run_times_sum = (
self.config["repeats"](backend) +
self.config["warmup"] # noqa
) # noqa
max_wait_sec = (MAX_TIMEOUT_SECOND_ONCE_INFER *
run_times_sum) # noqa
cmd_res = run_cmd(
bench_cmd,
wait_interval_sec=3,
max_timeout_sec=max_wait_sec,
) # noqa
perf_dict = self.parse_benchmark(cmd_res)
#################################
# summarize benchmark info
#################################
bench_record = {
"soc_code":
device_dict[device_serial_num]
["soc_code"], ## noqa
"product":
device_dict[device_serial_num][ # noqa
"product"], # noqa
"serial_num":
device_serial_num,
"platform":
platform,
"model_name":
model_name,
"repeats":
self.config["repeats"](backend),
"warmup":
self.config["warmup"],
"avg":
perf_dict["avg"],
"max":
perf_dict["max"],
"min":
perf_dict["min"],
"std_dev":
perf_dict["std_dev"],
"backend":
backend,
"cpu_thread_num":
cpu_thread_num,
"power_mode":
self.config["power_mode"],
"bind_cpu_idx":
device_dict[device_serial_num]["bind_cpu_idx"],
"cpu_max_freqs":
device_dict[device_serial_num][ # noqa
"cpu_max_freqs"],
"battery_level":
device_dict[device_serial_num][ # noqa
"battery_level"],
"system_version":
device_dict[device_serial_num][ # noqa
"system_version"],
"cmd":
bench_cmd,
"imei":
device_dict[device_serial_num]["imei"],
}
bench_dict[device_serial_num][platform][
model_name][ # noqa
backend][
cpu_thread_num] = bench_record # noqa
# bench_dict[model_name].append(bench_record)
logger.info(bench_record)
return bench_dict
