def return_valid_run_flavors(self):
# '' corresponds to no userbuffer mode
flavors = {i: [] for i in RUNTIMES}
# give higher precedence to command line arg
if self.userbuffer_mode != '':
if self.userbuffer_mode in BUFFER_MODES:
for runtime in flavors:
flavors[runtime].append(self.userbuffer_mode)
else:
raise ConfigError('Wrong userbuffer mode {} specified'.format(
self.userbuffer_mode))
else:
if self.buffertypes is "All":
for runtime in flavors:
flavors[runtime].extend(RUNTIME_BUFFER_MODES[runtime])
else:
if not all([x in BUFFER_MODES for x in self.buffertypes]):
raise ConfigError(
'Wrong buffer mode specified in config file')
for runtime in flavors:
flavors[runtime].extend(self.buffertypes)
runtimes = []
runtimes.extend(self.runtimes)
# TODO figure out a better way to decide whether GPU_s is supported
if RUNTIME_GPU in runtimes and \
any([x in self._artifacts_config[CONFIG_ARTIFACTS_KEY] for x in CONFIG_ARTIFACTS_COMPILER_KEY_TARGET_ANDROID_OSPACE]) and \
not self.cpu_fallback:
runtimes.insert(runtimes.index(RUNTIME_GPU) + 1, RUNTIME_GPU_ONLY)
for runtime in flavors:
flavors[runtime] = [
x for x in flavors[runtime]
if x in RUNTIME_BUFFER_MODES[runtime]
]
return [(i, (lambda: j, lambda: "")[j == "float"]()) for i in runtimes
for j in flavors[i]]
def __default_path(artifact_path):
_abs_path = artifact_path
if '.dlc' in _abs_path:
if 'MODELZOO' in os.environ:
_abs_path = os.path.join(os.environ['MODELZOO'], _abs_path)
return _abs_path
if not os.path.isabs(artifact_path):
# relative to current directory
_abs_path = os.path.abspath(artifact_path)
if not os.path.exists(_abs_path):
raise ConfigError(artifact_path + " does not exist")
return _abs_path
def __device_artifacts_helper(self, runtime):
'''
Given a runtime (CPU, DSP, GPU, etc), return the artifacts folder
Note that the logic is as simple as:
1. There should be at least one artifacts folder.
2. If you are asking for GPU_S, there needs to be 2
3. GPU_S will return the one ending with "_s", all others will get the first one
'''
if runtime == RUNTIME_GPU_ONLY:
anchor = SNPE_RUNTIME_GPU_S_LIB
else:
anchor = SNPE_RUNTIME_LIB
for _compiler, _artifacts in self._artifacts_config[
CONFIG_ARTIFACTS_KEY].items():
if anchor in _artifacts:
return os.path.join(self.device_path, ARTIFACT_DIR, _compiler)
raise ConfigError('Unable to find device artifacts')
def __default_artifact_path(compiler, artifact):
# first check for ZDL_ROOT, because if you have both, contributor role takes precedence
if ZDL_ROOT not in os.environ:
if SNPE_SDK_ROOT not in os.environ:
raise ConfigError(
"Environment variables 'SNPE_ROOT' and 'ZDL_ROOT' are not defined, absolute path is needed for "
+ artifact + " in snpebm_artifacts.json")
_sdk_root = os.environ[SNPE_SDK_ROOT]
_base_name = os.path.basename(artifact)
if _base_name.endswith(".so") or _base_name.startswith("lib"):
return os.path.join(_sdk_root, "lib", compiler, artifact)
else:
return os.path.join(_sdk_root, "bin", compiler, artifact)
else:
_zdl_root = os.environ[ZDL_ROOT]
_base_name = os.path.basename(artifact)
if _base_name.endswith(".so") or _base_name.startswith("lib"):
return os.path.join(_zdl_root, compiler, "lib", artifact)
else:
return os.path.join(_zdl_root, compiler, "bin", artifact)
def device_artifacts_lib(self):
raise ConfigError('Deprecated call')
def __init__(self, config, host_artifacts):
self._name = config[CONFIG_MODEL_KEY][CONFIG_MODEL_NAME_SUBKEY]
self._dlc = DnnModel.__default_path(
config[CONFIG_MODEL_KEY][CONFIG_MODEL_DLC_SUBKEY])
self._dev_root_dir = os.path.join(config[CONFIG_DEVICE_PATH_KEY],
self._name)
self._host_artifacts = host_artifacts
subkeys = []
for sub_key in config[CONFIG_MODEL_KEY]:
subkeys.append(sub_key)
if "InputList" in subkeys:
self._input_list_name = os.path.basename(
config[CONFIG_MODEL_KEY][CONFIG_MODEL_INPUTLIST_SUBKEY])
self._artifacts = []
self._artifacts.append([self._dlc, self._dev_root_dir])
self._artifacts.append([
DnnModel.__default_path(
config[CONFIG_MODEL_KEY][CONFIG_MODEL_INPUTLIST_SUBKEY]),
self._dev_root_dir
])
for data in config[CONFIG_MODEL_KEY][CONFIG_MODEL_DATA_SUBKEY]:
_abs_data_path = DnnModel.__default_path(data)
self._artifacts.append([
_abs_data_path,
os.path.join(self._dev_root_dir, os.path.basename(data))
])
elif "RandomInput" in subkeys:
# Get the input dimensions and generate random data
input_layer_dim = self.input_layers
num_inputs = config[CONFIG_MODEL_KEY][
CONFIG_MODEL_RANDOMINPUT_SUBKEY]
if num_inputs < 1:
raise ConfigError('Input can not be less than 1')
random_data_dim_str = input_layer_dim[0].split(":")[-1]
random_data_dim = random_data_dim_str.split(",")
random_data_dim = list(map(int, random_data_dim))
random_inputs_dir = os.path.join(config[CONFIG_HOST_ROOTPATH_KEY],
'random_inputs')
_abs_random_inputs_path = os.path.abspath(random_inputs_dir)
if os.path.isdir(_abs_random_inputs_path):
shutil.rmtree(_abs_random_inputs_path)
os.makedirs(_abs_random_inputs_path)
self._input_list_name = "random_raw_list.txt"
input_file_path = os.path.join(_abs_random_inputs_path,
self._input_list_name)
input_file = open(input_file_path, 'w')
for i in range(1, num_inputs + 1):
raw_filepath = os.path.join(_abs_random_inputs_path,
'random_input_' + str(i) + '.raw')
rand_raw = np.random.uniform(
-1.0, +1.0, random_data_dim).astype(np.float32)
with open(raw_filepath, 'wb') as fid:
fid.write(rand_raw)
raw_rel_path = os.path.join(
os.path.basename(random_inputs_dir),
os.path.basename(raw_filepath))
input_file.write(raw_rel_path + "\n")
input_file.close()
self._artifacts = []
self._artifacts.append([self._dlc, self._dev_root_dir])
self._artifacts.append([input_file_path, self._dev_root_dir])
self._artifacts.append([
_abs_random_inputs_path,
os.path.join(self._dev_root_dir,
os.path.basename(_abs_random_inputs_path))
])
def __quick_verify__(self):
try:
# Check that we have at least all top level keys
# note that this will give an exception on any key that
# isn't present
# NOTE: We do not validate the values provided for that key
for _key in CONFIG_JSON_ROOTKEYS:
# check optional keys
if not _key is CONFIG_PERF_PROFILE_KEY and not _key is CONFIG_CPU_FALLBACK_KEY and not _key is CONFIG_HOST_NAME_KEY and not _key is CONFIG_BUFFERTYPES_KEY and not _key is CONFIG_PROFILING_LEVEL_KEY:
if self._cfg_from_json[_key] is 'null':
raise ConfigError("Missing value for " + _key)
# Check for no foreign top level keys
for _key in self._cfg_from_json.keys():
if _key not in CONFIG_JSON_ROOTKEYS:
raise ConfigError("Found invalid top level key: " + _key)
except KeyError as ke:
raise ConfigError('Missing key in config file: ' + repr(ke))
# We have all top level keys. Check some of the sub keys
if 'RandomInput' in self._cfg_from_json[CONFIG_MODEL_KEY]:
subkeys = CONFIG_JSON_MODEL_COMMON_SUBKEYS + CONFIG_JSON_MODEL_RANDOM_INPUT_SUBKEYS
for _key in subkeys:
if not _key in self._cfg_from_json[CONFIG_MODEL_KEY]:
raise ConfigError("No " + CONFIG_MODEL_KEY + ":" + _key +
" found")
else:
subkeys = CONFIG_JSON_MODEL_COMMON_SUBKEYS + CONFIG_JSON_MODEL_DEFINED_INPUT_SUBKEYS
for _key in subkeys:
if not _key in self._cfg_from_json[CONFIG_MODEL_KEY]:
raise ConfigError("No " + CONFIG_MODEL_KEY + ":" + _key +
" found")
# All relative paths are relative to the current directory
for _key in [CONFIG_HOST_ROOTPATH_KEY, CONFIG_HOST_RESULTSDIR_KEY]:
_value = self._cfg_from_json[_key]
if not os.path.isabs(_value):
_abs_path = os.path.abspath(_value)
if os.path.isfile(_abs_path):
raise ConfigError(_key + " is not a directory")
self._cfg_from_json[_key] = os.path.abspath(_value)
# Check artifacts paths, relative path is not supported
# currently hardcoded to ARM as target architecture
for _arch in self.architectures:
for _compiler, _artifacts in list(
self._artifacts_config[CONFIG_ARTIFACTS_KEY].items()):
if (not _compiler.startswith(_arch)) and (
not _compiler.startswith(ARCH_X86) and
(not _compiler.startswith(ARCH_DSP))):
continue
for _artifact_path in _artifacts:
if (not os.path.isabs(_artifact_path)
) and os.path.dirname(_artifact_path):
raise ConfigError(
"{0} does not support relative path".format(
CONFIG_ARTIFACTS_KEY))
elif os.path.isabs(_artifact_path) and (
not os.path.exists(_artifact_path)):
raise ConfigError(
"{0} does not exist".format(_artifact_path))
elif not os.path.dirname(_artifact_path):
if not os.path.exists(
self.__default_artifact_path(
_compiler, _artifact_path)):
raise ConfigError(
"Could not find {0} for {1}, path used {2}".
format(
_artifact_path, _compiler,
self.__default_artifact_path(
_compiler, _artifact_path)))
# at the momemnt, despite devices takes in a list, benchmark does not work correctly when multiple devices are specified
device_list = self._cfg_from_json.get(CONFIG_DEVICES_KEY, None)
if len(device_list) == 0 or not device_list[0]:
raise ConfigError('Benchmark does not have any device specified')
elif len(device_list) != 1:
raise ConfigError(
'Benchmark does not yet support more than 1 device')
# Measurements allowed are "timing" and "mem"
if 0 == len(self._cfg_from_json.get(CONFIG_MEASUREMENTS_KEY, None)):
raise ConfigError('Benchmark does not specify what to measure')
else:
for measurement in self._cfg_from_json.get(CONFIG_MEASUREMENTS_KEY,
None):
if measurement not in [MEASURE_TIMING, MEASURE_MEM]:
raise ConfigError('"%s" is unknown measurement' %
measurement)
def __init__(self, cfg_file, cfg_from_json, outputbasedir, devicelist,
hostname, deviceostype, userbuffer_mode, perfprofile,
profilinglevel, enable_init_caching):
config_prefix = os.path.join(
os.path.abspath(os.path.dirname(__file__)), '../')
self._cfg_from_json = cfg_from_json
if deviceostype and deviceostype not in CONFIG_VALID_DEVICEOSTYPES:
raise ConfigError(
'Device OS Type not valid. Only specify one of %s' %
CONFIG_VALID_DEVICEOSTYPES)
if deviceostype == CONFIG_DEVICEOSTYPES_LE:
self._architectures = [ARCH_ARM]
self._platform_os = PLATFORM_OS_LINUX
self._compiler = COMPILER_GCC49
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_LE_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_LE64_GCC49:
self._architectures = [ARCH_AARCH64]
self._platform_os = PLATFORM_OS_LINUX
self._compiler = COMPILER_GCC49
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_LE64_GCC49_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_LE64_GCC53:
self._architectures = [ARCH_AARCH64]
self._platform_os = PLATFORM_OS_LINUX
self._compiler = COMPILER_GCC53
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_LE64_GCC53_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_ANDROID_AARCH64:
self._architectures = [ARCH_AARCH64]
self._compiler = COMPILER_CLANG60
self._platform_os = PLATFORM_OS_ANDROID
self._stl_library = STL_LIBCXX_SHARED
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_ANDROID_AARCH64_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_LE_OE_GCC64:
self._architectures = [ARCH_ARM]
self._platform_os = PLATFORM_OS_LINUX
self._compiler = COMPILER_GCC64
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_LE_OE_GCC64_HF_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_LE64_OE_GCC64:
self._architectures = [ARCH_AARCH64]
self._platform_os = PLATFORM_OS_LINUX
self._compiler = COMPILER_GCC64
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_LE64_OE_GCC64_ARTIFACTS_JSON))
elif deviceostype == CONFIG_DEVICEOSTYPES_QNX64_GCC54:
self._architectures = [ARCH_AARCH64]
self._platform_os = PLATFORM_OS_QNX
self._compiler = COMPILER_GCC54
self._stl_library = None
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_QNX64_GCC54_ARTIFACTS_JSON))
elif deviceostype:
# presumed to be Android arm32
self._architectures = [ARCH_ARM]
self._platform_os = PLATFORM_OS_ANDROID
self._compiler = COMPILER_CLANG60
self._stl_library = STL_LIBCXX_SHARED
self._artifacts_config = load_json(
os.path.join(config_prefix, "snpebm",
SNPE_BENCH_ANDROID_ARM32_ARTIFACTS_JSON))
self.__override_cfgfile__(outputbasedir, devicelist, hostname)
self.__quick_verify__()
self._dnnmodel = DnnModel(self._cfg_from_json, self.host_artifacts)
self.userbuffer_mode = userbuffer_mode
self.enable_init_caching = enable_init_caching
try:
self.hostname = self._cfg_from_json[CONFIG_HOST_NAME_KEY]
except KeyError:
self._cfg_from_json[CONFIG_HOST_NAME_KEY] = 'localhost'
try:
self.cpu_fallback = self._cfg_from_json[CONFIG_CPU_FALLBACK_KEY]
if not isinstance(self.cpu_fallback, bool):
raise ConfigError(CONFIG_CPU_FALLBACK_KEY + " key in " +
cfg_file + " expects boolean type. Found " +
str(type(self.cpu_fallback)))
except KeyError:
self.cpu_fallback = False
try:
if not self._cfg_from_json[CONFIG_PERF_PROFILE_KEY] is 'null':
self.perfprofile = self._cfg_from_json[CONFIG_PERF_PROFILE_KEY]
except KeyError as ke:
self.perfprofile = perfprofile
try:
if not self._cfg_from_json[CONFIG_PROFILING_LEVEL_KEY] is 'null':
self.profilinglevel = self._cfg_from_json[
CONFIG_PROFILING_LEVEL_KEY]
except KeyError as ke:
self.profilinglevel = profilinglevel
try:
self.buffertypes = self._cfg_from_json[CONFIG_BUFFERTYPES_KEY]
except KeyError:
self.buffertypes = "All"
# Required to set it to default 'high_performance' in case
# perf_profile is not provided as runtime argument or in the config file
if self.perfprofile == '' and perfprofile == '':
self.perfprofile = 'high_performance'
# Required to give preference to runtime argument in case
# perf_profile is provided both as runtime argument and in the config file
elif perfprofile != '':
self.perfprofile = perfprofile
if self.profilinglevel == '' and profilinglevel == '':
self.profilinglevel = 'basic'
elif profilinglevel != '':
self.profilinglevel = profilinglevel