def test_get_metric_list_of_str():
ie = IECore()
param = ie.get_metric("CPU", "OPTIMIZATION_CAPABILITIES")
assert isinstance(param, list), "Parameter value for 'OPTIMIZATION_CAPABILITIES' " \
f"metric must be a list but {type(param)} is returned"
assert all(isinstance(v, str) for v in param), "Not all of the parameter values for 'OPTIMIZATION_CAPABILITIES' " \
"metric are strings!"
def test_get_metric_tuple_of_two_ints():
ie = IECore()
param = ie.get_metric("CPU", "RANGE_FOR_STREAMS")
assert isinstance(param, tuple), "Parameter value for 'RANGE_FOR_STREAMS' " \
f"metric must be tuple but {type(param)} is returned"
assert all(isinstance(v, int) for v in param), "Not all of the parameter values for 'RANGE_FOR_STREAMS' " \
"metric are integers!"
def test_get_metric_tuple_of_three_ints():
ie = IECore()
param = ie.get_metric("CPU", "RANGE_FOR_ASYNC_INFER_REQUESTS")
assert isinstance(param, tuple), "Parameter value for 'RANGE_FOR_ASYNC_INFER_REQUESTS' " \
"metric must be tuple but {} is returned".format(type(param))
assert all(isinstance(v, int) for v in param), "Not all of the parameter values for " \
"'RANGE_FOR_ASYNC_INFER_REQUESTS' metric are integers!"
def boot_myriad():
myriad = {}
model_xml = args.model
model_bin = os.path.splitext(model_xml)[0] + ".bin"
# ------------ Load Inference Engine and network files -------------
log.info("Loading Inference Engine")
ie = IECore()
log.info("IR Engine info:")
log.info(" Available IR Devices: {}".format(ie.available_devices))
if 'MYRIAD' in ie.available_devices:
verinfo = ie.get_versions('MYRIAD')['MYRIAD']
log.info(" Myriad Plugin version: {}.{}".format(
verinfo.major, verinfo.minor))
log.info(" Myriad Build: {}".format(verinfo.build_number))
devices = ie.get_metric(metric_name='AVAILABLE_DEVICES',
device_name='MYRIAD')
if (len(devices) > 1):
ir_device = "MULTI:"
for device in devices:
ir_device += "MYRIAD." + device + ","
ir_device = ir_device[:-1]
else:
ir_device = "MYRIAD"
else:
ir_device = "CPU"
log.warn("!! MYRIAD DEVICE NOT FOUND. USING CPU !!")
log.info(" Selected Device(s): {}".format(ir_device))
log.info("Loading network files XML [{}] and BIN [{}]".format(
model_xml, model_bin))
net = IENetwork(model=model_xml, weights=model_bin)
net_input_layer_name = next(iter(net.inputs))
n, c, h, w = net.inputs[net_input_layer_name].shape
log.info(" Network shape: [{},{},{},{}]".format(n, c, h, w))
log.info("Loading labels file [{}]".format(args.labels.name))
labels_map = [x.strip() for x in args.labels]
log.info("Loading network to device... Please wait..")
exec_net = ie.load_network(network=net,
device_name=ir_device,
num_requests=4)
ir_num_requests = exec_net.get_metric('OPTIMAL_NUMBER_OF_INFER_REQUESTS')
log.info(" Optimal Number of Infer Requests: {}".format(ir_num_requests))
log.info("Done loading network")
myriad = {
'n': n,
'c': c,
'h': h,
'w': w,
'labels_map': labels_map,
'net': net,
'exec_net': exec_net,
'net_input_layer_name': net_input_layer_name,
}
return myriad
def test_register_plugin():
ie = IECore()
if ie.get_metric("CPU", "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to MKLDNNPlugin specific test")
ie.register_plugin("MKLDNNPlugin", "BLA")
net = ie.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie.load_network(net, "BLA")
assert isinstance(exec_net, ExecutableNetwork), "Cannot load the network to the registered plugin with name 'BLA'"
def get_cpu_name():
ie = IECore()
try:
cpuname = ie.get_metric('CPU', 'FULL_DEVICE_NAME')
except TypeError:
cpuname = 'Undefined'
del ie
return cpuname
def get_cpu_name():
ie = IECore()
if 'CPU' in ie.available_devices:
cpuname = ie.get_metric('CPU', 'FULL_DEVICE_NAME')
else:
cpuname = 'Undefined'
del ie
return cpuname
def get_gpu_name():
ie = IECore()
if 'GPU' in ie.available_devices:
gpuname = ie.get_metric('GPU', 'FULL_DEVICE_NAME')
else:
gpuname = 'Underfined'
del ie
return gpuname
def main():
ie = IECore()
print("Available devices:")
for device in ie.available_devices:
print("\tDevice: {}".format(device))
print("\tMetrics:")
for metric in ie.get_metric(device, "SUPPORTED_METRICS"):
try:
metric_val = ie.get_metric(device, metric)
print("\t\t{}: {}".format(metric, param_to_string(metric_val)))
except TypeError:
print("\t\t{}: UNSUPPORTED TYPE".format(metric))
print("\n\tDefault values for device configuration keys:")
for cfg in ie.get_metric(device, "SUPPORTED_CONFIG_KEYS"):
try:
cfg_val = ie.get_config(device, cfg)
print("\t\t{}: {}".format(cfg, param_to_string(cfg_val)))
except TypeError:
print("\t\t{}: UNSUPPORTED TYPE".format(cfg))
def test_query_network(device):
ie = IECore()
if device == "CPU":
if ie.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to ngraph")
import ngraph as ng
net = ie.read_network(model=test_net_xml, weights=test_net_bin)
query_res = ie.query_network(net, device)
func_net = ng.function_from_cnn(net)
ops_net = func_net.get_ordered_ops()
ops_net_names = [op.friendly_name for op in ops_net]
assert [key for key in query_res.keys() if key not in ops_net_names] == [], \
"Not all network layers present in query_network results"
assert next(iter(set(query_res.values()))) == device, "Wrong device for some layers"
def test_register_plugins():
ie = IECore()
if ie.get_metric("CPU", "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to MKLDNNPlugin specific test")
if platform == "linux" or platform == "linux2":
ie.register_plugins(plugins_xml)
elif platform == "darwin":
ie.register_plugins(plugins_osx_xml)
elif platform == "win32":
ie.register_plugins(plugins_win_xml)
net = ie.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie.load_network(net, "CUSTOM")
assert isinstance(exec_net,
ExecutableNetwork), "Cannot load the network to the registered plugin with name 'CUSTOM' " \
"registred in the XML file"
def test_buffer_values_after_add_outputs(device):
path_to_repo = os.environ["MODELS_PATH"]
test_net_xml_fp16 = os.path.join(path_to_repo, "models", "test_model", 'test_model_fp16.xml')
test_net_bin_fp16 = os.path.join(path_to_repo, "models", "test_model", 'test_model_fp16.bin')
ie_core = IECore()
if device == "CPU":
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to ngraph")
net = ie_core.read_network(model=test_net_xml_fp16, weights=test_net_bin_fp16)
output_layer = "22"
net.add_outputs(output_layer)
exec_net = ie_core.load_network(net, device)
feed_dict = {
'data': np.random.normal(0, 1, (1, 3, 32, 32)).astype(np.float32)
}
result = exec_net.infer(feed_dict)
assert np.all(abs(result[output_layer])<30)
assert result[output_layer].dtype == np.float16
def test_serialize(device):
ie = IECore()
if device == "CPU":
if ie.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to ngraph")
import ngraph as ng
net = ie.read_network(model=test_net_xml, weights=test_net_bin)
net.serialize("./serialized_net.xml", "./serialized_net.bin")
serialized_net = ie.read_network(model="./serialized_net.xml",
weights="./serialized_net.bin")
func_net = ng.function_from_cnn(net)
ops_net = func_net.get_ordered_ops()
ops_net_names = [op.friendly_name for op in ops_net]
func_serialized_net = ng.function_from_cnn(serialized_net)
ops_serialized_net = func_serialized_net.get_ordered_ops()
ops_serialized_net_names = [op.friendly_name for op in ops_serialized_net]
assert ops_serialized_net_names == ops_net_names
os.remove("./serialized_net.xml")
os.remove("./serialized_net.bin")
def benchmark_model(model_path: PathLike,
device: str = "CPU",
seconds: int = 60, api: str = "async",
batch: int = 1,
cache_dir: PathLike = "model_cache"):
"""
Benchmark model `model_path` with `benchmark_app`. Returns the output of `benchmark_app`
without logging info, and information about the device
:param model_path: path to IR model xml file, or ONNX model
:param device: device to benchmark on. For example, "CPU" or "MULTI:CPU,GPU"
:param seconds: number of seconds to run benchmark_app
:param api: API. Possible options: sync or async
:param batch: Batch size
:param cache_dir: Directory that contains model/kernel cache files
"""
ie = IECore()
model_path = Path(model_path)
if ("GPU" in device) and ("GPU" not in ie.available_devices):
raise ValueError(f"A GPU device is not available. Available devices are: {ie.available_devices}")
else:
benchmark_command = f"benchmark_app -m {model_path} -d {device} -t {seconds} -api {api} -b {batch} -cdir {cache_dir}"
display(Markdown(f"**Benchmark {model_path.name} with {device} for {seconds} seconds with {api} inference**"));
display(Markdown(f"Benchmark command: `{benchmark_command}`"));
benchmark_output = get_ipython().run_line_magic('sx', '$benchmark_command')
benchmark_result = [line for line in benchmark_output
if not (line.startswith(r"[") or line.startswith(" ") or line == "")]
print("\n".join(benchmark_result))
print()
if "MULTI" in device:
devices = device.replace("MULTI:", "").split(",")
for single_device in devices:
device_name = ie.get_metric(device_name=single_device, metric_name='FULL_DEVICE_NAME')
print(f"{single_device} device: {device_name}")
else:
print(f"Device: {ie.get_metric(device_name=device, metric_name='FULL_DEVICE_NAME')}")
def test_get_metric_str():
ie = IECore()
param = ie.get_metric("CPU", "FULL_DEVICE_NAME")
assert isinstance(param, str), "Parameter value for 'FULL_DEVICE_NAME' " \
"metric must be string but {} is returned".format(type(param))
class Classification:
def __init__(self, device: str, number_infer_requests, number_iterations, duration_seconds, api_type):
self.device = device
self.ie = IECore()
self.nireq = number_infer_requests
self.niter = number_iterations
self.duration_seconds = get_duration_seconds(duration_seconds, self.niter, self.device)
self.api_type = api_type
self.device_number_streams = {}
def __del__(self):
del self.ie
def add_extension(self, path_to_extension: str=None, path_to_cldnn_config: str=None):
if GPU_DEVICE_NAME in self.device:
if path_to_cldnn_config:
self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config}, GPU_DEVICE_NAME)
logger.info('GPU extensions is loaded {}'.format(path_to_cldnn_config))
if CPU_DEVICE_NAME in self.device or MYRIAD_DEVICE_NAME in self.device:
if path_to_extension:
self.ie.add_extension(extension_path=path_to_extension, device_name=CPU_DEVICE_NAME)
logger.info('CPU extensions is loaded {}'.format(path_to_extension))
def get_version_info(self) -> str:
logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format('', 'API version', get_version()))
version_string = 'Device info\n'
for device, version in self.ie.get_versions(self.device).items():
version_string += '{: <9}{}\n'.format('', device)
version_string += '{: <9}{:.<24}{} {}.{}\n'.format('', version.description, ' version', version.major,
version.minor)
version_string += '{: <9}{:.<24} {}\n'.format('', 'Build', version.build_number)
return version_string
@staticmethod
def reshape(ie_network: IENetwork, batch_size: int):
new_shapes = {}
for input_layer_name, input_layer in ie_network.inputs.items():
new_shapes[input_layer_name] = get_blob_shape(input_layer, batch_size)
if new_shapes:
logger.info('Resizing network to batch = {}'.format(batch_size))
ie_network.reshape(new_shapes)
def set_config(self, number_streams: int, api_type: str = 'async',
number_threads: int = None, infer_threads_pinning: int = None):
devices = parse_devices(self.device)
self.device_number_streams = parse_nstreams_value_per_device(devices, number_streams)
for device_name in self.device_number_streams.keys():
key = device_name + "_THROUGHPUT_STREAMS"
supported_config_keys = self.ie.get_metric(device_name, 'SUPPORTED_CONFIG_KEYS')
if key not in supported_config_keys:
raise Exception("Device " + device_name + " doesn't support config key '" + key + "'! " +
"Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>");
for device in devices:
if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys
# limit threading for CPU portion of inference
if number_threads:
self.ie.set_config({'CPU_THREADS_NUM': str(number_threads)}, device)
if MULTI_DEVICE_NAME in self.device and GPU_DEVICE_NAME in self.device:
self.ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME)
else:
# pin threads for CPU portion of inference
self.ie.set_config({'CPU_BIND_THREAD': infer_threads_pinning}, device)
# for CPU execution, more throughput-oriented execution via streams
# for pure CPU execution, more throughput-oriented execution via streams
if api_type == 'async':
cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}
if device in self.device_number_streams.keys():
cpu_throughput['CPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device))
self.ie.set_config(cpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(device, 'CPU_THROUGHPUT_STREAMS')
elif device == GPU_DEVICE_NAME:
if api_type == 'async':
gpu_throughput = {'GPU_THROUGHPUT_STREAMS': 'GPU_THROUGHPUT_AUTO'}
if device in self.device_number_streams.keys():
gpu_throughput['GPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device))
self.ie.set_config(gpu_throughput, device)
self.device_number_streams[device] = self.ie.get_config(device, 'GPU_THROUGHPUT_STREAMS')
if MULTI_DEVICE_NAME in self.device and CPU_DEVICE_NAME in self.device:
# multi-device execution with the CPU+GPU performs best with GPU trottling hint,
# which releases another CPU thread (that is otherwise used by the GPU driver for active polling)
self.ie.set_config({'CLDNN_PLUGIN_THROTTLE': '1'}, device)
elif device == MYRIAD_DEVICE_NAME:
self.ie.set_config({'LOG_LEVEL': 'LOG_INFO'}, MYRIAD_DEVICE_NAME)
def read_network(self, path_to_model: str):
xml_filename = os.path.abspath(path_to_model)
head, tail = os.path.splitext(xml_filename)
bin_filename = os.path.abspath(head + BIN_EXTENSION)
ie_network = self.ie.read_network(xml_filename, bin_filename)
input_info = ie_network.inputs
if not input_info:
raise AttributeError('No inputs info is provided')
return ie_network
def load_network(self, ie_network: IENetwork, perf_counts: bool):
config = {'PERF_COUNT': ('YES' if perf_counts else 'NO')}
exe_network = self.ie.load_network(ie_network,
self.device,
config=config,
num_requests=1 if self.api_type == 'sync' else self.nireq or 0)
# Number of requests
self.nireq = len(exe_network.requests)
return exe_network
def infer(self, exe_network, batch_size, progress_bar=None):
progress_count = 0
infer_requests = exe_network.requests
# warming up - out of scope
if self.api_type == 'sync':
infer_requests[0].infer()
else:
infer_requests[0].async_infer()
status = exe_network.wait()
if status != StatusCode.OK:
raise Exception("Wait for all requests is failed with status code {}!".format(status))
out_blob = next(iter(exe_network.outputs))
start_time = datetime.utcnow()
exec_time = 0
iteration = 0
times = []
in_fly = set()
# Start inference & calculate performance
# to align number if iterations to guarantee that last infer requests are executed in the same conditions **/
#(self.duration_seconds and exec_time < self.duration_seconds) or \
while (self.niter and iteration < self.niter) or \
(len(in_fly) == iteration*self.nireq) or \
(self.api_type == 'async' and iteration % self.nireq):
if self.api_type == 'sync':
infer_requests[0].infer()
times.append(infer_requests[0].latency)
else:
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
status = exe_network.wait(num_requests=1)
if status != StatusCode.OK:
raise Exception("Wait for idle request failed!")
infer_request_id = exe_network.get_idle_request_id()
if infer_request_id < 0:
raise Exception("Invalid request id!")
if infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
else:
in_fly.add(infer_request_id)
infer_requests[infer_request_id].async_infer()
#times.append(infer_requests[infer_request_id].latency)
iteration += 1
exec_time = (datetime.utcnow() - start_time).total_seconds()
if progress_bar:
if self.duration_seconds:
# calculate how many progress intervals are covered by current iteration.
# depends on the current iteration time and time of each progress interval.
# Previously covered progress intervals must be skipped.
progress_interval_time = self.duration_seconds / progress_bar.total_num
new_progress = int(exec_time / progress_interval_time - progress_count)
progress_bar.add_progress(new_progress)
progress_count += new_progress
elif self.niter:
progress_bar.add_progress(1)
# wait the latest inference executions
inference_output = []
status = exe_network.wait()
for infer_request in infer_requests:
output = infer_request.outputs[out_blob]
inference_output.append(output.tolist())
if status != StatusCode.OK:
raise Exception("Wait for all requests is failed with status code {}!".format(status))
total_duration_sec = (datetime.utcnow() - start_time).total_seconds()
for infer_request_id in in_fly:
times.append(infer_requests[infer_request_id].latency)
times.sort()
latency_ms = median(times)
fps = batch_size * 1000 / latency_ms if self.api_type == 'sync' else batch_size * iteration / total_duration_sec
if progress_bar:
progress_bar.finish()
return inference_output,fps, latency_ms, total_duration_sec, iteration
