def test_output_shape(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
output = exec_net.output(0)
expected_shape = Shape([1, 10])
assert str(output.get_shape()) == str(expected_shape)
def test_infer_list_as_inputs(device):
num_inputs = 4
input_shape = [2, 1]
dtype = np.float32
params = [ops.parameter(input_shape, dtype) for _ in range(num_inputs)]
model = Model(ops.relu(ops.concat(params, 1)), params)
core = Core()
compiled_model = core.compile_model(model, device)
def check_fill_inputs(request, inputs):
for input_idx in range(len(inputs)):
assert np.array_equal(
request.get_input_tensor(input_idx).data, inputs[input_idx])
request = compiled_model.create_infer_request()
inputs = [np.random.normal(size=input_shape).astype(dtype)]
request.infer(inputs)
check_fill_inputs(request, inputs)
inputs = [
np.random.normal(size=input_shape).astype(dtype)
for _ in range(num_inputs)
]
request.infer(inputs)
check_fill_inputs(request, inputs)
def test_ports_as_inputs(device):
input_shape = [2, 2]
param_a = ops.parameter(input_shape, np.float32)
param_b = ops.parameter(input_shape, np.float32)
model = Model(ops.add(param_a, param_b), [param_a, param_b])
core = Core()
compiled = core.compile_model(model, device)
request = compiled.create_infer_request()
arr_1 = np.array([[1, 2], [3, 4]], dtype=np.float32)
arr_2 = np.array([[3, 4], [1, 2]], dtype=np.float32)
tensor1 = Tensor(arr_1)
tensor2 = Tensor(arr_2)
res = request.infer({
compiled.inputs[0]: tensor1,
compiled.inputs[1]: tensor2
})
assert np.array_equal(res[compiled.outputs[0]],
tensor1.data + tensor2.data)
res = request.infer({
request.model_inputs[0]: tensor1,
request.model_inputs[1]: tensor2
})
assert np.array_equal(res[request.model_outputs[0]],
tensor1.data + tensor2.data)
def test_read_model_from_ir():
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
assert isinstance(func, Model)
func = core.read_model(model=test_net_xml)
assert isinstance(func, Model)
def test_batched_tensors(device):
core = Core()
# TODO: remove when plugins will support set_input_tensors
core.register_plugin("openvino_template_plugin", "TEMPLATE")
batch = 4
one_shape = [1, 2, 2, 2]
one_shape_size = np.prod(one_shape)
batch_shape = [batch, 2, 2, 2]
data1 = ops.parameter(batch_shape, np.float32)
data1.set_friendly_name("input0")
data1.get_output_tensor(0).set_names({"tensor_input0"})
data1.set_layout(Layout("N..."))
constant = ops.constant([1], np.float32)
op1 = ops.add(data1, constant)
op1.set_friendly_name("Add0")
res1 = ops.result(op1)
res1.set_friendly_name("Result0")
res1.get_output_tensor(0).set_names({"tensor_output0"})
model = Model([res1], [data1])
compiled = core.compile_model(model, "TEMPLATE")
req = compiled.create_infer_request()
# Allocate 8 chunks, set 'user tensors' to 0, 2, 4, 6 chunks
buffer = np.zeros([batch * 2, *batch_shape[1:]], dtype=np.float32)
tensors = []
for i in range(batch):
# non contiguous memory (i*2)
tensors.append(
Tensor(np.expand_dims(buffer[i * 2], 0), shared_memory=True))
req.set_input_tensors(tensors)
with pytest.raises(RuntimeError) as e:
req.get_tensor("tensor_input0")
assert "get_tensor shall not be used together with batched set_tensors/set_input_tensors" in str(
e.value)
actual_tensor = req.get_tensor("tensor_output0")
actual = actual_tensor.data
for test_num in range(0, 5):
for i in range(0, batch):
tensors[i].data[:] = test_num + 10
req.infer() # Adds '1' to each element
# Reference values for each batch:
_tmp = np.array([test_num + 11] * one_shape_size,
dtype=np.float32).reshape([2, 2, 2])
for j in range(0, batch):
assert np.array_equal(actual[j], _tmp)
def test_get_metric_tuple_of_two_ints():
ie = Core()
param = ie.get_metric("CPU", "RANGE_FOR_STREAMS")
assert isinstance(param, tuple), "Parameter value for 'RANGE_FOR_STREAMS' " \
f"metric must be tuple but {type(param)} is returned"
assert all(isinstance(v, int) for v in param), \
"Not all of the parameter values for 'RANGE_FOR_STREAMS' metric are integers!"
def test_register_plugin():
ie = Core()
ie.register_plugin("MKLDNNPlugin", "BLA")
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = ie.compile_model(func, "BLA")
assert isinstance(exec_net, CompiledModel), \
"Cannot load the network to the registered plugin with name 'BLA'"
def test_input_element_type(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
assert input_node.get_element_type() == Type.f32
def test_input_get_shape(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
assert str(input_node.get_shape()) == str(Shape([1, 10]))
def profile_forward(pt_path, vino_path, dummy_input, samples=1000):
# Initialize Pytorch model
nn = Connect4NN()
nn.load_state_dict(torch.load(pt_path))
compiled_nn = torch.jit.script(nn)
# Load the network in Inference Engine
ie = Core()
model_ir = ie.read_model(model=vino_path)
compiled_model_ir = ie.compile_model(model=model_ir, device_name="CPU")
# Get input and output layers
input_layer_ir = next(iter(compiled_model_ir.inputs))
output_layer_ir = next(iter(compiled_model_ir.outputs))
# Run profiling Pytorch
start = time.perf_counter()
for i in range(samples):
nn(dummy_input)
print(f"PT took: {time.perf_counter() - start}")
start = time.perf_counter()
for i in range(samples):
compiled_nn(dummy_input)
print(f"Compiled PT took: {time.perf_counter() - start}")
# Run profiling OpenVINO
start = time.perf_counter()
for i in range(samples):
compiled_model_ir([dummy_input])[output_layer_ir]
print(f"OpenVINO took: {time.perf_counter() - start}")
def test_input_type(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
assert isinstance(input_node, Input)
def test_infer_new_request_numpy(device):
ie = Core()
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
img = read_image()
exec_net = ie.compile_model(func, device)
res = exec_net.infer_new_request({"data": img})
assert np.argmax(res[list(res)[0]]) == 2
def test_inputs(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
inputs = exec_net.inputs
assert isinstance(inputs, list)
assert len(inputs) == 1
def test_input_get_index(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.input(0)
expected_idx = 0
assert input.get_index() == expected_idx
def test_results_async_infer(device):
jobs = 8
num_request = 4
core = Core()
model = core.read_model(test_net_xml, test_net_bin)
compiled = core.compile_model(model, device)
infer_queue = AsyncInferQueue(compiled, num_request)
jobs_done = [{"finished": False, "latency": 0} for _ in range(jobs)]
def callback(request, job_id):
jobs_done[job_id]["finished"] = True
jobs_done[job_id]["latency"] = request.latency
img = read_image()
infer_queue.set_callback(callback)
assert infer_queue.is_ready
for i in range(jobs):
infer_queue.start_async({"data": img}, i)
infer_queue.wait_all()
request = compiled.create_infer_request()
outputs = request.infer({0: img})
for i in range(num_request):
np.allclose(list(outputs.values()), list(infer_queue[i].results.values()))
def __init__(self, model_path, device, stride):
self.device = device
self.stride = stride
log.info('OpenVINO Runtime')
log.info('\tbuild: {}'.format(get_version()))
self.core = Core()
log.info('Reading model {}'.format(model_path))
self.model = self.core.read_model(model_path)
required_output_keys = {'features', 'heatmaps', 'pafs'}
for output_tensor_name in required_output_keys:
try:
self.model.output(output_tensor_name)
except RuntimeError:
raise RuntimeError(
"The demo supports only topologies with the following output keys: {}"
.format(', '.join(required_output_keys)))
self.input_tensor_name = self.model.inputs[0].get_any_name()
compiled_model = self.core.compile_model(self.model, self.device)
self.infer_request = compiled_model.create_infer_request()
log.info('The model {} is loaded to {}'.format(model_path,
self.device))
def test_get_metric_list_of_str():
ie = Core()
param = ie.get_metric("CPU", "OPTIMIZATION_CAPABILITIES")
assert isinstance(param, list), "Parameter value for 'OPTIMIZATION_CAPABILITIES' " \
f"metric must be a list but {type(param)} is returned"
assert all(isinstance(v, str) for v in param), \
"Not all of the parameter values for 'OPTIMIZATION_CAPABILITIES' metric are strings!"
def test_default_version_IR_V11_seperate_paths():
core = Core()
xml_path = "./serialized_function.xml"
bin_path = "./serialized_function.bin"
shape = [100, 100, 2]
parameter_a = ov.parameter(shape, dtype=np.float32, name="A")
parameter_b = ov.parameter(shape, dtype=np.float32, name="B")
parameter_c = ov.parameter(shape, dtype=np.float32, name="C")
parameter_d = ov.parameter(shape, dtype=np.float32, name="D")
model = ov.floor(
ov.minimum(ov.abs(parameter_a), ov.multiply(parameter_b, parameter_c)))
func = Model(model, [parameter_a, parameter_b, parameter_c], "Model")
pass_manager = Manager()
pass_manager.register_pass("Serialize",
xml_path=xml_path,
bin_path=bin_path,
version="IR_V11")
pass_manager.run_passes(func)
res_func = core.read_model(model=xml_path, weights=bin_path)
assert func.get_parameters() == res_func.get_parameters()
assert func.get_ordered_ops() == res_func.get_ordered_ops()
os.remove(xml_path)
os.remove(bin_path)
def test_get_metric_tuple_of_three_ints():
ie = Core()
param = ie.get_metric("CPU", "RANGE_FOR_ASYNC_INFER_REQUESTS")
assert isinstance(param, tuple), "Parameter value for 'RANGE_FOR_ASYNC_INFER_REQUESTS' " \
f"metric must be tuple but {type(param)} is returned"
assert all(isinstance(v, int) for v in param), "Not all of the parameter values for " \
"'RANGE_FOR_ASYNC_INFER_REQUESTS' metric are integers!"
def main():
args = build_argparser().parse_args()
start_time = perf_counter()
with wave.open(args.input, 'rb') as wave_read:
channel_num, sample_width, sampling_rate, pcm_length, compression_type, _ = wave_read.getparams(
)
assert sample_width == 2, "Only 16-bit WAV PCM supported"
assert compression_type == 'NONE', "Only linear PCM WAV files supported"
assert channel_num == 1, "Only mono WAV PCM supported"
assert sampling_rate == 16000, "Only 16 KHz audio supported"
audio = np.frombuffer(wave_read.readframes(pcm_length * channel_num),
dtype=np.int16).reshape(
(pcm_length, channel_num))
log_melspectrum = QuartzNet.audio_to_melspectrum(audio.flatten(),
sampling_rate)
log.info('OpenVINO Runtime')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
quartz_net = QuartzNet(core, args.model, log_melspectrum.shape,
args.device)
character_probs = quartz_net.infer(log_melspectrum)
transcription = QuartzNet.ctc_greedy_decode(character_probs)
total_latency = (perf_counter() - start_time) * 1e3
log.info("Metrics report:")
log.info("\tLatency: {:.1f} ms".format(total_latency))
print(transcription)
def test_unregister_plugin(device):
ie = Core()
ie.unload_plugin(device)
func = ie.read_model(model=test_net_xml, weights=test_net_bin)
with pytest.raises(RuntimeError) as e:
ie.load_network(func, device)
assert f"Device with '{device}' name is not registered in the InferenceEngine" in str(e.value)
def get_plugin(device: str, cpu_ext: str = None, config: str = None):
core = Core()
# log.info('{} plugin:\n API version ............ {}'.format(device, plugin.version), extra={'no_lvl': True})
set_plugin_config(core=core, device=device, config=config)
if cpu_ext and 'CPU' in device:
set_cpu_extensions(core=core, cpu_ext=cpu_ext)
return core
def test_output_type(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
output = exec_net.output(0)
output_type = output.get_element_type().get_type_name()
assert output_type == "f32"
def main():
args = build_argparser().parse_args()
# Plugin initialization
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
core = Core()
if 'GPU' in args.device:
core.set_property("GPU", {"GPU_ENABLE_LOOP_UNROLLING": "NO", "CACHE_DIR": "./"})
# Read IR
log.info('Reading model {}'.format(args.model))
model = core.read_model(args.model)
if len(model.inputs) != 1:
raise RuntimeError("Demo supports only single input topologies")
input_tensor_name = model.inputs[0].get_any_name()
if args.output_blob is not None:
output_tensor_name = args.output_blob
else:
if len(model.outputs) != 1:
raise RuntimeError("Demo supports only single output topologies")
output_tensor_name = model.outputs[0].get_any_name()
characters = get_characters(args)
codec = CTCCodec(characters, args.designated_characters, args.top_k)
if len(codec.characters) != model.output(output_tensor_name).shape[2]:
raise RuntimeError("The text recognition model does not correspond to decoding character list")
input_batch_size, input_channel, input_height, input_width = model.inputs[0].shape
# Read and pre-process input image (NOTE: one image only)
preprocessing_start_time = perf_counter()
input_image = preprocess_input(args.input, height=input_height, width=input_width)[None, :, :, :]
preprocessing_total_time = perf_counter() - preprocessing_start_time
if input_batch_size != input_image.shape[0]:
raise RuntimeError("The model's input batch size should equal the input image's batch size")
if input_channel != input_image.shape[1]:
raise RuntimeError("The model's input channel should equal the input image's channel")
# Loading model to the plugin
compiled_model = core.compile_model(model, args.device)
infer_request = compiled_model.create_infer_request()
log.info('The model {} is loaded to {}'.format(args.model, args.device))
# Start sync inference
start_time = perf_counter()
for _ in range(args.number_iter):
infer_request.infer(inputs={input_tensor_name: input_image})
preds = infer_request.get_tensor(output_tensor_name).data[:]
result = codec.decode(preds)
print(result)
total_latency = ((perf_counter() - start_time) / args.number_iter + preprocessing_total_time) * 1e3
log.info("Metrics report:")
log.info("\tLatency: {:.1f} ms".format(total_latency))
sys.exit()
def test_input_get_rt_info(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
rt_info = input_node.get_rt_info()
assert isinstance(rt_info, RTMap)
def test_input_get_source_output(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
name = input_node.get_source_output().get_node().get_friendly_name()
assert name == "fc_out"
def test_input_get_partial_shape(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
expected_partial_shape = PartialShape([1, 10])
assert input_node.get_partial_shape() == expected_partial_shape
def test_const_output_docs(device):
core = Core()
func = core.read_model(model=test_net_xml, weights=test_net_bin)
exec_net = core.compile_model(func, device)
input = exec_net.output(0)
input_node = input.get_node().inputs()[0]
exptected_string = "openvino.runtime.Input wraps ov::Input<Node>"
assert input_node.__doc__ == exptected_string
def get_inputs_info(self, precision) -> dict:
core = Core()
net = core.read_model(self.model, self.weights)
inputs_info = {}
for item in net.inputs:
inputs_info[item.get_any_name()] = list(item.shape)
return inputs_info
def Option_1():
#! [Option_1]
core = Core()
# Read a network in IR or ONNX format
model = core.read_model(model_path)
compiled_model = core.compile_model(model=model,
device_name="MULTI:CPU,GPU")
