def test_async_infer_wait_finish(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=1)
img = read_image()
request = exec_net.requests[0]
request.async_infer({'data': img})
request.wait(ie.WaitMode.RESULT_READY)
res = request.output_blobs['fc_out'].buffer
assert np.argmax(res) == 2
del exec_net
del ie_core
del net
def test_async_infer_many_req(device):
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=5)
img = read_image()
for id in range(5):
request_handler = exec_net.start_async(request_id=id,
inputs={'data': img})
request_handler.wait()
res = request_handler.output_blobs['fc_out'].buffer
assert np.argmax(res) == 2
del exec_net
del ie_core
def test_write_to_input_blobs_directly(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
executable_network = ie_core.load_network(net, device, num_requests=1)
img = read_image()
request = executable_network.requests[0]
input_data = request.input_blobs["data"]
input_data.buffer[:] = img
assert np.array_equal(
executable_network.requests[0].input_blobs["data"].buffer, img)
del executable_network
del ie_core
del net
def test_set_blob_with_incorrect_name():
function = create_encoder([4, 4, 20, 20])
net = ng.function_to_cnn(function)
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
tensor_desc = exec_net.requests[0].input_blobs["data"].tensor_desc
tensor_desc.dims = [4, 4, 20, 20]
blob = ie.Blob(tensor_desc)
with pytest.raises(RuntimeError) as e:
exec_net.requests[0].set_blob("incorrect_name", blob)
assert f"Failed to find input or output with name: 'incorrect_name'" in str(
e.value)
def test_get_perf_counts(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
ie_core.set_config({"PERF_COUNT": "YES"}, device)
exec_net = ie_core.load_network(net, device)
img = read_image()
request = exec_net.requests[0]
request.infer({'data': img})
pc = request.get_perf_counts()
assert pc['29']["status"] == "EXECUTED"
del exec_net
del ie_core
del net
def test_async_infer_dynamic_network_3_requests(shapes):
function = create_encoder([3, 4, 20, 20])
net = ng.function_to_cnn(function)
net.reshape({"data": [3, 4, (20, 50), (20, 50)]})
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE", num_requests=3)
for i, request in enumerate(exec_net.requests):
request.async_infer({"data": np.ones(shapes[i])})
for i, request in enumerate(exec_net.requests):
status = request.wait(ie.WaitMode.RESULT_READY)
assert status == ie.StatusCode.OK
assert request.output_blobs['out'].tensor_desc.dims == shapes[i]
def test_exec_graph_info_deallocation(device):
ie_core = ie.IECore()
if device == "CPU":
if ie_core.get_metric(device,
"FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip(
"Can't run on ARM plugin due-to get_exec_graph_info method isn't implemented"
)
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device)
exec_graph_info = exec_net.get_exec_graph_info()
del ie_core
del exec_net
del exec_graph_info
def test_multi_out_data(device):
# Regression test CVS-23965
# Check that CDataPtr for all output layers not copied between outputs map items
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
net.add_outputs(['28/Reshape'])
exec_net = ie_core.load_network(net, device)
assert "fc_out" in exec_net.outputs and "28/Reshape" in exec_net.outputs
assert isinstance(exec_net.outputs["fc_out"], ie.CDataPtr)
assert isinstance(exec_net.outputs["28/Reshape"], ie.CDataPtr)
assert exec_net.outputs["fc_out"].name == "fc_out" and exec_net.outputs["fc_out"].shape == [1, 10]
assert exec_net.outputs["28/Reshape"].name == "28/Reshape" and exec_net.outputs["28/Reshape"].shape == [1, 5184]
del ie_core
pass
def test_infer_dynamic_network_without_set_shape(shape, p_shape, ref_shape):
function = create_encoder(shape)
net = ng.function_to_cnn(function)
net.reshape({"data": p_shape})
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
exec_net.infer({"data": np.ones(ref_shape)})
assert exec_net.requests[0].input_blobs["data"].tensor_desc.dims == ref_shape
request = exec_net.requests[0]
request.async_infer({"data": np.ones(ref_shape)})
status = request.wait(ie.WaitMode.RESULT_READY)
assert status == ie.StatusCode.OK
assert request.output_blobs['out'].tensor_desc.dims == ref_shape
def test_export_import():
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, "MYRIAD")
exported_net_file = 'exported_model.bin'
exec_net.export(exported_net_file)
assert os.path.exists(exported_net_file)
exec_net = ie_core.import_network(exported_net_file, "MYRIAD")
os.remove(exported_net_file)
img = read_image()
res = exec_net.infer({'data': img})
assert np.argmax(res['fc_out'][0]) == 3
del exec_net
del ie_core
def test_exec_graph(device):
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device)
img = read_image()
res = exec_net.infer({'data': img})
exec_graph = exec_net.get_exec_graph_info()
exec_graph_file = 'exec_graph.xml'
exec_graph.serialize(exec_graph_file)
assert os.path.exists(exec_graph_file)
os.remove(exec_graph_file)
del exec_net
del exec_graph
del ie_core
def test_async_infer_fill_inputs(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=1)
img = read_image()
request = exec_net.requests[0]
request.input_blobs['data'].buffer[:] = img
request.async_infer()
status_end = request.wait()
assert status_end == ie.StatusCode.OK
res = request.output_blobs['fc_out'].buffer
assert np.argmax(res[0]) == 2
del exec_net
del ie_core
del net
def test_blob_setter_with_preprocess(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(network=net, device_name=device, num_requests=1)
img = read_image()
tensor_desc = ie.TensorDesc("FP32", [1, 3, 32, 32], "NCHW")
img_blob = ie.Blob(tensor_desc, img)
preprocess_info = ie.PreProcessInfo()
preprocess_info.mean_variant = ie.MeanVariant.MEAN_IMAGE
request = exec_net.requests[0]
request.set_blob('data', img_blob, preprocess_info)
pp = request.preprocess_info["data"]
assert pp.mean_variant == ie.MeanVariant.MEAN_IMAGE
def test_set_batch_size(device):
ie_core = ie.IECore()
ie_core.set_config({"DYN_BATCH_ENABLED": "YES"}, device)
net = ie_core.read_network(test_net_xml, test_net_bin)
net.batch_size = 10
data = np.zeros(shape=net.inputs['data'].shape)
exec_net = ie_core.load_network(net, device)
data[0] = read_image()[0]
request = exec_net.requests[0]
request.set_batch(1)
request.infer({'data': data})
assert np.allclose(int(round(request.output_blobs['fc_out'].buffer[0][2])), 1), "Incorrect data for 1st batch"
del exec_net
del ie_core
del net
def test_set_blob_after_async_infer():
from conftest import create_ngraph_function
import ngraph as ng
function = create_ngraph_function([ng.Dimension(0,5), ng.Dimension(4), ng.Dimension(20), ng.Dimension(20)])
net = ng.function_to_cnn(function)
ie_core = ie.IECore()
ie_core.register_plugin("templatePlugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
request = exec_net.requests[0]
tensor_desc = request.input_blobs['data'].tensor_desc
tensor_desc.dims = [2, 4, 20, 20]
blob = ie.Blob(tensor_desc)
request.async_infer({"data": np.ones([4, 4, 20, 20])})
with pytest.raises(RuntimeError) as e:
request.set_blob("data", blob)
assert "REQUEST_BUSY" in str(e.value)
def test_wait_before_start(device):
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
num_requests = 5
exec_net = ie_core.load_network(net, device, num_requests=num_requests)
img = read_image()
requests = exec_net.requests
for id in range(num_requests):
status = requests[id].wait()
assert status == ie.StatusCode.INFER_NOT_STARTED
request_handler = exec_net.start_async(request_id=id, inputs={'data': img})
status = requests[id].wait()
assert status == ie.StatusCode.OK
assert np.argmax(request_handler.output_blobs['fc_out'].buffer) == 2
del exec_net
del ie_core
def test_set_blob_after_async_infer():
function = create_encoder([1, 4, 20, 20])
net = ng.function_to_cnn(function)
net.reshape({"data": [(0, 5), 4, 20, 20]})
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
request = exec_net.requests[0]
tensor_desc = request.input_blobs['data'].tensor_desc
tensor_desc.dims = [2, 4, 20, 20]
blob = ie.Blob(tensor_desc)
request.async_infer({"data": np.ones([4, 4, 20, 20])})
with pytest.raises(RuntimeError) as e:
request.set_blob("data", blob)
assert "REQUEST_BUSY" in str(e.value)
request.wait()
def test_set_blob_with_incorrect_size():
function = create_encoder([4, 4, 20, 20])
net = ng.function_to_cnn(function)
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
tensor_desc = exec_net.requests[0].input_blobs["data"].tensor_desc
tensor_desc.dims = [tensor_desc.dims[0]*2, 4, 20, 20]
blob = ie.Blob(tensor_desc)
print(exec_net.requests[0].output_blobs)
with pytest.raises(RuntimeError) as e:
exec_net.requests[0].set_blob("data", blob)
assert f"Input blob size is not equal network input size" in str(e.value)
with pytest.raises(RuntimeError) as e:
exec_net.requests[0].set_blob("out", blob)
assert f"Output blob size is not equal network output size" in str(e.value)
def test_async_infer_wait_while_callback_will_not_finish(device):
def callback(status, callback_status):
time.sleep(0.01)
callback_status['finished'] = True
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=1)
callback_status = {}
callback_status['finished'] = False
request = exec_net.requests[0]
request.set_completion_callback(callback, py_data=callback_status)
img = read_image()
request.async_infer({'data': img})
request.wait()
assert callback_status['finished'] == True
def test_infer_dynamic_network_twice():
shape, p_shape = [1, 4, 20, 20], [(0,5), 4, 20, 20]
ref_shape1, ref_shape2 = [2, 4, 20, 20], [3, 4, 20, 20]
function = create_encoder(shape)
net = ng.function_to_cnn(function)
net.reshape({"data": p_shape})
ie_core = ie.IECore()
ie_core.register_plugin("ov_template_plugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
request = exec_net.requests[0]
request.infer({"data": np.ones(ref_shape1)})
assert exec_net.requests[0].input_blobs["data"].tensor_desc.dims == ref_shape1
assert request.output_blobs['out'].tensor_desc.dims == ref_shape1
request.infer({"data": np.ones(ref_shape2)})
assert exec_net.requests[0].input_blobs["data"].tensor_desc.dims == ref_shape2
assert request.output_blobs['out'].tensor_desc.dims == ref_shape2
def test_inputs_deprecated(device):
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=5)
with warnings.catch_warnings(record=True) as w:
assert len(exec_net.inputs) == 1
assert "data" in exec_net.inputs
assert isinstance(exec_net.inputs['data'], ie.DataPtr)
assert len(w) == 3
for i in range (len(w)):
assert "'inputs' property of ExecutableNetwork class is deprecated. " \
"To access DataPtrs user need to use 'input_data' property " \
"of InputInfoCPtr objects which " \
"can be acessed by 'input_info' property." in str(w[i].message)
del exec_net
del ie_core
def test_get_perf_counts(device):
ie_core = ie.IECore()
if device == "CPU":
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to ngraph")
net = ie_core.read_network(test_net_xml, test_net_bin)
ie_core.set_config({"PERF_COUNT": "YES"}, device)
exec_net = ie_core.load_network(net, device)
img = read_image()
request = exec_net.requests[0]
request.infer({'data': img})
pc = request.get_perf_counts()
assert pc['29']["status"] == "EXECUTED"
assert pc['29']["layer_type"] == "FullyConnected"
del exec_net
del ie_core
del net
def test_set_batch_size(device):
ie_core = ie.IECore()
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to dynamic batch isn't supported")
ie_core.set_config({"DYN_BATCH_ENABLED": "YES"}, device)
net = ie_core.read_network(test_net_xml, test_net_bin)
net.batch_size = 10
data = np.zeros(shape=net.input_info['data'].input_data.shape)
exec_net = ie_core.load_network(net, device)
data[0] = read_image()[0]
request = exec_net.requests[0]
request.set_batch(1)
request.infer({'data': data})
assert np.allclose(int(round(request.output_blobs['fc_out'].buffer[0][2])), 1), "Incorrect data for 1st batch"
del exec_net
del ie_core
del net
def test_exec_graph(device):
ie_core = ie.IECore()
if device == "CPU":
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin due-to get_exec_graph_info method isn't implemented")
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device)
img = read_image()
res = exec_net.infer({'data': img})
exec_graph = exec_net.get_exec_graph_info()
exec_graph_file = 'exec_graph.xml'
exec_graph.serialize(exec_graph_file)
assert os.path.exists(exec_graph_file)
os.remove(exec_graph_file)
del exec_net
del exec_graph
del ie_core
def test_infer_net_from_buffer(device):
ie_core = ie.IECore()
with open(test_net_bin, 'rb') as f:
bin = f.read()
with open(test_net_xml, 'rb') as f:
xml = f.read()
net = ie_core.read_network(model=xml, weights=bin, init_from_buffer=True)
net2 = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
exec_net = ie_core.load_network(net, device)
exec_net2 = ie_core.load_network(net2, device)
img = read_image()
res = exec_net.infer({'data': img})
res2 = exec_net2.infer({'data': img})
del ie_core
del exec_net
del exec_net2
assert np.allclose(res['fc_out'], res2['fc_out'], atol=1E-4, rtol=1E-4)
def test_wait_for_callback(device):
def callback(status, callbacks_info):
time.sleep(0.01)
callbacks_info['finished'] += 1
ie_core = ie.IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
num_requests = 3
exec_net = ie_core.load_network(net, device, num_requests=num_requests)
callbacks_info = {}
callbacks_info['finished'] = 0
img = read_image()
for request in exec_net.requests:
request.set_completion_callback(callback, callbacks_info)
request.async_infer({'data': img})
exec_net.wait(num_requests)
assert callbacks_info['finished'] == num_requests
def test_blob_setter(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net_1 = ie_core.load_network(network=net, device_name=device, num_requests=1)
net.inputs['data'].layout = "NHWC"
exec_net_2 = ie_core.load_network(network=net, device_name=device, num_requests=1)
img = read_image()
res_1 = np.sort(exec_net_1.infer({"data": img})['fc_out'])
img = np.transpose(img, axes=(0, 2, 3, 1)).astype(np.float32)
tensor_desc = ie.IETensorDesc("FP32", [1, 3, 32, 32], "NHWC")
img_blob = ie.IEBlob(tensor_desc, img)
request = exec_net_2.requests[0]
request.set_blob('data', img_blob)
request.infer()
res_2 = np.sort(request.output_blobs['fc_out'].buffer)
assert np.allclose(res_1, res_2, atol=1e-2, rtol=1e-2)
def test_infer_modify_outputs(device):
ie_core = ie.IECore()
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net = ie_core.load_network(net, device, num_requests=1)
img = read_image()
request = exec_net.requests[0]
outputs0 = exec_net.infer({'data': img})
status_end = request.wait()
assert status_end == ie.StatusCode.OK
assert np.argmax(outputs0['fc_out']) == 2
outputs0['fc_out'][:] = np.zeros(shape=(1, 10), dtype=np.float32)
outputs1 = request.output_blobs
assert np.argmax(outputs1['fc_out'].buffer) == 2
outputs1['fc_out'].buffer[:] = np.ones(shape=(1, 10), dtype=np.float32)
outputs2 = request.output_blobs
assert np.argmax(outputs2['fc_out'].buffer) == 2
del exec_net
del ie_core
del net
def test_infer_dynamic_network_with_set_blob(shape, p_shape, ref_shape):
from conftest import create_ngraph_function
import ngraph as ng
function = create_ngraph_function(shape)
net = ng.function_to_cnn(function)
net.reshape({"data": p_shape})
ie_core = ie.IECore()
ie_core.register_plugin("templatePlugin", "TEMPLATE")
exec_net = ie_core.load_network(net, "TEMPLATE")
tensor_desc = exec_net.requests[0].input_blobs["data"].tensor_desc
tensor_desc.dims = ref_shape
blob = ie.Blob(tensor_desc)
exec_net.requests[0].set_blob("data", blob)
assert exec_net.requests[0].input_blobs["data"].tensor_desc.dims == ref_shape
request = exec_net.requests[0]
request.infer({"data": np.ones(ref_shape)})
request.async_infer({"data": np.ones(ref_shape)})
status = request.wait(ie.WaitMode.RESULT_READY)
assert status == ie.StatusCode.OK
assert request.output_blobs["out"].tensor_desc.dims == ref_shape
def test_blob_setter(device):
ie_core = ie.IECore()
if device == "CPU":
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin")
net = ie_core.read_network(test_net_xml, test_net_bin)
exec_net_1 = ie_core.load_network(network=net, device_name=device, num_requests=1)
net.input_info['data'].layout = "NHWC"
exec_net_2 = ie_core.load_network(network=net, device_name=device, num_requests=1)
img = read_image()
res_1 = np.sort(exec_net_1.infer({"data": img})['fc_out'])
img = np.transpose(img, axes=(0, 2, 3, 1)).astype(np.float32)
tensor_desc = ie.TensorDesc("FP32", [1, 3, 32, 32], "NHWC")
img_blob = ie.Blob(tensor_desc, img)
request = exec_net_2.requests[0]
request.set_blob('data', img_blob)
request.infer()
res_2 = np.sort(request.output_blobs['fc_out'].buffer)
assert np.allclose(res_1, res_2, atol=1e-2, rtol=1e-2)
