def test_cannot_set_shape_preallocated_memory():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC")
array = np.ones([1, 3, 127, 127], dtype=np.float32)
blob = Blob(tensor_desc, array)
with pytest.raises(RuntimeError) as e:
blob.set_shape([1, 4, 128, 128])
assert "Cannot call setShape for Blobs created on top of preallocated memory" in str(e.value)
def test_write_to_buffer(precision, numpy_precision):
tensor_desc = TensorDesc(precision, [1, 3, 127, 127], "NCHW")
array = np.zeros(shape=(1, 3, 127, 127), dtype=numpy_precision)
blob = Blob(tensor_desc, array)
ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=numpy_precision)
blob.buffer[:] = ones_arr
assert np.array_equal(blob.buffer, ones_arr)
def test_write_numpy_scalar_int64():
tensor_desc = TensorDesc("I64", [], "SCALAR")
scalar = np.array(0, dtype=np.int64)
blob = Blob(tensor_desc, scalar)
scalar_to_write = np.array(1, dtype=np.int64)
blob.buffer[:] = scalar_to_write
assert np.array_equal(blob.buffer, np.atleast_1d(scalar_to_write))
def test_write_to_buffer_fp32():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW")
array = np.zeros(shape=(1, 3, 127, 127), dtype=np.float32)
blob = Blob(tensor_desc, array)
ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=np.float32)
blob.buffer[:] = ones_arr
assert np.array_equal(blob.buffer, ones_arr)
def test_write_to_buffer_uint16():
tensor_desc = TensorDesc("U16", [1, 3, 127, 127], "NCHW")
array = np.zeros(shape=(1, 3, 127, 127), dtype=np.uint16)
blob = Blob(tensor_desc, array)
ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=np.uint16)
blob.buffer[:] = ones_arr
assert np.array_equal(blob.buffer, ones_arr)
def predict(self, inputs, metadata=None, **kwargs):
"""
Args:
inputs: dictionary where keys are input layers names and values are data for them.
metadata: metadata of input representations
Returns:
raw data from network.
"""
if self._lstm_inputs:
return self._predict_sequential(inputs, metadata)
results = []
for infer_inputs in inputs:
if self._do_reshape:
input_shapes = {
layer_name: data.shape
for layer_name, data in infer_inputs.items()
}
self._reshape_input(input_shapes)
if self._use_set_blob:
has_info = hasattr(self.exec_network, 'input_info')
for key, input_data in infer_inputs.items():
if has_info:
ie_input_info = OrderedDict([
(name, data.input_data) for name, data in
self.exec_network.input_info.items()
])
else:
ie_input_info = self.exec_network.inputs
layout = self._target_layout_mapping.get(
key, ie_input_info[key].layout)
tensor_desc = TensorDesc(ie_input_info[key].precision,
input_data.shape, layout)
preprocess_info = self._preprocess_info.get(key)
if preprocess_info is not None:
self.exec_network.requests[0].set_blob(
key, Blob(tensor_desc, input_data),
preprocess_info)
else:
self.exec_network.requests[0].set_blob(
key, Blob(tensor_desc, input_data))
result = self.exec_network.infer(
infer_inputs
) if not self._use_set_blob else self.exec_network.infer()
results.append(result)
if metadata is not None:
for meta_ in metadata:
meta_['input_shape'] = self.inputs_info_for_meta()
if self._output_layouts:
meta_['output_layout'] = self._output_layouts
self._do_reshape = False
self._use_set_blob = self.disable_resize_to_input
return results
def predict(self, inputs, metadata=None, **kwargs):
if self._lstm_inputs:
return self._predict_sequential(inputs, metadata)
results = []
for infer_inputs in inputs:
if self._do_reshape:
input_shapes = {
layer_name: data.shape
for layer_name, data in infer_inputs.items()
}
self._reshape_input(input_shapes)
if self._use_set_blob:
has_info = hasattr(self.exec_network, 'input_info')
for key, input_data in infer_inputs.items():
if has_info:
ie_input_info = OrderedDict([
(name, data.input_data) for name, data in
self.exec_network.input_info.items()
])
else:
ie_input_info = self.exec_network.inputs
layout = self._target_layout_mapping.get(
key, ie_input_info[key].layout)
tensor_desc = TensorDesc(ie_input_info[key].precision,
input_data.shape, layout)
preprocess_info = self._preprocess_info.get(key)
if preprocess_info is not None:
self.exec_network.requests[0].set_blob(
key, Blob(tensor_desc, input_data),
preprocess_info)
else:
self.exec_network.requests[0].set_blob(
key, Blob(tensor_desc, input_data))
result = self.exec_network.infer(
infer_inputs
) if not self._use_set_blob else self.exec_network.infer()
results.append(result)
if self.reset_memory_state:
for state in self.exec_network.requests[0].query_state():
state.reset()
if metadata is not None:
self._fill_meta(metadata,
None if not self.dyn_input_layers else inputs[-1])
self._do_reshape = False
self._use_set_blob = self.disable_resize_to_input
return results
def test_init_with_numpy(shape, layout):
tensor_desc = TensorDesc("FP32", shape, layout)
array = np.ones(shape=shape, dtype=np.float32)
blob = Blob(tensor_desc, array)
assert isinstance(blob.buffer, np.ndarray)
assert np.shares_memory(blob.buffer, array)
assert blob.tensor_desc == tensor_desc
def test_incompatible_array_and_td():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW")
array = np.zeros(shape=(1, 2, 3, 4), dtype=np.float32)
with pytest.raises(AttributeError) as e:
Blob(tensor_desc, array)
assert "Number of elements in provided numpy array 24 and " \
"required by TensorDesc 48387 are not equal" in str(e.value)
def test_set_mean_image():
ie_core = IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
tensor_desc = TensorDesc("FP32", [0, 127, 127], "CHW")
mean_image_blob = Blob(tensor_desc)
preprocess_info = net.input_info["data"].preprocess_info
preprocess_info.set_mean_image(mean_image_blob)
assert preprocess_info.mean_variant == MeanVariant.MEAN_IMAGE
def test_set_shape():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC")
blob = Blob(tensor_desc)
blob.set_shape([1, 4, 128, 128])
assert blob.tensor_desc.dims == [1, 4, 128, 128]
assert blob.buffer.shape == (1, 4, 128, 128)
array = np.ones([1, 3, 127, 127], dtype=np.float32)
blob = Blob(tensor_desc, array)
blob.set_shape([1, 4, 128, 128])
assert blob.tensor_desc.dims == [1, 4, 128, 128]
assert blob.buffer.shape == (1, 4, 128, 128)
def test_set_mean_image_for_channel():
ie_core = IECore()
net = ie_core.read_network(model=test_net_xml, weights=test_net_bin)
tensor_desc = TensorDesc("FP32", [127, 127], "HW")
mean_image_blob = Blob(tensor_desc)
preprocess_info = net.input_info["data"].preprocess_info
preprocess_info.init(1)
preprocess_info.set_mean_image_for_channel(mean_image_blob, 0)
pre_process_channel = preprocess_info[0]
assert isinstance(pre_process_channel.mean_data, Blob)
assert pre_process_channel.mean_data.tensor_desc.dims == [127, 127]
assert preprocess_info.mean_variant == MeanVariant.MEAN_IMAGE
def test_incompatible_input_precision():
import cv2
n, c, h, w = (1, 3, 32, 32)
image = cv2.imread(path_to_image)
if image is None:
raise FileNotFoundError("Input image not found")
image = cv2.resize(image, (h, w)) / 255
image = image.transpose((2, 0, 1))
image = image.reshape((n, c, h, w))
tensor_desc = TensorDesc("FP32", [1, 3, 32, 32], "NCHW")
with pytest.raises(ValueError) as e:
Blob(tensor_desc, image)
assert "Data type float64 of provided numpy array " \
"doesn't match to TensorDesc precision FP32" in str(e.value)
def test_query_state_write_buffer(device, input_shape, data_type, mode):
ie_core = ie.IECore()
if device == "CPU":
if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin")
layout = ["C", "HW", "CHW", "NCHW"]
np_data_type = {"FP32": np.float32, "FP16": np.float16, "I32": np.int32}
from openvino.inference_engine import TensorDesc, Blob
net = ie.IENetwork(create_function_with_memory(input_shape, np_data_type[data_type]))
ie_core = ie.IECore()
exec_net = ie_core.load_network(network=net, device_name=device, num_requests=1)
request = exec_net.requests[0]
mem_states = request.query_state()
mem_state = mem_states[0]
assert mem_state.name == 'var_id_667'
# todo: Uncomment after fix 45611,
# CPU plugin returns outputs and memory state in FP32 in case of FP16 original precision
#assert mem_state.state.tensor_desc.precision == data_type
for i in range(1, 10):
if mode == "set_init_memory_state":
# create initial value
const_init = 5
init_array = np.full(input_shape, const_init, dtype=np_data_type[mem_state.state.tensor_desc.precision])
tensor_desc = TensorDesc(mem_state.state.tensor_desc.precision, input_shape, layout[len(input_shape) - 1])
blob = Blob(tensor_desc, init_array)
mem_state.state = blob
res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])})
expected_res = np.full(input_shape, 1 + const_init, dtype=np_data_type[data_type])
elif mode == "reset_memory_state":
# reset initial state of ReadValue to zero
mem_state.reset()
res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])})
# always ones
expected_res = np.full(input_shape, 1, dtype=np_data_type[data_type])
else:
res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])})
expected_res = np.full(input_shape, i, dtype=np_data_type[data_type])
assert np.allclose(res['MemoryAdd'], expected_res, atol=1e-6), \
"Expected values: {} \n Actual values: {} \n".format(expected_res, res)
def main():
log.basicConfig(format="[ %(levelname)s ] %(message)s",
level=log.INFO,
stream=sys.stdout)
args = build_argparser().parse_args()
log.info("Initializing Inference Engine")
ie = IECore()
version = ie.get_versions(args.device)[args.device]
version_str = "{}.{}.{}".format(version.major, version.minor,
version.build_number)
log.info("Plugin version is {}".format(version_str))
# read IR
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin))
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
# check input and output names
input_shapes = {
k: v.input_data.shape
for k, v in ie_encoder.input_info.items()
}
input_names = list(ie_encoder.input_info.keys())
output_names = list(ie_encoder.outputs.keys())
assert "input" in input_names, "'input' is not presented in model"
assert "output" in output_names, "'output' is not presented in model"
state_inp_names = [n for n in input_names if "state" in n]
state_param_num = sum(np.prod(input_shapes[n]) for n in state_inp_names)
log.info("state_param_num = {} ({:.1f}Mb)".format(state_param_num,
state_param_num * 4e-6))
# load model to the device
log.info("Loading model to the {}".format(args.device))
ie_encoder_exec = ie.load_network(network=ie_encoder,
device_name=args.device)
sample_inp = wav_read(args.input)
input_size = input_shapes["input"][1]
res = None
samples_out = []
samples_times = []
while sample_inp is not None and sample_inp.shape[0] > 0:
if sample_inp.shape[0] > input_size:
input = sample_inp[:input_size]
sample_inp = sample_inp[input_size:]
else:
input = np.pad(sample_inp,
((0, input_size - sample_inp.shape[0]), ),
mode='constant')
sample_inp = None
#forms input
inputs = {"input": input[None, :]}
#add states to input
for n in state_inp_names:
if res:
inputs[n] = res[n.replace('inp', 'out')].buffer
else:
#on the first iteration fill states by zeros
inputs[n] = np.zeros(input_shapes[n], dtype=np.float32)
t0 = time.perf_counter()
# Set inputs manually through InferRequest functionality to speedup
infer_request_ptr = ie_encoder_exec.requests[0]
for n, data in inputs.items():
info_ptr = ie_encoder.input_info[n]
blob = Blob(info_ptr.tensor_desc, data)
infer_request_ptr.set_blob(n, blob, info_ptr.preprocess_info)
# infer by IE
infer_request_ptr.infer()
res = infer_request_ptr.output_blobs
t1 = time.perf_counter()
samples_times.append(t1 - t0)
samples_out.append(res["output"].buffer.squeeze(0))
log.info("Sequence of length {:0.2f}s is processed by {:0.2f}s".format(
sum(s.shape[0] for s in samples_out) / 16000,
sum(samples_times),
))
sample_out = np.concatenate(samples_out, 0)
wav_write(args.output, sample_out)
def test_get_buffer():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW")
array = np.ones(shape=(1, 3, 127, 127), dtype=np.float32)
blob = Blob(tensor_desc, array)
assert np.array_equal(blob.buffer, array)
def main():
args = build_argparser().parse_args()
log.info('OpenVINO Inference Engine')
log.info('\tbuild: {}'.format(get_version()))
ie = IECore()
# read IR
log.info("Reading model {}".format(args.model))
model_xml = args.model
model_bin = model_xml.with_suffix(".bin")
ie_encoder = ie.read_network(model=model_xml, weights=model_bin)
# check input and output names
input_shapes = {
k: v.input_data.shape
for k, v in ie_encoder.input_info.items()
}
input_names = list(ie_encoder.input_info.keys())
output_names = list(ie_encoder.outputs.keys())
assert "input" in input_names, "'input' is not presented in model"
assert "output" in output_names, "'output' is not presented in model"
state_inp_names = [n for n in input_names if "state" in n]
state_param_num = sum(np.prod(input_shapes[n]) for n in state_inp_names)
log.debug("State_param_num = {} ({:.1f}Mb)".format(state_param_num,
state_param_num * 4e-6))
# load model to the device
ie_encoder_exec = ie.load_network(network=ie_encoder,
device_name=args.device)
log.info('The model {} is loaded to {}'.format(args.model, args.device))
start_time = perf_counter()
sample_inp = wav_read(args.input)
input_size = input_shapes["input"][1]
res = None
samples_out = []
while sample_inp is not None and sample_inp.shape[0] > 0:
if sample_inp.shape[0] > input_size:
input = sample_inp[:input_size]
sample_inp = sample_inp[input_size:]
else:
input = np.pad(sample_inp,
((0, input_size - sample_inp.shape[0]), ),
mode='constant')
sample_inp = None
#forms input
inputs = {"input": input[None, :]}
#add states to input
for n in state_inp_names:
if res:
inputs[n] = res[n.replace('inp', 'out')].buffer
else:
#on the first iteration fill states by zeros
inputs[n] = np.zeros(input_shapes[n], dtype=np.float32)
# Set inputs manually through InferRequest functionality to speedup
infer_request_ptr = ie_encoder_exec.requests[0]
for n, data in inputs.items():
info_ptr = ie_encoder.input_info[n]
blob = Blob(info_ptr.tensor_desc, data)
infer_request_ptr.set_blob(n, blob, info_ptr.preprocess_info)
# infer by IE
infer_request_ptr.infer()
res = infer_request_ptr.output_blobs
samples_out.append(res["output"].buffer.squeeze(0))
total_latency = (perf_counter() - start_time) * 1e3
log.info("Metrics report:")
log.info("\tLatency: {:.1f} ms".format(total_latency))
log.info("\tSample length: {:.1f} ms".format(
len(samples_out) * input_size / 16.0))
sample_out = np.concatenate(samples_out, 0)
wav_write(args.output, sample_out)
def test_get_tensor_desc():
tensor_desc = TensorDesc("FP32", [1, 127, 127, 3], "NHWC")
blob = Blob(tensor_desc)
assert blob.tensor_desc == tensor_desc
def test_init_with_numpy():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW")
array = np.ones(shape=(1, 3, 127, 127), dtype=np.float32)
blob = Blob(tensor_desc, array)
assert isinstance(blob.buffer, np.ndarray)
assert blob.tensor_desc == tensor_desc
data8uint = np.frombuffer(in_memory_file.getvalue(), np.uint8) # Convert string to an unsigned int array
image = cv2.cvtColor(cv2.imdecode(data8uint, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
if image.shape[:-1] != (h, w):
image = cv2.resize(image, (w, h))
image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW
image_with_n = np.expand_dims(image, 0)
it = datagen.flow(image_with_n, batch_size=n)
n_aug_samples = n - 1
image_batch = np.ndarray(shape=(n, c, h, w), dtype=np.float32)
image_batch[0] = image
for i in range(n_aug_samples):
image_batch[i + 1] = next(it)
image_batch = preprocess_input(image_batch, data_format="channels_first", mode="tf")
image_batch = Blob(net.input_info[input_blob].tensor_desc, image_batch)
image_batches.append({"name": image_name, "batch": image_batch})
num_loops = int(os.getenv("NUMBER_OF_TEST_LOOPS", "1"))
def work_generator():
for i in range(num_loops):
for image_batch in image_batches:
yield image_batch
work_queue = queue.Queue(maxsize=num_requests)
wait_for_work_timeout_seconds = int(os.getenv("WAIT_FOR_WORK_TIMEOUT", "60"))
def get_more_work(request_id):
try:
logger.debug("Request index {} getting more work from queue.".format(request_id))
return work_queue.get(block=True, timeout=wait_for_work_timeout_seconds)
def test_set_shape():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC")
blob = Blob(tensor_desc)
blob.set_shape([1, 4, 128, 128])
assert blob.tensor_desc.dims == [1, 4, 128, 128]
assert blob.buffer.shape == (1, 4, 128, 128)
def test_init_with_tensor_desc():
tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC")
blob = Blob(tensor_desc)
assert isinstance(blob.buffer, np.ndarray)
assert blob.tensor_desc == tensor_desc
