def get_binary_tensors(binary_paths, info, batch_sizes):
num_shapes = len(info.shapes)
num_binaries = len(binary_paths)
niter = max(num_shapes, num_binaries)
processed_frames = 0
tensors = []
for i in range(niter):
shape_id = i % num_shapes
dtype = get_dtype(info.element_type)
shape = list(info.shapes[shape_id])
binaries = np.ndarray(shape=shape, dtype=dtype)
if info.layout.has_name('N'):
shape[info.layout.get_index_by_name('N')] = 1
binary_index = processed_frames
current_batch_size = batch_sizes[shape_id]
for b in range(current_batch_size):
binary_index %= num_binaries
binary_filename = binary_paths[binary_index]
logger.info("Prepare binary file " + binary_filename)
binary_file_size = os.path.getsize(binary_filename)
blob_size = dtype().nbytes * int(np.prod(shape))
if blob_size != binary_file_size:
raise Exception(
f"File {binary_filename} contains {binary_file_size} bytes but network expects {blob_size}"
)
binaries[b] = np.reshape(np.fromfile(binary_filename, dtype),
shape)
binary_index += 1
processed_frames += current_batch_size
tensors.append(Tensor(binaries))
return tensors
def __call__(self, *input_values: NumericData) -> List[NumericData]:
"""Run computation on input values and return result."""
# Input validation
if len(input_values) < len(self.parameters):
raise UserInputError(
"Expected %s params, received not enough %s values.",
len(self.parameters), len(input_values))
param_names = [param.friendly_name for param in self.parameters]
input_shapes = [get_shape(input_value) for input_value in input_values]
if self.network_cache.get(str(input_shapes)) is None:
function = self.function
if self.function.is_dynamic():
function.reshape(
dict(
zip(param_names,
[PartialShape(i) for i in input_shapes])))
self.network_cache[str(input_shapes)] = function
else:
function = self.network_cache[str(input_shapes)]
executable_network = self.runtime.backend.compile_model(
function, self.runtime.backend_name)
for parameter, input in zip(self.parameters, input_values):
parameter_shape = parameter.get_output_partial_shape(0)
input_shape = PartialShape([]) if isinstance(
input, (int, float)) else PartialShape(input.shape)
if not parameter_shape.compatible(input_shape):
raise UserInputError(
"Provided tensor's shape: %s does not match the expected: %s.",
input_shape,
parameter_shape,
)
is_bfloat16 = any(
parameter.get_output_element_type(0) == Type.bf16
for parameter in self.parameters)
if is_bfloat16:
input_values = self.convert_to_tensors(input_values)
request = executable_network.create_infer_request()
result_buffers = request.infer(dict(zip(param_names, input_values)))
# # Note: other methods to get result_buffers from request
# # First call infer with no return value:
# request.infer(dict(zip(param_names, input_values)))
# # Now use any of following options:
# result_buffers = [request.get_tensor(n).data for n in request.outputs]
# result_buffers = [request.get_output_tensor(i).data for i in range(len(request.outputs))]
# result_buffers = [t.data for t in request.output_tensors]
# # Since OV overwrite result data type we have to convert results to the original one.
original_dtypes = [
get_dtype(result.get_output_element_type(0))
for result in self.results
]
converted_buffers = self.convert_buffers(result_buffers,
original_dtypes)
return converted_buffers
def get_image_info_tensors(image_sizes, layer):
im_infos = []
for shape, image_size in zip(layer.shapes, image_sizes):
im_info = np.ndarray(shape, dtype=get_dtype(layer.element_type))
for b in range(shape[0]):
for i in range(shape[1]):
im_info[b][i] = image_size if i in [0, 1] else 1
im_infos.append(Tensor(im_info))
return im_infos
def read_binary_file(bin_file, model_input):
log.info(f"Prepare binary file {str(bin_file)}")
binary_file_size = os.path.getsize(bin_file)
tensor_size = model_input.tensor.size
if tensor_size != binary_file_size:
raise Exception(
f"File {bin_file} contains {binary_file_size} bytes but model expects {tensor_size}"
)
return np.reshape(
np.fromfile(bin_file, get_dtype(model_input.element_type)),
list(model_input.shape))
def normalize_inputs(py_dict: dict, py_types: dict) -> dict:
"""Normalize a dictionary of inputs to Tensors."""
for k, val in py_dict.items():
if not isinstance(k, (str, int)):
raise TypeError(
"Incompatible key type for tensor named: {}".format(k))
try:
ov_type = py_types[k]
except KeyError:
raise KeyError("Port for tensor named {} was not found!".format(k))
py_dict[k] = (val if isinstance(val, Tensor) else Tensor(
np.array(val, get_dtype(ov_type))))
return py_dict
def fill_tensors_with_random(layer):
dtype = get_dtype(layer.element_type)
rand_min, rand_max = (0, 1) if dtype == np.bool else (np.iinfo(np.uint8).min, np.iinfo(np.uint8).max)
# np.random.uniform excludes high: add 1 to have it generated
if np.dtype(dtype).kind in ['i', 'u', 'b']:
rand_max += 1
rs = np.random.RandomState(np.random.MT19937(np.random.SeedSequence(0)))
input_tensors = []
for shape in layer.shapes:
if shape:
input_tensors.append(Tensor(rs.uniform(rand_min, rand_max, list(shape)).astype(dtype)))
else:
input_tensors.append(Tensor(rs.uniform(rand_min, rand_max)))
return input_tensors
def test_query_state_write_buffer(device, input_shape, data_type, mode):
core = Core()
if device == "CPU":
if core.get_property(device,
"FULL_DEVICE_NAME") == "arm_compute::NEON":
pytest.skip("Can't run on ARM plugin")
from openvino.runtime import Tensor
from openvino.runtime.utils.types import get_dtype
model = create_model_with_memory(input_shape, data_type)
compiled = core.compile_model(model=model, device_name=device)
request = compiled.create_infer_request()
mem_states = request.query_state()
mem_state = mem_states[0]
assert mem_state.name == "var_id_667"
# todo: Uncomment after fix 45611,
# CPU plugin returns outputs and memory state in FP32 in case of FP16 original precision
# assert mem_state.state.tensor_desc.precision == data_type
for i in range(1, 10):
if mode == "set_init_memory_state":
# create initial value
const_init = 5
init_array = np.full(input_shape,
const_init,
dtype=get_dtype(mem_state.state.element_type))
tensor = Tensor(init_array)
mem_state.state = tensor
res = request.infer({0: np.full(input_shape, 1, dtype=data_type)})
expected_res = np.full(input_shape,
1 + const_init,
dtype=data_type)
elif mode == "reset_memory_state":
# reset initial state of ReadValue to zero
mem_state.reset()
res = request.infer({0: np.full(input_shape, 1, dtype=data_type)})
# always ones
expected_res = np.full(input_shape, 1, dtype=data_type)
else:
res = request.infer({0: np.full(input_shape, 1, dtype=data_type)})
expected_res = np.full(input_shape, i, dtype=data_type)
assert np.allclose(res[list(res)[0]], expected_res, atol=1e-6), \
"Expected values: {} \n Actual values: {} \n".format(expected_res, res)
def convert_dict_items(inputs: dict, py_types: dict) -> dict:
"""Helper function converting dictionary items to Tensors."""
# Create new temporary dictionary.
# new_inputs will be used to transfer data to inference calls,
# ensuring that original inputs are not overwritten with Tensors.
new_inputs = {}
for k, val in inputs.items():
if not isinstance(k, (str, int, ConstOutput)):
raise TypeError("Incompatible key type for tensor: {}".format(k))
try:
ov_type = py_types[k]
except KeyError:
raise KeyError("Port for tensor {} was not found!".format(k))
# Convert numpy arrays or copy Tensors
new_inputs[k] = (val if isinstance(val, Tensor) else Tensor(
np.array(val, get_dtype(ov_type), copy=False)))
return new_inputs
def normalize_inputs(inputs: Union[dict, list], py_types: dict) -> dict:
"""Normalize a dictionary of inputs to Tensors."""
if isinstance(inputs, list):
inputs = {index: input for index, input in enumerate(inputs)}
for k, val in inputs.items():
if not isinstance(k, (str, int)):
raise TypeError("Incompatible key type for tensor named: {}".format(k))
try:
ov_type = py_types[k]
except KeyError:
raise KeyError("Port for tensor named {} was not found!".format(k))
inputs[k] = (
val
if isinstance(val, Tensor)
else Tensor(np.array(val, get_dtype(ov_type)))
)
return inputs
def test_simple_computation_on_ndarrays(dtype):
runtime = get_runtime()
shape = [2, 2]
parameter_a = ops.parameter(shape, dtype=dtype, name="A")
parameter_b = ops.parameter(shape, dtype=dtype, name="B")
parameter_c = ops.parameter(shape, dtype=dtype, name="C")
model = (parameter_a + parameter_b) * parameter_c
computation = runtime.computation(model, parameter_a, parameter_b,
parameter_c)
np_dtype = get_dtype(dtype) if isinstance(dtype, Type) else dtype
value_a = np.array([[1, 2], [3, 4]], dtype=np_dtype)
value_b = np.array([[5, 6], [7, 8]], dtype=np_dtype)
value_c = np.array([[2, 3], [4, 5]], dtype=np_dtype)
result = computation(value_a, value_b, value_c)
assert np.allclose(result, np.array([[12, 24], [40, 60]], dtype=np_dtype))
value_a = np.array([[9, 10], [11, 12]], dtype=np_dtype)
value_b = np.array([[13, 14], [15, 16]], dtype=np_dtype)
value_c = np.array([[5, 4], [3, 2]], dtype=np_dtype)
result = computation(value_a, value_b, value_c)
assert np.allclose(result, np.array([[110, 96], [78, 56]], dtype=np_dtype))
def get_image_tensors(image_paths, info, batch_sizes):
processed_frames = 0
widthes = info.widthes if info.is_dynamic else [info.width]
heights = info.heights if info.is_dynamic else [info.height]
tensors = []
process_with_original_shapes = False
num_shapes = len(info.shapes)
if num_shapes == 0:
process_with_original_shapes = True
num_images = len(image_paths)
niter = max(num_shapes, num_images)
for i in range(niter):
shape = list(info.shapes[i % num_shapes]) if num_shapes else []
dtype = get_dtype(info.element_type)
images = np.ndarray(shape=shape, dtype=dtype)
image_index = processed_frames
current_batch_size = 1 if process_with_original_shapes else batch_sizes[
i % num_shapes]
for b in range(current_batch_size):
image_index %= num_images
image_filename = image_paths[image_index]
logger.info(f'Prepare image {image_filename}')
image = cv2.imread(image_filename)
if process_with_original_shapes:
logger.info(
f'Image will be processed with original shape - {image.shape[:-1]}'
)
elif info.layout.has_name('H') and info.layout.has_name('W'):
new_im_size = (widthes[i % num_shapes],
heights[i % num_shapes])
if image.shape[:-1] != new_im_size:
logger.warning(
f"Image is resized from ({image.shape[:-1]}) to ({new_im_size})"
)
image = cv2.resize(image, new_im_size)
if info.scale.size or info.mean.size:
blue, green, red = cv2.split(image)
if info.mean.size:
blue = np.subtract(blue, info.mean[0])
green = np.subtract(green, info.mean[1])
red = np.subtract(red, info.mean[2])
if info.scale.size:
blue = np.divide(blue, info.scale[0])
green = np.divide(green, info.scale[1])
red = np.divide(red, info.scale[2])
image = cv2.merge([blue, green, red])
if str(info.layout) in ['[N,C,H,W]', '[C,H,W]']:
image = image.transpose((2, 0, 1))
if process_with_original_shapes:
if len(info.partial_shape) == 4:
image = np.expand_dims(image, 0)
p_shape = PartialShape(image.shape)
if info.partial_shape.compatible(p_shape):
info.data_shapes.append(p_shape.to_shape())
else:
raise Exception(
f"Data shape '{str(p_shape)}' provided for input '{info.name}' "
f"is not compatible with partial shape '{str(info.partial_shape)}' for this input."
)
tensors.append(Tensor(image.astype(dtype)))
else:
try:
images[b] = image
except ValueError:
raise Exception(
f"Image shape {image.shape} is not compatible with input shape {shape}! "
f"Make sure -i parameter is valid.")
image_index += 1
processed_frames += current_batch_size
if not process_with_original_shapes:
tensors.append(Tensor(images))
return tensors
def dump_element_type(ov_type: Type):
try:
return str(get_dtype(ov_type))
except:
return 'None'
def get_input_layer_dtype(self, model, node_name):
from openvino.runtime.utils.types import get_dtype # pylint: disable=E0401
for input in model.inputs:
if node_name == input.get_any_name():
return get_dtype(input.get_element_type())
