def get_binary_tensors(binary_paths, info, batch_sizes): num_shapes = len(info.shapes) num_binaries = len(binary_paths) niter = max(num_shapes, num_binaries) processed_frames = 0 tensors = [] for i in range(niter): shape_id = i % num_shapes dtype = get_dtype(info.element_type) shape = list(info.shapes[shape_id]) binaries = np.ndarray(shape=shape, dtype=dtype) if info.layout.has_name('N'): shape[info.layout.get_index_by_name('N')] = 1 binary_index = processed_frames current_batch_size = batch_sizes[shape_id] for b in range(current_batch_size): binary_index %= num_binaries binary_filename = binary_paths[binary_index] logger.info("Prepare binary file " + binary_filename) binary_file_size = os.path.getsize(binary_filename) blob_size = dtype().nbytes * int(np.prod(shape)) if blob_size != binary_file_size: raise Exception( f"File {binary_filename} contains {binary_file_size} bytes but network expects {blob_size}" ) binaries[b] = np.reshape(np.fromfile(binary_filename, dtype), shape) binary_index += 1 processed_frames += current_batch_size tensors.append(Tensor(binaries)) return tensors
def __call__(self, *input_values: NumericData) -> List[NumericData]: """Run computation on input values and return result.""" # Input validation if len(input_values) < len(self.parameters): raise UserInputError( "Expected %s params, received not enough %s values.", len(self.parameters), len(input_values)) param_names = [param.friendly_name for param in self.parameters] input_shapes = [get_shape(input_value) for input_value in input_values] if self.network_cache.get(str(input_shapes)) is None: function = self.function if self.function.is_dynamic(): function.reshape( dict( zip(param_names, [PartialShape(i) for i in input_shapes]))) self.network_cache[str(input_shapes)] = function else: function = self.network_cache[str(input_shapes)] executable_network = self.runtime.backend.compile_model( function, self.runtime.backend_name) for parameter, input in zip(self.parameters, input_values): parameter_shape = parameter.get_output_partial_shape(0) input_shape = PartialShape([]) if isinstance( input, (int, float)) else PartialShape(input.shape) if not parameter_shape.compatible(input_shape): raise UserInputError( "Provided tensor's shape: %s does not match the expected: %s.", input_shape, parameter_shape, ) is_bfloat16 = any( parameter.get_output_element_type(0) == Type.bf16 for parameter in self.parameters) if is_bfloat16: input_values = self.convert_to_tensors(input_values) request = executable_network.create_infer_request() result_buffers = request.infer(dict(zip(param_names, input_values))) # # Note: other methods to get result_buffers from request # # First call infer with no return value: # request.infer(dict(zip(param_names, input_values))) # # Now use any of following options: # result_buffers = [request.get_tensor(n).data for n in request.outputs] # result_buffers = [request.get_output_tensor(i).data for i in range(len(request.outputs))] # result_buffers = [t.data for t in request.output_tensors] # # Since OV overwrite result data type we have to convert results to the original one. original_dtypes = [ get_dtype(result.get_output_element_type(0)) for result in self.results ] converted_buffers = self.convert_buffers(result_buffers, original_dtypes) return converted_buffers
def get_image_info_tensors(image_sizes, layer): im_infos = [] for shape, image_size in zip(layer.shapes, image_sizes): im_info = np.ndarray(shape, dtype=get_dtype(layer.element_type)) for b in range(shape[0]): for i in range(shape[1]): im_info[b][i] = image_size if i in [0, 1] else 1 im_infos.append(Tensor(im_info)) return im_infos
def read_binary_file(bin_file, model_input): log.info(f"Prepare binary file {str(bin_file)}") binary_file_size = os.path.getsize(bin_file) tensor_size = model_input.tensor.size if tensor_size != binary_file_size: raise Exception( f"File {bin_file} contains {binary_file_size} bytes but model expects {tensor_size}" ) return np.reshape( np.fromfile(bin_file, get_dtype(model_input.element_type)), list(model_input.shape))
def normalize_inputs(py_dict: dict, py_types: dict) -> dict: """Normalize a dictionary of inputs to Tensors.""" for k, val in py_dict.items(): if not isinstance(k, (str, int)): raise TypeError( "Incompatible key type for tensor named: {}".format(k)) try: ov_type = py_types[k] except KeyError: raise KeyError("Port for tensor named {} was not found!".format(k)) py_dict[k] = (val if isinstance(val, Tensor) else Tensor( np.array(val, get_dtype(ov_type)))) return py_dict
def fill_tensors_with_random(layer): dtype = get_dtype(layer.element_type) rand_min, rand_max = (0, 1) if dtype == np.bool else (np.iinfo(np.uint8).min, np.iinfo(np.uint8).max) # np.random.uniform excludes high: add 1 to have it generated if np.dtype(dtype).kind in ['i', 'u', 'b']: rand_max += 1 rs = np.random.RandomState(np.random.MT19937(np.random.SeedSequence(0))) input_tensors = [] for shape in layer.shapes: if shape: input_tensors.append(Tensor(rs.uniform(rand_min, rand_max, list(shape)).astype(dtype))) else: input_tensors.append(Tensor(rs.uniform(rand_min, rand_max))) return input_tensors
def test_query_state_write_buffer(device, input_shape, data_type, mode): core = Core() if device == "CPU": if core.get_property(device, "FULL_DEVICE_NAME") == "arm_compute::NEON": pytest.skip("Can't run on ARM plugin") from openvino.runtime import Tensor from openvino.runtime.utils.types import get_dtype model = create_model_with_memory(input_shape, data_type) compiled = core.compile_model(model=model, device_name=device) request = compiled.create_infer_request() mem_states = request.query_state() mem_state = mem_states[0] assert mem_state.name == "var_id_667" # todo: Uncomment after fix 45611, # CPU plugin returns outputs and memory state in FP32 in case of FP16 original precision # assert mem_state.state.tensor_desc.precision == data_type for i in range(1, 10): if mode == "set_init_memory_state": # create initial value const_init = 5 init_array = np.full(input_shape, const_init, dtype=get_dtype(mem_state.state.element_type)) tensor = Tensor(init_array) mem_state.state = tensor res = request.infer({0: np.full(input_shape, 1, dtype=data_type)}) expected_res = np.full(input_shape, 1 + const_init, dtype=data_type) elif mode == "reset_memory_state": # reset initial state of ReadValue to zero mem_state.reset() res = request.infer({0: np.full(input_shape, 1, dtype=data_type)}) # always ones expected_res = np.full(input_shape, 1, dtype=data_type) else: res = request.infer({0: np.full(input_shape, 1, dtype=data_type)}) expected_res = np.full(input_shape, i, dtype=data_type) assert np.allclose(res[list(res)[0]], expected_res, atol=1e-6), \ "Expected values: {} \n Actual values: {} \n".format(expected_res, res)
def convert_dict_items(inputs: dict, py_types: dict) -> dict: """Helper function converting dictionary items to Tensors.""" # Create new temporary dictionary. # new_inputs will be used to transfer data to inference calls, # ensuring that original inputs are not overwritten with Tensors. new_inputs = {} for k, val in inputs.items(): if not isinstance(k, (str, int, ConstOutput)): raise TypeError("Incompatible key type for tensor: {}".format(k)) try: ov_type = py_types[k] except KeyError: raise KeyError("Port for tensor {} was not found!".format(k)) # Convert numpy arrays or copy Tensors new_inputs[k] = (val if isinstance(val, Tensor) else Tensor( np.array(val, get_dtype(ov_type), copy=False))) return new_inputs
def normalize_inputs(inputs: Union[dict, list], py_types: dict) -> dict: """Normalize a dictionary of inputs to Tensors.""" if isinstance(inputs, list): inputs = {index: input for index, input in enumerate(inputs)} for k, val in inputs.items(): if not isinstance(k, (str, int)): raise TypeError("Incompatible key type for tensor named: {}".format(k)) try: ov_type = py_types[k] except KeyError: raise KeyError("Port for tensor named {} was not found!".format(k)) inputs[k] = ( val if isinstance(val, Tensor) else Tensor(np.array(val, get_dtype(ov_type))) ) return inputs
def test_simple_computation_on_ndarrays(dtype): runtime = get_runtime() shape = [2, 2] parameter_a = ops.parameter(shape, dtype=dtype, name="A") parameter_b = ops.parameter(shape, dtype=dtype, name="B") parameter_c = ops.parameter(shape, dtype=dtype, name="C") model = (parameter_a + parameter_b) * parameter_c computation = runtime.computation(model, parameter_a, parameter_b, parameter_c) np_dtype = get_dtype(dtype) if isinstance(dtype, Type) else dtype value_a = np.array([[1, 2], [3, 4]], dtype=np_dtype) value_b = np.array([[5, 6], [7, 8]], dtype=np_dtype) value_c = np.array([[2, 3], [4, 5]], dtype=np_dtype) result = computation(value_a, value_b, value_c) assert np.allclose(result, np.array([[12, 24], [40, 60]], dtype=np_dtype)) value_a = np.array([[9, 10], [11, 12]], dtype=np_dtype) value_b = np.array([[13, 14], [15, 16]], dtype=np_dtype) value_c = np.array([[5, 4], [3, 2]], dtype=np_dtype) result = computation(value_a, value_b, value_c) assert np.allclose(result, np.array([[110, 96], [78, 56]], dtype=np_dtype))
def get_image_tensors(image_paths, info, batch_sizes): processed_frames = 0 widthes = info.widthes if info.is_dynamic else [info.width] heights = info.heights if info.is_dynamic else [info.height] tensors = [] process_with_original_shapes = False num_shapes = len(info.shapes) if num_shapes == 0: process_with_original_shapes = True num_images = len(image_paths) niter = max(num_shapes, num_images) for i in range(niter): shape = list(info.shapes[i % num_shapes]) if num_shapes else [] dtype = get_dtype(info.element_type) images = np.ndarray(shape=shape, dtype=dtype) image_index = processed_frames current_batch_size = 1 if process_with_original_shapes else batch_sizes[ i % num_shapes] for b in range(current_batch_size): image_index %= num_images image_filename = image_paths[image_index] logger.info(f'Prepare image {image_filename}') image = cv2.imread(image_filename) if process_with_original_shapes: logger.info( f'Image will be processed with original shape - {image.shape[:-1]}' ) elif info.layout.has_name('H') and info.layout.has_name('W'): new_im_size = (widthes[i % num_shapes], heights[i % num_shapes]) if image.shape[:-1] != new_im_size: logger.warning( f"Image is resized from ({image.shape[:-1]}) to ({new_im_size})" ) image = cv2.resize(image, new_im_size) if info.scale.size or info.mean.size: blue, green, red = cv2.split(image) if info.mean.size: blue = np.subtract(blue, info.mean[0]) green = np.subtract(green, info.mean[1]) red = np.subtract(red, info.mean[2]) if info.scale.size: blue = np.divide(blue, info.scale[0]) green = np.divide(green, info.scale[1]) red = np.divide(red, info.scale[2]) image = cv2.merge([blue, green, red]) if str(info.layout) in ['[N,C,H,W]', '[C,H,W]']: image = image.transpose((2, 0, 1)) if process_with_original_shapes: if len(info.partial_shape) == 4: image = np.expand_dims(image, 0) p_shape = PartialShape(image.shape) if info.partial_shape.compatible(p_shape): info.data_shapes.append(p_shape.to_shape()) else: raise Exception( f"Data shape '{str(p_shape)}' provided for input '{info.name}' " f"is not compatible with partial shape '{str(info.partial_shape)}' for this input." ) tensors.append(Tensor(image.astype(dtype))) else: try: images[b] = image except ValueError: raise Exception( f"Image shape {image.shape} is not compatible with input shape {shape}! " f"Make sure -i parameter is valid.") image_index += 1 processed_frames += current_batch_size if not process_with_original_shapes: tensors.append(Tensor(images)) return tensors
def dump_element_type(ov_type: Type): try: return str(get_dtype(ov_type)) except: return 'None'
def get_input_layer_dtype(self, model, node_name): from openvino.runtime.utils.types import get_dtype # pylint: disable=E0401 for input in model.inputs: if node_name == input.get_any_name(): return get_dtype(input.get_element_type())