def predict(self, inputs, metadata=None, **kwargs): """ Args: inputs: dictionary where keys are input layers names and values are data for them. metadata: metadata of input representations Returns: raw data from network. """ if self._lstm_inputs: return self._predict_sequential(inputs, metadata) results = [] for infer_inputs in inputs: if self._do_reshape: input_shapes = { layer_name: data.shape for layer_name, data in infer_inputs.items() } self._reshape_input(input_shapes) if self._use_set_blob: has_info = hasattr(self.exec_network, 'input_info') for key, input_data in infer_inputs.items(): if has_info: ie_input_info = OrderedDict([ (name, data.input_data) for name, data in self.exec_network.input_info.items() ]) else: ie_input_info = self.exec_network.inputs layout = self._target_layout_mapping.get( key, ie_input_info[key].layout) tensor_desc = TensorDesc(ie_input_info[key].precision, input_data.shape, layout) preprocess_info = self._preprocess_info.get(key) if preprocess_info is not None: self.exec_network.requests[0].set_blob( key, Blob(tensor_desc, input_data), preprocess_info) else: self.exec_network.requests[0].set_blob( key, Blob(tensor_desc, input_data)) result = self.exec_network.infer( infer_inputs ) if not self._use_set_blob else self.exec_network.infer() results.append(result) if metadata is not None: for meta_ in metadata: meta_['input_shape'] = self.inputs_info_for_meta() if self._output_layouts: meta_['output_layout'] = self._output_layouts self._do_reshape = False self._use_set_blob = self.disable_resize_to_input return results
def test_set_shape(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC") blob = Blob(tensor_desc) blob.set_shape([1, 4, 128, 128]) assert blob.tensor_desc.dims == [1, 4, 128, 128] assert blob.buffer.shape == (1, 4, 128, 128) array = np.ones([1, 3, 127, 127], dtype=np.float32) blob = Blob(tensor_desc, array) blob.set_shape([1, 4, 128, 128]) assert blob.tensor_desc.dims == [1, 4, 128, 128] assert blob.buffer.shape == (1, 4, 128, 128)
def predict(self, inputs, metadata=None, **kwargs): if self._lstm_inputs: return self._predict_sequential(inputs, metadata) results = [] for infer_inputs in inputs: if self._do_reshape: input_shapes = { layer_name: data.shape for layer_name, data in infer_inputs.items() } self._reshape_input(input_shapes) if self._use_set_blob: has_info = hasattr(self.exec_network, 'input_info') for key, input_data in infer_inputs.items(): if has_info: ie_input_info = OrderedDict([ (name, data.input_data) for name, data in self.exec_network.input_info.items() ]) else: ie_input_info = self.exec_network.inputs layout = self._target_layout_mapping.get( key, ie_input_info[key].layout) tensor_desc = TensorDesc(ie_input_info[key].precision, input_data.shape, layout) preprocess_info = self._preprocess_info.get(key) if preprocess_info is not None: self.exec_network.requests[0].set_blob( key, Blob(tensor_desc, input_data), preprocess_info) else: self.exec_network.requests[0].set_blob( key, Blob(tensor_desc, input_data)) result = self.exec_network.infer( infer_inputs ) if not self._use_set_blob else self.exec_network.infer() results.append(result) if self.reset_memory_state: for state in self.exec_network.requests[0].query_state(): state.reset() if metadata is not None: self._fill_meta(metadata, None if not self.dyn_input_layers else inputs[-1]) self._do_reshape = False self._use_set_blob = self.disable_resize_to_input return results
def test_cannot_set_shape_preallocated_memory(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC") array = np.ones([1, 3, 127, 127], dtype=np.float32) blob = Blob(tensor_desc, array) with pytest.raises(RuntimeError) as e: blob.set_shape([1, 4, 128, 128]) assert "Cannot call setShape for Blobs created on top of preallocated memory" in str(e.value)
def test_write_numpy_scalar_int64(): tensor_desc = TensorDesc("I64", [], "SCALAR") scalar = np.array(0, dtype=np.int64) blob = Blob(tensor_desc, scalar) scalar_to_write = np.array(1, dtype=np.int64) blob.buffer[:] = scalar_to_write assert np.array_equal(blob.buffer, np.atleast_1d(scalar_to_write))
def test_write_to_buffer(precision, numpy_precision): tensor_desc = TensorDesc(precision, [1, 3, 127, 127], "NCHW") array = np.zeros(shape=(1, 3, 127, 127), dtype=numpy_precision) blob = Blob(tensor_desc, array) ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=numpy_precision) blob.buffer[:] = ones_arr assert np.array_equal(blob.buffer, ones_arr)
def test_init_with_numpy(shape, layout): tensor_desc = TensorDesc("FP32", shape, layout) array = np.ones(shape=shape, dtype=np.float32) blob = Blob(tensor_desc, array) assert isinstance(blob.buffer, np.ndarray) assert np.shares_memory(blob.buffer, array) assert blob.tensor_desc == tensor_desc
def test_incompatible_array_and_td(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW") array = np.zeros(shape=(1, 2, 3, 4), dtype=np.float32) with pytest.raises(AttributeError) as e: Blob(tensor_desc, array) assert "Number of elements in provided numpy array 24 and " \ "required by TensorDesc 48387 are not equal" in str(e.value)
def test_write_to_buffer_fp32(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW") array = np.zeros(shape=(1, 3, 127, 127), dtype=np.float32) blob = Blob(tensor_desc, array) ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=np.float32) blob.buffer[:] = ones_arr assert np.array_equal(blob.buffer, ones_arr)
def test_write_to_buffer_uint16(): tensor_desc = TensorDesc("U16", [1, 3, 127, 127], "NCHW") array = np.zeros(shape=(1, 3, 127, 127), dtype=np.uint16) blob = Blob(tensor_desc, array) ones_arr = np.ones(shape=(1, 3, 127, 127), dtype=np.uint16) blob.buffer[:] = ones_arr assert np.array_equal(blob.buffer, ones_arr)
def test_set_mean_image(): ie_core = IECore() net = ie_core.read_network(model=test_net_xml, weights=test_net_bin) tensor_desc = TensorDesc("FP32", [0, 127, 127], "CHW") mean_image_blob = Blob(tensor_desc) preprocess_info = net.input_info["data"].preprocess_info preprocess_info.set_mean_image(mean_image_blob) assert preprocess_info.mean_variant == MeanVariant.MEAN_IMAGE
def test_set_mean_image_for_channel(): ie_core = IECore() net = ie_core.read_network(model=test_net_xml, weights=test_net_bin) tensor_desc = TensorDesc("FP32", [127, 127], "HW") mean_image_blob = Blob(tensor_desc) preprocess_info = net.input_info["data"].preprocess_info preprocess_info.init(1) preprocess_info.set_mean_image_for_channel(mean_image_blob, 0) pre_process_channel = preprocess_info[0] assert isinstance(pre_process_channel.mean_data, Blob) assert pre_process_channel.mean_data.tensor_desc.dims == [127, 127] assert preprocess_info.mean_variant == MeanVariant.MEAN_IMAGE
def test_incompatible_input_precision(): import cv2 n, c, h, w = (1, 3, 32, 32) image = cv2.imread(path_to_image) if image is None: raise FileNotFoundError("Input image not found") image = cv2.resize(image, (h, w)) / 255 image = image.transpose((2, 0, 1)) image = image.reshape((n, c, h, w)) tensor_desc = TensorDesc("FP32", [1, 3, 32, 32], "NCHW") with pytest.raises(ValueError) as e: Blob(tensor_desc, image) assert "Data type float64 of provided numpy array " \ "doesn't match to TensorDesc precision FP32" in str(e.value)
def test_query_state_write_buffer(device, input_shape, data_type, mode): ie_core = ie.IECore() if device == "CPU": if ie_core.get_metric(device, "FULL_DEVICE_NAME") == "arm_compute::NEON": pytest.skip("Can't run on ARM plugin") layout = ["C", "HW", "CHW", "NCHW"] np_data_type = {"FP32": np.float32, "FP16": np.float16, "I32": np.int32} from openvino.inference_engine import TensorDesc, Blob net = ie.IENetwork(create_function_with_memory(input_shape, np_data_type[data_type])) ie_core = ie.IECore() exec_net = ie_core.load_network(network=net, device_name=device, num_requests=1) request = exec_net.requests[0] mem_states = request.query_state() mem_state = mem_states[0] assert mem_state.name == 'var_id_667' # todo: Uncomment after fix 45611, # CPU plugin returns outputs and memory state in FP32 in case of FP16 original precision #assert mem_state.state.tensor_desc.precision == data_type for i in range(1, 10): if mode == "set_init_memory_state": # create initial value const_init = 5 init_array = np.full(input_shape, const_init, dtype=np_data_type[mem_state.state.tensor_desc.precision]) tensor_desc = TensorDesc(mem_state.state.tensor_desc.precision, input_shape, layout[len(input_shape) - 1]) blob = Blob(tensor_desc, init_array) mem_state.state = blob res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])}) expected_res = np.full(input_shape, 1 + const_init, dtype=np_data_type[data_type]) elif mode == "reset_memory_state": # reset initial state of ReadValue to zero mem_state.reset() res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])}) # always ones expected_res = np.full(input_shape, 1, dtype=np_data_type[data_type]) else: res = exec_net.infer({"input_data": np.full(input_shape, 1, dtype=np_data_type[data_type])}) expected_res = np.full(input_shape, i, dtype=np_data_type[data_type]) assert np.allclose(res['MemoryAdd'], expected_res, atol=1e-6), \ "Expected values: {} \n Actual values: {} \n".format(expected_res, res)
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info("Initializing Inference Engine") ie = IECore() version = ie.get_versions(args.device)[args.device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) log.info("Plugin version is {}".format(version_str)) # read IR model_xml = args.model model_bin = model_xml.with_suffix(".bin") log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) ie_encoder = ie.read_network(model=model_xml, weights=model_bin) # check input and output names input_shapes = { k: v.input_data.shape for k, v in ie_encoder.input_info.items() } input_names = list(ie_encoder.input_info.keys()) output_names = list(ie_encoder.outputs.keys()) assert "input" in input_names, "'input' is not presented in model" assert "output" in output_names, "'output' is not presented in model" state_inp_names = [n for n in input_names if "state" in n] state_param_num = sum(np.prod(input_shapes[n]) for n in state_inp_names) log.info("state_param_num = {} ({:.1f}Mb)".format(state_param_num, state_param_num * 4e-6)) # load model to the device log.info("Loading model to the {}".format(args.device)) ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device) sample_inp = wav_read(args.input) input_size = input_shapes["input"][1] res = None samples_out = [] samples_times = [] while sample_inp is not None and sample_inp.shape[0] > 0: if sample_inp.shape[0] > input_size: input = sample_inp[:input_size] sample_inp = sample_inp[input_size:] else: input = np.pad(sample_inp, ((0, input_size - sample_inp.shape[0]), ), mode='constant') sample_inp = None #forms input inputs = {"input": input[None, :]} #add states to input for n in state_inp_names: if res: inputs[n] = res[n.replace('inp', 'out')].buffer else: #on the first iteration fill states by zeros inputs[n] = np.zeros(input_shapes[n], dtype=np.float32) t0 = time.perf_counter() # Set inputs manually through InferRequest functionality to speedup infer_request_ptr = ie_encoder_exec.requests[0] for n, data in inputs.items(): info_ptr = ie_encoder.input_info[n] blob = Blob(info_ptr.tensor_desc, data) infer_request_ptr.set_blob(n, blob, info_ptr.preprocess_info) # infer by IE infer_request_ptr.infer() res = infer_request_ptr.output_blobs t1 = time.perf_counter() samples_times.append(t1 - t0) samples_out.append(res["output"].buffer.squeeze(0)) log.info("Sequence of length {:0.2f}s is processed by {:0.2f}s".format( sum(s.shape[0] for s in samples_out) / 16000, sum(samples_times), )) sample_out = np.concatenate(samples_out, 0) wav_write(args.output, sample_out)
def test_get_buffer(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW") array = np.ones(shape=(1, 3, 127, 127), dtype=np.float32) blob = Blob(tensor_desc, array) assert np.array_equal(blob.buffer, array)
def main(): args = build_argparser().parse_args() log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) ie = IECore() # read IR log.info("Reading model {}".format(args.model)) model_xml = args.model model_bin = model_xml.with_suffix(".bin") ie_encoder = ie.read_network(model=model_xml, weights=model_bin) # check input and output names input_shapes = { k: v.input_data.shape for k, v in ie_encoder.input_info.items() } input_names = list(ie_encoder.input_info.keys()) output_names = list(ie_encoder.outputs.keys()) assert "input" in input_names, "'input' is not presented in model" assert "output" in output_names, "'output' is not presented in model" state_inp_names = [n for n in input_names if "state" in n] state_param_num = sum(np.prod(input_shapes[n]) for n in state_inp_names) log.debug("State_param_num = {} ({:.1f}Mb)".format(state_param_num, state_param_num * 4e-6)) # load model to the device ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device) log.info('The model {} is loaded to {}'.format(args.model, args.device)) start_time = perf_counter() sample_inp = wav_read(args.input) input_size = input_shapes["input"][1] res = None samples_out = [] while sample_inp is not None and sample_inp.shape[0] > 0: if sample_inp.shape[0] > input_size: input = sample_inp[:input_size] sample_inp = sample_inp[input_size:] else: input = np.pad(sample_inp, ((0, input_size - sample_inp.shape[0]), ), mode='constant') sample_inp = None #forms input inputs = {"input": input[None, :]} #add states to input for n in state_inp_names: if res: inputs[n] = res[n.replace('inp', 'out')].buffer else: #on the first iteration fill states by zeros inputs[n] = np.zeros(input_shapes[n], dtype=np.float32) # Set inputs manually through InferRequest functionality to speedup infer_request_ptr = ie_encoder_exec.requests[0] for n, data in inputs.items(): info_ptr = ie_encoder.input_info[n] blob = Blob(info_ptr.tensor_desc, data) infer_request_ptr.set_blob(n, blob, info_ptr.preprocess_info) # infer by IE infer_request_ptr.infer() res = infer_request_ptr.output_blobs samples_out.append(res["output"].buffer.squeeze(0)) total_latency = (perf_counter() - start_time) * 1e3 log.info("Metrics report:") log.info("\tLatency: {:.1f} ms".format(total_latency)) log.info("\tSample length: {:.1f} ms".format( len(samples_out) * input_size / 16.0)) sample_out = np.concatenate(samples_out, 0) wav_write(args.output, sample_out)
def test_get_tensor_desc(): tensor_desc = TensorDesc("FP32", [1, 127, 127, 3], "NHWC") blob = Blob(tensor_desc) assert blob.tensor_desc == tensor_desc
def test_init_with_numpy(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NCHW") array = np.ones(shape=(1, 3, 127, 127), dtype=np.float32) blob = Blob(tensor_desc, array) assert isinstance(blob.buffer, np.ndarray) assert blob.tensor_desc == tensor_desc
data8uint = np.frombuffer(in_memory_file.getvalue(), np.uint8) # Convert string to an unsigned int array image = cv2.cvtColor(cv2.imdecode(data8uint, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB) if image.shape[:-1] != (h, w): image = cv2.resize(image, (w, h)) image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW image_with_n = np.expand_dims(image, 0) it = datagen.flow(image_with_n, batch_size=n) n_aug_samples = n - 1 image_batch = np.ndarray(shape=(n, c, h, w), dtype=np.float32) image_batch[0] = image for i in range(n_aug_samples): image_batch[i + 1] = next(it) image_batch = preprocess_input(image_batch, data_format="channels_first", mode="tf") image_batch = Blob(net.input_info[input_blob].tensor_desc, image_batch) image_batches.append({"name": image_name, "batch": image_batch}) num_loops = int(os.getenv("NUMBER_OF_TEST_LOOPS", "1")) def work_generator(): for i in range(num_loops): for image_batch in image_batches: yield image_batch work_queue = queue.Queue(maxsize=num_requests) wait_for_work_timeout_seconds = int(os.getenv("WAIT_FOR_WORK_TIMEOUT", "60")) def get_more_work(request_id): try: logger.debug("Request index {} getting more work from queue.".format(request_id)) return work_queue.get(block=True, timeout=wait_for_work_timeout_seconds)
def test_set_shape(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC") blob = Blob(tensor_desc) blob.set_shape([1, 4, 128, 128]) assert blob.tensor_desc.dims == [1, 4, 128, 128] assert blob.buffer.shape == (1, 4, 128, 128)
def test_init_with_tensor_desc(): tensor_desc = TensorDesc("FP32", [1, 3, 127, 127], "NHWC") blob = Blob(tensor_desc) assert isinstance(blob.buffer, np.ndarray) assert blob.tensor_desc == tensor_desc