def test_infer_new_request_tensor_numpy_copy(device): ie = Core() func = ie.read_model(model=test_net_xml, weights=test_net_bin) img = read_image() tensor = Tensor(img) exec_net = ie.compile_model(func, device) res_tensor = exec_net.infer_new_request({"data": tensor}) res_img = exec_net.infer_new_request({"data": tensor}) assert np.argmax(res_tensor[list(res_tensor)[0]]) == 2 assert np.argmax(res_tensor[list(res_tensor)[0]]) == np.argmax( res_img[list(res_img)[0]])
def test_infer_numpy_model_from_buffer(device): core = Core() with open(test_net_bin, "rb") as f: bin = f.read() with open(test_net_xml, "rb") as f: xml = f.read() func = core.read_model(model=xml, weights=bin) img = read_image() exec_net = core.compile_model(func, device) res = exec_net.infer_new_request({"data": img}) assert np.argmax(res[list(res)[0]]) == 2
def __init__(self, device='CPU'): """ Constructor :param device: specify the target device to infer on; CPU, GPU, FPGA, HDDL, MYRIAD or HETERO: is acceptable :param extension: path to the extension library with custom layers """ self._tmp_dir = create_tmp_dir() self.device = device self.model = None self.infer_request = None self._ie = Core()
def test_read_model_as_path(): core = Core() func = core.read_model(model=Path(test_net_xml), weights=Path(test_net_bin)) assert isinstance(func, Model) func = core.read_model(model=test_net_xml, weights=Path(test_net_bin)) assert isinstance(func, Model) func = core.read_model(model=Path(test_net_xml)) assert isinstance(func, Model)
def test_input_update_rt_info(device): core = Core() func = core.read_model(model=test_net_xml, weights=test_net_bin) exec_net = core.compile_model(func, device) input = exec_net.output(0) input_node = input.get_node().inputs()[0] rt = input_node.get_rt_info() rt["test12345"] = "test" for k, v in input_node.get_rt_info().items(): assert k == "test12345" assert isinstance(v, Parameter)
def test_net_from_buffer_valid(): core = Core() with open(test_net_bin, "rb") as f: bin = f.read() with open(model_path()[0], "rb") as f: xml = f.read() func = core.read_model(model=xml, weights=bin) ref_func = core.read_model(model=test_net_xml, weights=test_net_bin) assert func.get_parameters() == ref_func.get_parameters() assert func.get_results() == ref_func.get_results() assert func.get_ordered_ops() == ref_func.get_ordered_ops()
def part5(): #! [part5] core = Core() model = core.read_model(model_path) core.set_property(device_name="CPU", properties={}) core.set_property(device_name="MYRIAD", properties={}) compiled_model = core.compile_model(model=model) compiled_model = core.compile_model(model=model, device_name="AUTO")
def available_devices_2(): #! [available_devices_2] match_list = [] all_devices = "MULTI:" dev_match_str = "MYRIAD" core = Core() model = core.read_model(model_path) for d in core.available_devices: if dev_match_str in d: match_list.append(d) all_devices += ",".join(match_list) compiled_model = core.compile_model(model=model, device_name=all_devices)
def test_infer_tensor_numpy_shared_memory(device): ie = Core() func = ie.read_model(model=test_net_xml, weights=test_net_bin) img = read_image() img = np.ascontiguousarray(img) tensor = Tensor(img, shared_memory=True) exec_net = ie.compile_model(func, device) res_tensor = exec_net.infer_new_request({"data": tensor}) res_img = exec_net.infer_new_request({"data": tensor}) assert np.argmax(res_tensor[list(res_tensor)[0]]) == 2 assert np.argmax(res_tensor[list(res_tensor)[0]]) == np.argmax( res_img[list(res_img)[0]])
def __init__(self, model_path, device): log.info('OpenVINO Runtime') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading model {}'.format(model_path)) self.model = core.read_model(model_path) self.input_tensor_name = "Placeholder" compiled_model = core.compile_model(self.model, device) self.output_tensor = compiled_model.outputs[0] self.infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(model_path, device))
def test_const_output_get_names(device): core = Core() func = core.read_model(model=test_net_xml, weights=test_net_bin) exec_net = core.compile_model(func, device) input_name = "data" node = exec_net.input(input_name) expected_names = set() expected_names.add(input_name) assert node.get_names() == expected_names assert node.names == expected_names assert node.get_any_name() == input_name assert node.any_name == input_name
def test_export_import(): core = Core() model = core.read_model(model=test_net_xml, weights=test_net_bin) compiled = core.compile_model(model, "CPU") user_stream = compiled.export_model() new_compiled = core.import_model(user_stream, "CPU") img = read_image() res = new_compiled.infer_new_request({"data": img}) assert np.argmax(res[new_compiled.outputs[0]]) == 2
def test_infer_new_request_return_type(device): ie = Core() func = ie.read_model(model=test_net_xml, weights=test_net_bin) img = read_image() exec_net = ie.compile_model(func, device) res = exec_net.infer_new_request({"data": img}) arr = res[list(res)[0]][0] assert isinstance(arr, np.ndarray) assert arr.itemsize == 4 assert arr.shape == (10,) assert arr.dtype == "float32" assert arr.nbytes == 40
def __init__(self, config_entry, model_name='', delayed_model_loading=False, preprocessor=None, postpone_inputs_configuration=False): super().__init__(config_entry, model_name=model_name) self._set_variable = False self.ie_config = self.config.get('ie_config') self.ie_core = Core() if self.ie_config: ov_set_config(self.ie_core, self.ie_config) self._delayed_model_loading = delayed_model_loading dlsdk_launcher_config = DLSDKLauncherConfigValidator( 'OpenVINO_Launcher', fields=self.parameters(), delayed_model_loading=delayed_model_loading, ) dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core) device = self.config['device'].split('.') self._device = '.'.join((device[0].upper(), device[1])) if len(device) > 1 else device[0].upper() self.dynamic_shapes_policy = self.get_value_from_config('_undefined_shapes_resolving_policy') self._set_variable = False self._async_mode = False self._prepare_ie() self._delayed_model_loading = delayed_model_loading self._postpone_input_configuration = postpone_inputs_configuration self._preprocess_info = {} self._preprocess_steps = [] self.disable_resize_to_input = False self._do_reshape = False self._output_layouts = {} self._output_precisions = {} self.dyn_input_layers = [] self._partial_shapes = {} self.is_dynamic = False self.preprocessor = preprocessor self.infer_request = None self._num_requests = None if not delayed_model_loading: self._model, self._weights = automatic_model_search( self._model_name, self.get_value_from_config('model'), self.get_value_from_config('weights'), self.get_value_from_config('_model_type') ) self.load_network(log=not postpone_inputs_configuration, preprocessing=preprocessor) self.allow_reshape_input = self.get_value_from_config('allow_reshape_input') and self.network is not None if not postpone_inputs_configuration: self.try_to_set_default_layout() else: self.allow_reshape_input = self.get_value_from_config('allow_reshape_input') self._target_layout_mapping = {} self._lstm_inputs = None if '_list_lstm_inputs' in self.config: self._configure_lstm_inputs() self.reset_memory_state = self.get_value_from_config('reset_memory_state')
def test_set_tensors(device): core = Core() func = core.read_model(test_net_xml, test_net_bin) exec_net = core.compile_model(func, device) data1 = read_image() tensor1 = Tensor(data1) data2 = np.ones(shape=(1, 10), dtype=np.float32) tensor2 = Tensor(data2) data3 = np.ones(shape=(1, 3, 32, 32), dtype=np.float32) tensor3 = Tensor(data3) data4 = np.zeros(shape=(1, 10), dtype=np.float32) tensor4 = Tensor(data4) request = exec_net.create_infer_request() request.set_tensors({"data": tensor1, "fc_out": tensor2}) t1 = request.get_tensor("data") t2 = request.get_tensor("fc_out") assert np.allclose(tensor1.data, t1.data, atol=1e-2, rtol=1e-2) assert np.allclose(tensor2.data, t2.data, atol=1e-2, rtol=1e-2) request.set_output_tensors({0: tensor2}) output_node = exec_net.outputs[0] t3 = request.get_tensor(output_node) assert np.allclose(tensor2.data, t3.data, atol=1e-2, rtol=1e-2) request.set_input_tensors({0: tensor1}) output_node = exec_net.inputs[0] t4 = request.get_tensor(output_node) assert np.allclose(tensor1.data, t4.data, atol=1e-2, rtol=1e-2) output_node = exec_net.inputs[0] request.set_tensor(output_node, tensor3) t5 = request.get_tensor(output_node) assert np.allclose(tensor3.data, t5.data, atol=1e-2, rtol=1e-2) request.set_input_tensor(tensor3) t6 = request.get_tensor(request.inputs[0]) assert np.allclose(tensor3.data, t6.data, atol=1e-2, rtol=1e-2) request.set_input_tensor(0, tensor1) t7 = request.get_tensor(request.inputs[0]) assert np.allclose(tensor1.data, t7.data, atol=1e-2, rtol=1e-2) request.set_output_tensor(tensor2) t8 = request.get_tensor(request.outputs[0]) assert np.allclose(tensor2.data, t8.data, atol=1e-2, rtol=1e-2) request.set_output_tensor(0, tensor4) t9 = request.get_tensor(request.outputs[0]) assert np.allclose(tensor4.data, t9.data, atol=1e-2, rtol=1e-2)
def test_import_onnx_with_external_data(): model_path = os.path.join(os.path.dirname(__file__), "models/external_data.onnx") ie = Core() func = ie.read_model(model=model_path) dtype = np.float32 value_a = np.array([1.0, 3.0, 5.0], dtype=dtype) value_b = np.array([3.0, 5.0, 1.0], dtype=dtype) # third input [5.0, 1.0, 3.0] read from external file runtime = get_runtime() computation = runtime.computation(func) result = computation(value_a, value_b) assert np.allclose(result, np.array([3.0, 3.0, 3.0], dtype=dtype))
def create_simple_request_and_inputs(device): input_shape = [2, 2] param_a = ops.parameter(input_shape, np.float32) param_b = ops.parameter(input_shape, np.float32) model = Model(ops.add(param_a, param_b), [param_a, param_b]) core = Core() compiled = core.compile_model(model, device) request = compiled.create_infer_request() arr_1 = np.array([[1, 2], [3, 4]], dtype=np.float32) arr_2 = np.array([[3, 4], [1, 2]], dtype=np.float32) return request, arr_1, arr_2
def test_infer_queue_is_ready(device): core = Core() param = ops.parameter([10]) model = Model(ops.relu(param), [param]) compiled = core.compile_model(model, device) infer_queue = AsyncInferQueue(compiled, 1) def callback(request, _): time.sleep(0.001) infer_queue.set_callback(callback) assert infer_queue.is_ready() infer_queue.start_async() assert not infer_queue.is_ready() infer_queue.wait_all()
def test_inputs_outputs_property(device): num_inputs = 10 input_shape = [1] params = [ops.parameter(input_shape, np.uint8) for _ in range(num_inputs)] model = Model(ops.split(ops.concat(params, 0), 0, num_inputs), params) core = Core() compiled = core.compile_model(model, device) request = compiled.create_infer_request() data = [np.atleast_1d(i) for i in range(num_inputs)] results = request.infer(data).values() for result, output_tensor in zip(results, request.outputs): assert np.array_equal(result, output_tensor.data) for input_data, input_tensor in zip(data, request.inputs): assert np.array_equal(input_data, input_tensor.data)
def test_export_import(): core = Core() func = core.read_model(model=test_net_xml, weights=test_net_bin) exec_net = core.compile_model(func, "CPU") exported_net_file = "exported_model.bin" exec_net.export_model(network_model=exported_net_file) assert os.path.exists(exported_net_file) exec_net = core.import_network(exported_net_file, "CPU") os.remove(exported_net_file) img = read_image() res = exec_net.infer({"data": img}) assert np.argmax(res["fc_out"][0]) == 3 del exec_net del core
class InferenceEngine: def __init__(self, net_model_xml_path, device, stride): self.device = device self.stride = stride log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) self.core = Core() log.info('Reading model {}'.format(net_model_xml_path)) self.model = self.core.read_model(net_model_xml_path) required_output_keys = {'features', 'heatmaps', 'pafs'} for output_tensor_name in required_output_keys: try: self.model.output(output_tensor_name) except RuntimeError: raise RuntimeError( "The demo supports only topologies with the following output keys: {}" .format(', '.join(required_output_keys))) self.input_tensor_name = self.model.inputs[0].get_any_name() compiled_model = self.core.compile_model(self.model, self.device) self.infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(net_model_xml_path, self.device)) def infer(self, img): img = img[0:img.shape[0] - (img.shape[0] % self.stride), 0:img.shape[1] - (img.shape[1] % self.stride)] n, c, h, w = self.model.inputs[0].shape if h != img.shape[0] or w != img.shape[1]: self.model.reshape({ self.input_tensor_name: PartialShape([n, c, img.shape[0], img.shape[1]]) }) compiled_model = self.core.compile_model(self.model, self.device) self.infer_request = compiled_model.create_infer_request() img = np.transpose(img, (2, 0, 1))[None, ] self.infer_request.infer({self.input_tensor_name: img}) inference_result = { name: self.infer_request.get_tensor(name).data[:] for name in {'features', 'heatmaps', 'pafs'} } inference_result = (inference_result['features'][0], inference_result['heatmaps'][0], inference_result['pafs'][0]) return inference_result
def __init__(self, model_path, device): log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading model {}'.format(model_path)) self.model = core.read_model(model_path) self.input_tensor_name = self.model.inputs[0].get_any_name() self.input_size = self.model.input(self.input_tensor_name).shape self.nchw_layout = self.input_size[1] == 3 compiled_model = core.compile_model(self.model, device) self.output_tensor = compiled_model.outputs[0] self.infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(model_path, device))
def __init__(self, model_xml, model_bin, device, output_name): log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading model {}'.format(model_xml)) self.model = core.read_model(model_xml, model_bin) compiled_model = core.compile_model(self.model, args.device) self.infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(model_xml, device)) self.input_tensor_name = "tokens" self.output_tensor_name = output_name self.model.output( self.output_tensor_name) # ensure a tensor with the name exists
def test_import_onnx_function(): model_path = os.path.join(os.path.dirname(__file__), "models/add_abc.onnx") ie = Core() func = ie.read_model(model=model_path) dtype = np.float32 value_a = np.array([1.0], dtype=dtype) value_b = np.array([2.0], dtype=dtype) value_c = np.array([3.0], dtype=dtype) runtime = get_runtime() computation = runtime.computation(func) result = computation(value_a, value_b, value_c) assert np.allclose(result, np.array([6], dtype=dtype))
def abs_model_with_data(device, ov_type, numpy_dtype): input_shape = [1, 4] param = ops.parameter(input_shape, ov_type) model = Model(ops.abs(param), [param]) core = Core() compiled_model = core.compile_model(model, device) request = compiled_model.create_infer_request() tensor1 = Tensor(ov_type, input_shape) tensor1.data[:] = np.array([6, -7, -8, 9]) array1 = np.array([[-1, 2, 5, -3]]).astype(numpy_dtype) return request, tensor1, array1
def test_infer_dynamic_model(device): core = Core() param = ops.parameter(PartialShape([-1, -1])) model = Model(ops.relu(param), [param]) compiled = core.compile_model(model, device) assert compiled.input().partial_shape.is_dynamic request = compiled.create_infer_request() shape1 = [1, 28] request.infer([np.random.normal(size=shape1)]) assert request.get_input_tensor().shape == Shape(shape1) shape2 = [1, 32] request.infer([np.random.normal(size=shape2)]) assert request.get_input_tensor().shape == Shape(shape2)
def __init__(self, model_path, input_name, output_name, quantiles): device = "CPU" log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading model {}'.format(model_path)) model = core.read_model(model_path) compiled_model = core.compile_model(model, device) self.infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(model_path, device)) self.input_tensor_name = input_name self.output_tensor_name = output_name self.quantiles = quantiles model.output( self.output_tensor_name) # ensure a tensor with the name exists
def test_get_version(device): ie = Core() version = ie.get_versions(device) assert isinstance(version, dict), "Returned version must be a dictionary" assert device in version, "{} plugin version wasn't found in versions" assert hasattr(version[device], "major"), "Returned version has no field 'major'" assert hasattr(version[device], "minor"), "Returned version has no field 'minor'" assert hasattr( version[device], "description"), "Returned version has no field 'description'" assert hasattr( version[device], "build_number"), "Returned version has no field 'build_number'"
def test_direct_infer(device): core = Core() with open(test_net_bin, "rb") as f: bin = f.read() with open(test_net_xml, "rb") as f: xml = f.read() model = core.read_model(model=xml, weights=bin) img = read_image() tensor = Tensor(img) comp_model = core.compile_model(model, device) res = comp_model({"data": tensor}) assert np.argmax(res[comp_model.outputs[0]]) == 2 ref = comp_model.infer_new_request({"data": tensor}) assert np.array_equal(ref[comp_model.outputs[0]], res[comp_model.outputs[0]])
def test_register_plugins(): ie = Core() if platform == "linux" or platform == "linux2": ie.register_plugins(plugins_xml) elif platform == "darwin": ie.register_plugins(plugins_osx_xml) elif platform == "win32": ie.register_plugins(plugins_win_xml) func = ie.read_model(model=test_net_xml, weights=test_net_bin) exec_net = ie.compile_model(func, "CUSTOM") assert isinstance(exec_net, CompiledModel), "Cannot load the network to " \ "the registered plugin with name 'CUSTOM' " \ "registered in the XML file"