def __init__(self, config_entry, model_name='', delayed_model_loading=False, preprocessor=None, postpone_inputs_configuration=False): super().__init__(config_entry, model_name=model_name) self._set_variable = False self.ie_config = self.config.get('ie_config') self.ie_core = ie.IECore(xml_config_file=str( self.ie_config)) if self.ie_config is not None else ie.IECore() self._delayed_model_loading = delayed_model_loading dlsdk_launcher_config = DLSDKLauncherConfigValidator( 'DLSDK_Launcher', fields=self.parameters(), delayed_model_loading=delayed_model_loading, ) dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core) device = self.config['device'].split('.') self._device = '.'.join( (device[0].upper(), device[1])) if len(device) > 1 else device[0].upper() self.dynamic_shapes_policy = self.get_value_from_config( '_undefined_shapes_resolving_policy') self._set_variable = False self._async_mode = False self._prepare_ie() self._delayed_model_loading = delayed_model_loading self._postpone_input_configuration = postpone_inputs_configuration self._preprocess_info = {} self._preprocess_steps = [] self.disable_resize_to_input = False self._do_reshape = False self._use_set_blob = False self._output_layouts = {} self._output_precisions = {} self.dyn_input_layers = [] self._partial_shapes = {} self.is_dynamic = False self.preprocessor = preprocessor if not delayed_model_loading: self._model, self._weights = automatic_model_search( self._model_name, self.get_value_from_config('model'), self.get_value_from_config('weights'), self.get_value_from_config('_model_type')) self.load_network(log=True, preprocessing=preprocessor) self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') and self.network is not None else: self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') self._target_layout_mapping = {} self._lstm_inputs = None if '_list_lstm_inputs' in self.config: self._configure_lstm_inputs() self.reset_memory_state = self.get_value_from_config( 'reset_memory_state')
def __init__(self, config_entry, model_name='', delayed_model_loading=False, preprocessor=None, postpone_inputs_configuration=False): super().__init__(config_entry, model_name=model_name) self._set_variable = False self.ie_config = self.config.get('ie_config') self.ie_core = ie.IECore(xml_config_file=str( self.ie_config)) if self.ie_config is not None else ie.IECore() self._delayed_model_loading = delayed_model_loading dlsdk_launcher_config = DLSDKLauncherConfigValidator( 'DLSDK_Launcher', fields=self.parameters(), delayed_model_loading=delayed_model_loading, ) dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core) device = self.config['device'].split('.') self._device = '.'.join( (device[0].upper(), device[1])) if len(device) > 1 else device[0].upper() self._set_variable = False self._async_mode = False self._prepare_bitstream_firmware(self.config) self._prepare_ie() self._delayed_model_loading = delayed_model_loading self._postpone_input_configuration = postpone_inputs_configuration self._preprocess_info = {} self._preprocess_steps = [] self.disable_resize_to_input = False self._do_reshape = False self._use_set_blob = False self._output_layouts = dict() self.preprocessor = preprocessor if not delayed_model_loading: if dlsdk_launcher_config.need_conversion: self._model, self._weights = DLSDKLauncher.convert_model( self.config, dlsdk_launcher_config.framework) else: self._model, self._weights = self.automatic_model_search() self.load_network(log=True, preprocessing=preprocessor) self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') and self.network is not None else: self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') self._target_layout_mapping = {} self._lstm_inputs = None if '_list_lstm_inputs' in self.config: self._configure_lstm_inputs()
def get_model_input_shape(xml, weights): core = ie.IECore() net = core.read_network(model=xml, weights=weights) assert (len(net.input_info) == 1) i0 = [k for k in net.input_info.keys()][0] exec_net = core.load_network(network=net, device_name="CPU") return exec_net.requests[0].input_blobs[i0].buffer.shape
def __init__(self, config_entry, model_name='', delayed_model_loading=False): super().__init__(config_entry, model_name) self._set_variable = False self.ie_config = self.config.get('ie_config') self.ie_core = ie.IECore(xml_config_file=str( self.ie_config)) if self.ie_config is not None else ie.IECore() self._delayed_model_loading = delayed_model_loading dlsdk_launcher_config = DLSDKLauncherConfigValidator( 'DLSDK_Launcher', fields=self.parameters(), delayed_model_loading=delayed_model_loading, ) dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core) device = self.config['device'].split('.') self._device = '.'.join( (device[0].upper(), device[1])) if len(device) > 1 else device[0].upper() self._set_variable = False self._async_mode = False self._prepare_bitstream_firmware(self.config) self._prepare_ie() self._delayed_model_loading = delayed_model_loading if not delayed_model_loading: if dlsdk_launcher_config.need_conversion: self._model, self._weights = DLSDKLauncher.convert_model( self.config, dlsdk_launcher_config.framework) else: self._model, self._weights = self.automatic_model_search() self.load_network(log=True) self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') and self.network is not None else: self.allow_reshape_input = self.get_value_from_config( 'allow_reshape_input') self._do_reshape = False
def openvino_infer(xml, weights, input_array, return_all=False): core = ie.IECore() net = core.read_network(model=xml, weights=weights) assert (len(net.input_info) == 1) i0 = [k for k in net.input_info.keys()][0] o0 = [k for k in net.outputs.keys()][0] exec_net = core.load_network(network=net, device_name="CPU") exec_net.requests[0].input_blobs[i0].buffer[:] = input_array exec_net.requests[0].infer() if return_all: return [ exec_net.requests[0].output_blobs[_].buffer for _ in net.outputs.keys() ] else: return exec_net.requests[0].output_blobs[o0].buffer
def openvino_activations(xml, weights, input_array, extension=None): core = ie.IECore() net = core.read_network(model=xml, weights=weights) assert (len(net.input_info) == 1) i0 = [k for k in net.input_info.keys()][0] o0 = [k for k in net.outputs.keys()][0] layers = [layer for layer in net.layers] for layer in layers: if net.layers[layer].type != "Const": net.add_outputs(layer) exec_net = core.load_network(network=net, device_name="CPU") exec_net.requests[0].input_blobs[i0].buffer[:] = input_array exec_net.requests[0].infer() activations = {} for key in net.outputs.keys(): activations[key] = exec_net.requests[0].output_blobs[key].buffer return activations
def openvino_infer(model_file, image): import openvino.inference_engine as ie weights = model_file.replace('.xml', '.bin') core = ie.IECore() net = core.read_network(model=model_file, weights=weights) assert (len(net.input_info) == 1) i0 = [k for k in net.input_info.keys()][0] outputs = [k for k in net.outputs.keys()] exec_net = core.load_network(network=net, device_name="CPU") input_size = exec_net.requests[0].input_blobs[i0].buffer.shape[-1] img = cv2.imread(image) if img.shape != (input_size, input_size, 3): img = cv2.resize(img, (input_size, input_size)) input_array = img.swapaxes(1, 2).swapaxes(0, 1).astype(np.float32) input_array = np.expand_dims(input_array, axis=0) exec_net.requests[0].input_blobs[i0].buffer[:] = input_array exec_net.requests[0].infer() return [ exec_net.requests[0].output_blobs[o].buffer.flatten() for o in outputs ]
def __init__(self, weights='yolov5s.pt', device=None, dnn=False, data=None): # Usage: # PyTorch: weights = *.pt # TorchScript: *.torchscript # CoreML: *.mlmodel # OpenVINO: *.xml # TensorFlow: *_saved_model # TensorFlow: *.pb # TensorFlow Lite: *.tflite # TensorFlow Edge TPU: *_edgetpu.tflite # ONNX Runtime: *.onnx # OpenCV DNN: *.onnx with dnn=True # TensorRT: *.engine from models.experimental import attempt_download, attempt_load # scoped to avoid circular import super().__init__() w = str(weights[0] if isinstance(weights, list) else weights) suffix = Path(w).suffix.lower() suffixes = [ '.pt', '.torchscript', '.onnx', '.engine', '.tflite', '.pb', '', '.mlmodel', '.xml' ] check_suffix(w, suffixes) # check weights have acceptable suffix pt, jit, onnx, engine, tflite, pb, saved_model, coreml, xml = ( suffix == x for x in suffixes) # backends stride, names = 64, [f'class{i}' for i in range(1000)] # assign defaults w = attempt_download(w) # download if not local if data: # data.yaml path (optional) with open(data, errors='ignore') as f: names = yaml.safe_load(f)['names'] # class names if pt: # PyTorch model = attempt_load(weights if isinstance(weights, list) else w, map_location=device) stride = max(int(model.stride.max()), 32) # model stride names = model.module.names if hasattr( model, 'module') else model.names # get class names self.model = model # explicitly assign for to(), cpu(), cuda(), half() elif jit: # TorchScript LOGGER.info(f'Loading {w} for TorchScript inference...') extra_files = {'config.txt': ''} # model metadata model = torch.jit.load(w, _extra_files=extra_files) if extra_files['config.txt']: d = json.loads(extra_files['config.txt']) # extra_files dict stride, names = int(d['stride']), d['names'] elif dnn: # ONNX OpenCV DNN LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...') check_requirements(('opencv-python>=4.5.4', )) net = cv2.dnn.readNetFromONNX(w) elif onnx: # ONNX Runtime LOGGER.info(f'Loading {w} for ONNX Runtime inference...') cuda = torch.cuda.is_available() check_requirements( ('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime')) import onnxruntime providers = ['CUDAExecutionProvider', 'CPUExecutionProvider' ] if cuda else ['CPUExecutionProvider'] session = onnxruntime.InferenceSession(w, providers=providers) elif xml: # OpenVINO LOGGER.info(f'Loading {w} for OpenVINO inference...') check_requirements( ('openvino-dev', ) ) # requires openvino-dev: https://pypi.org/project/openvino-dev/ import openvino.inference_engine as ie core = ie.IECore() network = core.read_network( model=w, weights=Path(w).with_suffix('.bin')) # *.xml, *.bin paths executable_network = core.load_network(network, device_name='CPU', num_requests=1) elif engine: # TensorRT LOGGER.info(f'Loading {w} for TensorRT inference...') import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0 Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr')) logger = trt.Logger(trt.Logger.INFO) with open(w, 'rb') as f, trt.Runtime(logger) as runtime: model = runtime.deserialize_cuda_engine(f.read()) bindings = OrderedDict() for index in range(model.num_bindings): name = model.get_binding_name(index) dtype = trt.nptype(model.get_binding_dtype(index)) shape = tuple(model.get_binding_shape(index)) data = torch.from_numpy(np.empty( shape, dtype=np.dtype(dtype))).to(device) bindings[name] = Binding(name, dtype, shape, data, int(data.data_ptr())) binding_addrs = OrderedDict( (n, d.ptr) for n, d in bindings.items()) context = model.create_execution_context() batch_size = bindings['images'].shape[0] elif coreml: # CoreML LOGGER.info(f'Loading {w} for CoreML inference...') import coremltools as ct model = ct.models.MLModel(w) else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU) if saved_model: # SavedModel LOGGER.info( f'Loading {w} for TensorFlow SavedModel inference...') import tensorflow as tf model = tf.keras.models.load_model(w) elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt LOGGER.info( f'Loading {w} for TensorFlow GraphDef inference...') import tensorflow as tf def wrap_frozen_graph(gd, inputs, outputs): x = tf.compat.v1.wrap_function( lambda: tf.compat.v1.import_graph_def(gd, name=""), []) # wrapped return x.prune( tf.nest.map_structure(x.graph.as_graph_element, inputs), tf.nest.map_structure(x.graph.as_graph_element, outputs)) graph_def = tf.Graph().as_graph_def() graph_def.ParseFromString(open(w, 'rb').read()) frozen_func = wrap_frozen_graph(gd=graph_def, inputs="x:0", outputs="Identity:0") elif tflite: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python try: import tflite_runtime.interpreter as tfl # prefer tflite_runtime if installed except ImportError: import tensorflow.lite as tfl if 'edgetpu' in w.lower( ): # Edge TPU https://coral.ai/software/#edgetpu-runtime LOGGER.info( f'Loading {w} for TensorFlow Lite Edge TPU inference...' ) delegate = { 'Linux': 'libedgetpu.so.1', 'Darwin': 'libedgetpu.1.dylib', 'Windows': 'edgetpu.dll' }[platform.system()] interpreter = tfl.Interpreter( model_path=w, experimental_delegates=[tfl.load_delegate(delegate)]) else: # Lite LOGGER.info( f'Loading {w} for TensorFlow Lite inference...') interpreter = tfl.Interpreter( model_path=w) # load TFLite model interpreter.allocate_tensors() # allocate input_details = interpreter.get_input_details() # inputs output_details = interpreter.get_output_details() # outputs self.__dict__.update(locals()) # assign all variables to self
# -*- coding: utf-8 -*- # Copyright (C) 2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 import openvino.inference_engine as ie print('OpenVINO version:', ie.get_version()) print('Available devices: ', ie.IECore().available_devices)
# Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # #! [ie:create_core] import numpy as np import openvino.inference_engine as ie core = ie.IECore() #! [ie:create_core] #! [ie:read_model] network = core.read_network("model.xml") #! [ie:read_model] #! [ie:compile_model] # Load network to the device and create infer requests exec_network = core.load_network(network, "CPU", num_requests=4) #! [ie:compile_model] #! [ie:create_infer_request] # Done in the previous step #! [ie:create_infer_request] #! [ie:get_input_tensor] infer_request = exec_network.requests[0] # Get input blobs mapped to input layers names input_blobs = infer_request.input_blobs data = input_blobs["data1"].buffer # Original I64 precision was converted to I32 assert data.dtype == np.int32 # Fill the first blob ...