def __init__(self,
config_entry,
model_name='',
delayed_model_loading=False,
preprocessor=None,
postpone_inputs_configuration=False):
super().__init__(config_entry, model_name=model_name)
self._set_variable = False
self.ie_config = self.config.get('ie_config')
self.ie_core = ie.IECore(xml_config_file=str(
self.ie_config)) if self.ie_config is not None else ie.IECore()
self._delayed_model_loading = delayed_model_loading
dlsdk_launcher_config = DLSDKLauncherConfigValidator(
'DLSDK_Launcher',
fields=self.parameters(),
delayed_model_loading=delayed_model_loading,
)
dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core)
device = self.config['device'].split('.')
self._device = '.'.join(
(device[0].upper(),
device[1])) if len(device) > 1 else device[0].upper()
self.dynamic_shapes_policy = self.get_value_from_config(
'_undefined_shapes_resolving_policy')
self._set_variable = False
self._async_mode = False
self._prepare_ie()
self._delayed_model_loading = delayed_model_loading
self._postpone_input_configuration = postpone_inputs_configuration
self._preprocess_info = {}
self._preprocess_steps = []
self.disable_resize_to_input = False
self._do_reshape = False
self._use_set_blob = False
self._output_layouts = {}
self._output_precisions = {}
self.dyn_input_layers = []
self._partial_shapes = {}
self.is_dynamic = False
self.preprocessor = preprocessor
if not delayed_model_loading:
self._model, self._weights = automatic_model_search(
self._model_name, self.get_value_from_config('model'),
self.get_value_from_config('weights'),
self.get_value_from_config('_model_type'))
self.load_network(log=True, preprocessing=preprocessor)
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input') and self.network is not None
else:
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input')
self._target_layout_mapping = {}
self._lstm_inputs = None
if '_list_lstm_inputs' in self.config:
self._configure_lstm_inputs()
self.reset_memory_state = self.get_value_from_config(
'reset_memory_state')
def __init__(self,
config_entry,
model_name='',
delayed_model_loading=False,
preprocessor=None,
postpone_inputs_configuration=False):
super().__init__(config_entry, model_name=model_name)
self._set_variable = False
self.ie_config = self.config.get('ie_config')
self.ie_core = ie.IECore(xml_config_file=str(
self.ie_config)) if self.ie_config is not None else ie.IECore()
self._delayed_model_loading = delayed_model_loading
dlsdk_launcher_config = DLSDKLauncherConfigValidator(
'DLSDK_Launcher',
fields=self.parameters(),
delayed_model_loading=delayed_model_loading,
)
dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core)
device = self.config['device'].split('.')
self._device = '.'.join(
(device[0].upper(),
device[1])) if len(device) > 1 else device[0].upper()
self._set_variable = False
self._async_mode = False
self._prepare_bitstream_firmware(self.config)
self._prepare_ie()
self._delayed_model_loading = delayed_model_loading
self._postpone_input_configuration = postpone_inputs_configuration
self._preprocess_info = {}
self._preprocess_steps = []
self.disable_resize_to_input = False
self._do_reshape = False
self._use_set_blob = False
self._output_layouts = dict()
self.preprocessor = preprocessor
if not delayed_model_loading:
if dlsdk_launcher_config.need_conversion:
self._model, self._weights = DLSDKLauncher.convert_model(
self.config, dlsdk_launcher_config.framework)
else:
self._model, self._weights = self.automatic_model_search()
self.load_network(log=True, preprocessing=preprocessor)
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input') and self.network is not None
else:
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input')
self._target_layout_mapping = {}
self._lstm_inputs = None
if '_list_lstm_inputs' in self.config:
self._configure_lstm_inputs()
def get_model_input_shape(xml, weights):
core = ie.IECore()
net = core.read_network(model=xml, weights=weights)
assert (len(net.input_info) == 1)
i0 = [k for k in net.input_info.keys()][0]
exec_net = core.load_network(network=net, device_name="CPU")
return exec_net.requests[0].input_blobs[i0].buffer.shape
def __init__(self,
config_entry,
model_name='',
delayed_model_loading=False):
super().__init__(config_entry, model_name)
self._set_variable = False
self.ie_config = self.config.get('ie_config')
self.ie_core = ie.IECore(xml_config_file=str(
self.ie_config)) if self.ie_config is not None else ie.IECore()
self._delayed_model_loading = delayed_model_loading
dlsdk_launcher_config = DLSDKLauncherConfigValidator(
'DLSDK_Launcher',
fields=self.parameters(),
delayed_model_loading=delayed_model_loading,
)
dlsdk_launcher_config.validate(self.config, ie_core=self.ie_core)
device = self.config['device'].split('.')
self._device = '.'.join(
(device[0].upper(),
device[1])) if len(device) > 1 else device[0].upper()
self._set_variable = False
self._async_mode = False
self._prepare_bitstream_firmware(self.config)
self._prepare_ie()
self._delayed_model_loading = delayed_model_loading
if not delayed_model_loading:
if dlsdk_launcher_config.need_conversion:
self._model, self._weights = DLSDKLauncher.convert_model(
self.config, dlsdk_launcher_config.framework)
else:
self._model, self._weights = self.automatic_model_search()
self.load_network(log=True)
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input') and self.network is not None
else:
self.allow_reshape_input = self.get_value_from_config(
'allow_reshape_input')
self._do_reshape = False
def openvino_infer(xml, weights, input_array, return_all=False):
core = ie.IECore()
net = core.read_network(model=xml, weights=weights)
assert (len(net.input_info) == 1)
i0 = [k for k in net.input_info.keys()][0]
o0 = [k for k in net.outputs.keys()][0]
exec_net = core.load_network(network=net, device_name="CPU")
exec_net.requests[0].input_blobs[i0].buffer[:] = input_array
exec_net.requests[0].infer()
if return_all:
return [
exec_net.requests[0].output_blobs[_].buffer
for _ in net.outputs.keys()
]
else:
return exec_net.requests[0].output_blobs[o0].buffer
def openvino_activations(xml, weights, input_array, extension=None):
core = ie.IECore()
net = core.read_network(model=xml, weights=weights)
assert (len(net.input_info) == 1)
i0 = [k for k in net.input_info.keys()][0]
o0 = [k for k in net.outputs.keys()][0]
layers = [layer for layer in net.layers]
for layer in layers:
if net.layers[layer].type != "Const":
net.add_outputs(layer)
exec_net = core.load_network(network=net, device_name="CPU")
exec_net.requests[0].input_blobs[i0].buffer[:] = input_array
exec_net.requests[0].infer()
activations = {}
for key in net.outputs.keys():
activations[key] = exec_net.requests[0].output_blobs[key].buffer
return activations
def openvino_infer(model_file, image):
import openvino.inference_engine as ie
weights = model_file.replace('.xml', '.bin')
core = ie.IECore()
net = core.read_network(model=model_file, weights=weights)
assert (len(net.input_info) == 1)
i0 = [k for k in net.input_info.keys()][0]
outputs = [k for k in net.outputs.keys()]
exec_net = core.load_network(network=net, device_name="CPU")
input_size = exec_net.requests[0].input_blobs[i0].buffer.shape[-1]
img = cv2.imread(image)
if img.shape != (input_size, input_size, 3):
img = cv2.resize(img, (input_size, input_size))
input_array = img.swapaxes(1, 2).swapaxes(0, 1).astype(np.float32)
input_array = np.expand_dims(input_array, axis=0)
exec_net.requests[0].input_blobs[i0].buffer[:] = input_array
exec_net.requests[0].infer()
return [
exec_net.requests[0].output_blobs[o].buffer.flatten() for o in outputs
]
def __init__(self,
weights='yolov5s.pt',
device=None,
dnn=False,
data=None):
# Usage:
# PyTorch: weights = *.pt
# TorchScript: *.torchscript
# CoreML: *.mlmodel
# OpenVINO: *.xml
# TensorFlow: *_saved_model
# TensorFlow: *.pb
# TensorFlow Lite: *.tflite
# TensorFlow Edge TPU: *_edgetpu.tflite
# ONNX Runtime: *.onnx
# OpenCV DNN: *.onnx with dnn=True
# TensorRT: *.engine
from models.experimental import attempt_download, attempt_load # scoped to avoid circular import
super().__init__()
w = str(weights[0] if isinstance(weights, list) else weights)
suffix = Path(w).suffix.lower()
suffixes = [
'.pt', '.torchscript', '.onnx', '.engine', '.tflite', '.pb', '',
'.mlmodel', '.xml'
]
check_suffix(w, suffixes) # check weights have acceptable suffix
pt, jit, onnx, engine, tflite, pb, saved_model, coreml, xml = (
suffix == x for x in suffixes) # backends
stride, names = 64, [f'class{i}'
for i in range(1000)] # assign defaults
w = attempt_download(w) # download if not local
if data: # data.yaml path (optional)
with open(data, errors='ignore') as f:
names = yaml.safe_load(f)['names'] # class names
if pt: # PyTorch
model = attempt_load(weights if isinstance(weights, list) else w,
map_location=device)
stride = max(int(model.stride.max()), 32) # model stride
names = model.module.names if hasattr(
model, 'module') else model.names # get class names
self.model = model # explicitly assign for to(), cpu(), cuda(), half()
elif jit: # TorchScript
LOGGER.info(f'Loading {w} for TorchScript inference...')
extra_files = {'config.txt': ''} # model metadata
model = torch.jit.load(w, _extra_files=extra_files)
if extra_files['config.txt']:
d = json.loads(extra_files['config.txt']) # extra_files dict
stride, names = int(d['stride']), d['names']
elif dnn: # ONNX OpenCV DNN
LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
check_requirements(('opencv-python>=4.5.4', ))
net = cv2.dnn.readNetFromONNX(w)
elif onnx: # ONNX Runtime
LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
cuda = torch.cuda.is_available()
check_requirements(
('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))
import onnxruntime
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'
] if cuda else ['CPUExecutionProvider']
session = onnxruntime.InferenceSession(w, providers=providers)
elif xml: # OpenVINO
LOGGER.info(f'Loading {w} for OpenVINO inference...')
check_requirements(
('openvino-dev', )
) # requires openvino-dev: https://pypi.org/project/openvino-dev/
import openvino.inference_engine as ie
core = ie.IECore()
network = core.read_network(
model=w,
weights=Path(w).with_suffix('.bin')) # *.xml, *.bin paths
executable_network = core.load_network(network,
device_name='CPU',
num_requests=1)
elif engine: # TensorRT
LOGGER.info(f'Loading {w} for TensorRT inference...')
import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download
check_version(trt.__version__, '7.0.0',
hard=True) # require tensorrt>=7.0.0
Binding = namedtuple('Binding',
('name', 'dtype', 'shape', 'data', 'ptr'))
logger = trt.Logger(trt.Logger.INFO)
with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
model = runtime.deserialize_cuda_engine(f.read())
bindings = OrderedDict()
for index in range(model.num_bindings):
name = model.get_binding_name(index)
dtype = trt.nptype(model.get_binding_dtype(index))
shape = tuple(model.get_binding_shape(index))
data = torch.from_numpy(np.empty(
shape, dtype=np.dtype(dtype))).to(device)
bindings[name] = Binding(name, dtype, shape, data,
int(data.data_ptr()))
binding_addrs = OrderedDict(
(n, d.ptr) for n, d in bindings.items())
context = model.create_execution_context()
batch_size = bindings['images'].shape[0]
elif coreml: # CoreML
LOGGER.info(f'Loading {w} for CoreML inference...')
import coremltools as ct
model = ct.models.MLModel(w)
else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)
if saved_model: # SavedModel
LOGGER.info(
f'Loading {w} for TensorFlow SavedModel inference...')
import tensorflow as tf
model = tf.keras.models.load_model(w)
elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
LOGGER.info(
f'Loading {w} for TensorFlow GraphDef inference...')
import tensorflow as tf
def wrap_frozen_graph(gd, inputs, outputs):
x = tf.compat.v1.wrap_function(
lambda: tf.compat.v1.import_graph_def(gd, name=""),
[]) # wrapped
return x.prune(
tf.nest.map_structure(x.graph.as_graph_element,
inputs),
tf.nest.map_structure(x.graph.as_graph_element,
outputs))
graph_def = tf.Graph().as_graph_def()
graph_def.ParseFromString(open(w, 'rb').read())
frozen_func = wrap_frozen_graph(gd=graph_def,
inputs="x:0",
outputs="Identity:0")
elif tflite: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
try:
import tflite_runtime.interpreter as tfl # prefer tflite_runtime if installed
except ImportError:
import tensorflow.lite as tfl
if 'edgetpu' in w.lower(
): # Edge TPU https://coral.ai/software/#edgetpu-runtime
LOGGER.info(
f'Loading {w} for TensorFlow Lite Edge TPU inference...'
)
delegate = {
'Linux': 'libedgetpu.so.1',
'Darwin': 'libedgetpu.1.dylib',
'Windows': 'edgetpu.dll'
}[platform.system()]
interpreter = tfl.Interpreter(
model_path=w,
experimental_delegates=[tfl.load_delegate(delegate)])
else: # Lite
LOGGER.info(
f'Loading {w} for TensorFlow Lite inference...')
interpreter = tfl.Interpreter(
model_path=w) # load TFLite model
interpreter.allocate_tensors() # allocate
input_details = interpreter.get_input_details() # inputs
output_details = interpreter.get_output_details() # outputs
self.__dict__.update(locals()) # assign all variables to self
# -*- coding: utf-8 -*-
# Copyright (C) 2021 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
import openvino.inference_engine as ie
print('OpenVINO version:', ie.get_version())
print('Available devices: ', ie.IECore().available_devices)
# Copyright (C) 2022 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
#! [ie:create_core]
import numpy as np
import openvino.inference_engine as ie
core = ie.IECore()
#! [ie:create_core]
#! [ie:read_model]
network = core.read_network("model.xml")
#! [ie:read_model]
#! [ie:compile_model]
# Load network to the device and create infer requests
exec_network = core.load_network(network, "CPU", num_requests=4)
#! [ie:compile_model]
#! [ie:create_infer_request]
# Done in the previous step
#! [ie:create_infer_request]
#! [ie:get_input_tensor]
infer_request = exec_network.requests[0]
# Get input blobs mapped to input layers names
input_blobs = infer_request.input_blobs
data = input_blobs["data1"].buffer
# Original I64 precision was converted to I32
assert data.dtype == np.int32
# Fill the first blob ...
