def _create_network(self, input_shapes=None):
assert self.plugin, "create_ie_plugin should be called before _create_network"
self.network = ie.IENetwork(model=str(self._model),
weights=str(self._weights))
self.original_outputs = self.network.outputs
outputs = self.config.get('outputs')
if outputs:
def output_preprocessing(output_string):
output_tuple = string_to_tuple(output_string,
casting_type=None)
if len(output_tuple) == 1:
return output_string
return tuple([output_tuple[0], int(output_tuple[1])])
preprocessed_outputs = [
output_preprocessing(output) for output in outputs
]
self.network.add_outputs(preprocessed_outputs)
if input_shapes is not None:
self.network.reshape(input_shapes)
self._batch = self.config.get('batch', self.network.batch_size)
if self._batch != self.network.batch_size:
self._set_batch_size(self._batch)
affinity_map_path = self.config.get('affinity_map')
if affinity_map_path and self._is_hetero():
self._set_affinity(affinity_map_path)
elif affinity_map_path:
warning('affinity_map config is applicable only for HETERO device')
def create_network(self) -> ie.IENetwork:
network = ie.IENetwork(self._configuration.model, self._configuration.weights)
if len(network.outputs) == 0:
raise ValueError("no outputs")
if len(network.inputs) == 0:
raise ValueError("no inputs")
return network
def benchmark_callback(self, network_inputs_data):
latencies = list()
if self._network:
ie_network = self._network.ie_network
else:
ie_network = ie.IENetwork(self._configuration.model,
self._configuration.weights)
plugin = ie.IEPlugin(self._configuration.device)
if self._configuration.cpu_extension:
plugin.add_cpu_extension(self._configuration.cpu_extension)
exec_network = plugin.load(ie_network)
# warming up
exec_network.infer(network_inputs_data)
for i in range(self._iterations_count):
start = datetime.datetime.now()
exec_network.infer(network_inputs_data)
latencies.append((datetime.datetime.now() - start).microseconds)
self._latency = numpy.mean(latencies) / 1000000.0
del ie_network
del exec_network
del plugin
def read_network(self, model, weights):
if 'read_network' in ie.IECore.__dict__:
network = self.ie_core.read_network(model=str(model),
weights=str(weights))
else:
network = ie.IENetwork(model=str(model), weights=str(weights))
return network
def openvino_random_input(xml, weights, scale=255.):
net = ie.IENetwork(model=xml, weights=weights)
assert (len(net.inputs) == 1)
i0 = [k for k in net.inputs.keys()][0]
plugin = ie.IEPlugin(device="CPU")
exec_net = plugin.load(network=net)
input_shape = exec_net.requests[0].inputs[i0].shape
input_array = np.random.random(input_shape).astype(np.float32) * scale
return input_array
def main():
####################### Device Initialization ########################
# Plugin initialization for specified device and load extensions library if specified
plugin = ie.IEPlugin(device="MYRIAD")
#########################################################################
######################### Load Neural Network #########################
# Read in Graph file (IR)
net = ie.IENetwork(model="cnn-mnist_inference.xml",
weights="cnn-mnist_inference.bin")
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
# Load network to the plugin
exec_net = plugin.load(network=net)
del net
########################################################################
######################### Obtain Input Tensor ########################
# Obtain and preprocess input tensor (image)
# Read and pre-process input image maybe we don't need to show these details
image_for_inference = cv2.imread("./1.JPG")
image_for_inference = cv2.cvtColor(image_for_inference, cv2.COLOR_BGR2GRAY)
image_for_inference = cv2.resize(image_for_inference, (28, 28))
image_for_inference = image_for_inference.astype(numpy.float32)
image_for_inference[:] = 1 - ((image_for_inference[:]) * (1.0 / 255.0))
image_for_inference = image_for_inference.reshape(-1, 28, 28)
# ########################################################################
# ########################## Start Inference ##########################
# # Start synchronous inference and get inference result
start_time = time.time()
req_handle = exec_net.start_async(0,
inputs={input_blob: image_for_inference})
# # ########################################################################
# res = exec_net.infer({input_blob:image_for_inference})
# # ######################## Get Inference Result #########################
status = req_handle.wait()
res = req_handle.outputs[out_blob]
# Do something with the results... (like print top 5)
print("FPS:", 1 / (time.time() - start_time))
print((1 - res[0]).argsort()[:1])
# ############################### Clean Up ############################
del exec_net
del plugin
def _create_network(self, input_shapes=None):
model_path = Path(self._model)
compiled_model = model_path.suffix == '.blob'
if compiled_model:
self.network = None
self.exec_network = self.ie_core.import_network(
str(self._model), self._device)
self.original_outputs = list(self.exec_network.outputs.keys())
first_input = next(iter(self.exec_network.inputs))
input_info = self.exec_network.inputs[first_input]
batch_pos = input_info.layout.find('N')
self._batch = input_info.shape[batch_pos] if batch_pos != -1 else 1
return
if self._weights is None:
self._weights = model_path.parent / (
model_path.name.split(model_path.suffix)[0] + '.bin')
if 'read_network' in ie.IECore.__dict__:
self.network = self.ie_core.read_network(model=str(self._model),
weights=str(
self._weights))
else:
self.network = ie.IENetwork(model=str(self._model),
weights=str(self._weights))
self.original_outputs = self.network.outputs
outputs = self.config.get('outputs')
if outputs:
def output_preprocessing(output_string):
output_tuple = string_to_tuple(output_string,
casting_type=None)
if len(output_tuple) == 1:
return output_string
return tuple([output_tuple[0], int(output_tuple[1])])
preprocessed_outputs = [
output_preprocessing(output) for output in outputs
]
self.network.add_outputs(preprocessed_outputs)
if input_shapes is not None:
self.network.reshape(input_shapes)
self._batch = self.config.get('batch', self.network.batch_size)
if self._batch != self.network.batch_size:
self._set_batch_size(self._batch)
affinity_map_path = self.config.get('affinity_map')
if affinity_map_path and self._is_hetero():
self._set_affinity(affinity_map_path)
elif affinity_map_path:
warning('affinity_map config is applicable only for HETERO device')
def __init__(self, config_entry, adapter):
super().__init__(config_entry, adapter)
def fit_to_input(data, input_layer):
shape_len = len(input_layer.shape)
if shape_len == 4:
return np.transpose(data, [0, 3, 1, 2])
if shape_len == 2:
if len(np.shape(data)) == 1:
return np.transpose([data])
return np.array(data)
dlsdk_launcher_config = DLSDKLauncherConfig('DLSDK_Launcher')
dlsdk_launcher_config.validate(self._config)
self._device = self._config['device'].upper()
self._set_variable = False
self._prepare_bitstream_firmware(self._config)
if dlsdk_launcher_config.need_conversion:
self._model, self._weights = DLSDKLauncher.convert_model(
self._config, dlsdk_launcher_config.framework)
else:
self._model = self._config['model']
self._weights = self._config['weights']
self._create_ie_plugin()
self.network = ie.IENetwork(model=str(self._model),
weights=str(self._weights))
self.original_outputs = self.network.outputs
outputs = self._config.get('outputs')
if outputs:
self.network.add_outputs(outputs)
self.input_feeder = InputFeeder(self._config.get('inputs') or [],
self.network.inputs,
prepare_input_data=fit_to_input)
self._batch = self._config.get('batch', self.network.batch_size)
if self._batch != self.network.batch_size:
self._set_batch_size(self._batch)
affinity_map_path = self._config.get('affinity_map')
if affinity_map_path and self._is_hetero():
self._set_affinity(affinity_map_path)
elif affinity_map_path:
warning('affinity_map config is applicable only for HETERO device')
self.exec_network = self.plugin.load(network=self.network)
self.allow_reshape_input = self._config.get('allow_reshape_input',
False)
def _create_network(self, input_shapes=None):
assert self.plugin, "_create_ie_plugin should be called before _create_network"
self.network = ie.IENetwork(model=str(self._model),
weights=str(self._weights))
self.original_outputs = self.network.outputs
outputs = self.config.get('outputs')
if outputs:
self.network.add_outputs(outputs)
if input_shapes is not None:
self.network.reshape(input_shapes)
self._batch = self.config.get('batch', self.network.batch_size)
if self._batch != self.network.batch_size:
self._set_batch_size(self._batch)
affinity_map_path = self.config.get('affinity_map')
if affinity_map_path and self._is_hetero():
self._set_affinity(affinity_map_path)
elif affinity_map_path:
warning('affinity_map config is applicable only for HETERO device')
def create_ie_network(model_xml, model_bin):
return ie.IENetwork(model_xml, model_bin)
def ie_network(self) -> ie.IENetwork:
if not self._ie_network:
self._ie_network = ie.IENetwork(self._model_path, self._weights_path)
return self._ie_network
def __init__(
self,
device,
face_detection_path,
facenet_path=None,
classifier=None,
bg_remove_path=None,
loaded_plugin=None,
debug=False):
self.use_classifiers = False
self.debug = debug
extensions = os.environ.get('INTEL_EXTENSIONS_PATH')
if loaded_plugin is not None:
plugin = loaded_plugin
else:
plugin = ie.IEPlugin(device=device)
if extensions and "CPU" in device:
for ext in extensions.split(':'):
print("LOAD extension from {}".format(ext))
plugin.add_cpu_extension(ext)
print('Load FACE DETECTION')
face_path = face_detection_path
weights_file = face_path[:face_path.rfind('.')] + '.bin'
net = ie.IENetwork(face_path, weights_file)
self.face_detect = nets.FaceDetect(plugin, net)
if facenet_path:
print('Load FACENET')
model_file = facenet_path
weights_file = model_file[:model_file.rfind('.')] + '.bin'
net = ie.IENetwork(model_file, weights_file)
self.facenet_input = list(net.inputs.keys())[0]
outputs = list(iter(net.outputs))
self.facenet_output = outputs[0]
self.face_net = plugin.load(net)
if classifier and len(classifier) > 0:
self.use_classifiers = bool(facenet_path)
self.classifiers = []
self.classifier_names = []
self.embedding_sizes = []
self.class_names = None
self.class_stats = None
for clfi, clf in enumerate(classifier):
# Load classifier
with open(clf, 'rb') as f:
print('Load CLASSIFIER %s' % clf)
opts = {'file': f}
if six.PY3:
opts['encoding'] = 'latin1'
(classifier, class_names, class_stats) = pickle.load(**opts)
if isinstance(classifier, svm.SVC):
embedding_size = classifier.shape_fit_[1]
clfn = "SVM classifier"
self.classifier_names.append("SVM")
elif isinstance(classifier, neighbors.KNeighborsClassifier):
embedding_size = classifier._fit_X.shape[1]
clfn = "kNN (neighbors %d) classifier" % classifier.n_neighbors
# self.classifier_names.append("kNN(%2d)" % classifier.n_neighbors)
self.classifier_names.append("kNN")
else:
# try embedding_size = 512
embedding_size = 512
clfn = type(classifier)
self.classifier_names.append("%d" % clfi)
print('Loaded %s, embedding size: %d' % (clfn, embedding_size))
if self.class_names is None:
self.class_names = class_names
elif class_names != self.class_names:
raise RuntimeError("Different class names in classifiers")
if self.class_stats is None:
self.class_stats = class_stats
elif class_stats != self.class_stats:
raise RuntimeError("Different class stats in classifiers")
self.embedding_sizes.append(embedding_size)
self.classifiers.append(classifier)
self.bg_remove = bg_remove.get_driver(bg_remove_path)
def collect(self, statistics: dict(),
full_network_result: InferenceResult) -> list:
'''
Method get layers which can be quantized and affect on final accuracy. Separate network is created for each layer.
'''
accuracy_drop_by_layer = list()
network = ie.IENetwork(self._configuration.model,
self._configuration.weights)
# if self._configuration.batch_size:
# # need to use reshape API
# network.batch_size = self._configuration.batch_size
try:
network_info = NetworkInfo(self._configuration.model)
# 2. go over all layers which affect accuracy and create network basing on it
quantization_layers = list()
index = 1
threads = list()
for layer in network.layers.values():
if self._normalizer.is_quantization_supported(layer.type):
layer_info = network_info.get_layer(layer.name)
if (len(layer_info.outputs) == 1) and (len(
layer_info.outputs[0].layer.inputs) == 1):
quantization_layer = QuantizationLayer(index, layer)
quantization_layers.append(quantization_layer)
threads.append(
SingleLayerNetworkThread(self, statistics,
full_network_result,
network, network_info,
quantization_layer))
index += 1
it = iter(threads)
threads_num = multiprocessing.cpu_count() * 2
active_threads = list()
while True:
active_threads.clear()
for thread_num in range(threads_num):
active_thread = next(it, None)
if not active_thread:
break
active_threads.append(active_thread)
active_thread.start()
for active_thread in active_threads:
active_thread.join()
if not active_thread:
debug("all layer networks were infered")
break
debug("all layer networks before #{} were infered".format(
active_thread.quantization_layer.index))
for thread in threads:
thread.join()
accuracy_drop_by_layer.append(thread.result)
accuracy_drop_by_layer.sort(
key=lambda accuracy_drop: accuracy_drop.value, reverse=True)
return accuracy_drop_by_layer
finally:
del network
def create_network_for_layer(
self,
weights: str,
quantization_layer: ie.IENetLayer,
quantization_layer_info: Layer,
activation_layer: ie.IENetLayer):
quantization_layer_weights = None
quantization_layer_biases = None
weights_file_path_has_to_be_removed = False
try:
if weights is None:
weights_file_path_has_to_be_removed = True
weights_file_path = os.path.join(tempfile._get_default_tempdir(), "model_" + next(tempfile._get_candidate_names()) + ".bin")
with open(weights_file_path, 'wb') as bin_file:
for blob_name, blob_content in quantization_layer.weights.items():
start = bin_file.tell()
blob_content.tofile(bin_file)
end = bin_file.tell()
if blob_name == 'weights':
quantization_layer_weights = Weights(start, end - start)
if blob_name == 'biases':
quantization_layer_biases = Biases(start, end - start)
if quantization_layer_weights is None:
raise ValueError("quantization layer '{}' doesn't contain weights".format(quantization_layer.name))
else:
weights_file_path = weights
quantization_layer_weights = quantization_layer_info.weights
quantization_layer_biases = quantization_layer_info.biases
if self.is_quantization_supported(quantization_layer.type):
input_layer_info = quantization_layer_info.inputs[0].layer
layers = [
Layer(
0,
"Input",
input_layer_info.name,
quantization_layer.precision,
{},
[],
input_layer_info.outputs[0].port.dim),
Layer(
1,
quantization_layer.type,
quantization_layer.name,
quantization_layer.precision,
quantization_layer.params,
quantization_layer_info.inputs[0].port.dim,
quantization_layer_info.outputs[0].port.dim,
quantization_layer_weights,
quantization_layer_biases)
]
if activation_layer:
activation_layer_info = quantization_layer_info.outputs[0].layer
reference_output_layer_name = activation_layer_info.name
outputs = activation_layer_info.outputs
output_layer_outputs_dim = \
outputs[0].port.dim if outputs else activation_layer_info.inputs[0].port.dim
layers.append(Layer(
len(layers),
activation_layer.type,
activation_layer.name,
activation_layer.precision,
activation_layer.params,
activation_layer_info.inputs[0].port.dim,
output_layer_outputs_dim))
else:
reference_output_layer_name = quantization_layer_info.name
output_layer_outputs_dim = quantization_layer_info.outputs[0].port.dim
layers.append(Layer(
len(layers),
"Power",
quantization_layer.name + "_",
quantization_layer.precision,
{'power': 1.0, 'scale': 1.0, 'shift': 0.0},
output_layer_outputs_dim,
output_layer_outputs_dim))
builder = NetworkBuilder().sequential(layers)
else:
raise ValueError("unsupported layer type '{}'".format(quantization_layer.type))
# filling weights and biases
temporary_file = tempfile.NamedTemporaryFile(delete=False)
try:
builder_str = str(builder)
network_content = str.encode(builder_str)
temporary_file.write(network_content)
temporary_file.close()
network_for_layer_model = temporary_file.name
network_for_layer = ie.IENetwork(network_for_layer_model, weights_file_path)
network_for_layer.add_outputs([quantization_layer.name + "_"])
finally:
if os.path.exists(temporary_file.name):
temporary_file.close()
os.remove(temporary_file.name)
finally:
if weights_file_path_has_to_be_removed and os.path.exists(weights_file_path):
os.remove(weights_file_path)
return network_for_layer, reference_output_layer_name
def _create_single_layer_networks(self, stat):
'''
Method get layers which can be quantized and affect on final accuracy. Separate network is created for each layer.
'''
network = ie.IENetwork(self._configuration.model,
self._configuration.weights)
# if self._configuration.batch_size:
# # need to use reshape API
# network.batch_size = self._configuration.batch_size
try:
network_info = NetworkInfo(self._configuration.model)
# CVS-14302: IE Network INT8 Normalizer: scale factor calculation is incorrect
# for layer_name, layer_statistics in stat.items():
# layer_info = network_info.get_layer(layer_name)
# if layer_info.type == 'Convolution' and \
# layer_info.outputs and \
# layer_info.outputs[0].layer.type == 'ReLU' and \
# layer_info.outputs[0].layer.outputs[0] and \
# len(layer_statistics.max_outputs) > len(stat[layer_info.outputs[0].layer.name].max_outputs):
# relu_max_outputs = stat[layer_info.outputs[0].layer.name].max_outputs
# relu_min_outputs = stat[layer_info.outputs[0].layer.name].min_outputs
# while len(layer_statistics.max_outputs) > len(relu_max_outputs):
# relu_max_outputs.append(relu_max_outputs[-1])
# relu_min_outputs.append(relu_min_outputs[-1])
single_layer_networks = dict()
layer_index = 1
for layer_to_clone in network.layers.values():
layer_to_clone_info = network_info.get_layer(
layer_to_clone.name)
if layer_to_clone.name in self._ignore_layer_names or \
not self._normalizer.is_quantization_supported(layer_to_clone.type) or \
len(layer_to_clone_info.outputs) != 1 or \
len(layer_to_clone_info.outputs[0].layer.inputs != 1):
continue
activation_layer = network.layers[
layer_to_clone_info.outputs[0].layer.name] if (
len(layer_to_clone_info.outputs) == 1
and self._normalizer.is_quantization_fusing_supported(
layer_to_clone_info,
layer_to_clone_info.outputs[0].layer)) else None
if activation_layer:
debug("create network #{} for layer {} ({}) -> {} ({})".
format(layer_index, layer_to_clone.name,
layer_to_clone.type, activation_layer.name,
activation_layer.type))
else:
debug("create network #{} for layer {} ({})".format(
layer_index, layer_to_clone.name, layer_to_clone.type))
layer_network, reference_output_layer_name = self._normalizer.create_network_for_layer(
self._configuration.weights, layer_to_clone,
layer_to_clone_info, activation_layer)
Network.reshape(layer_network, self._configuration.batch_size)
network_stats = {}
# TODO: initialize only neccessary statistic
for layer_name, node_statistic in stat.items():
network_stats[layer_name] = ie.LayerStats(
min=tuple(node_statistic.min_outputs),
max=tuple(node_statistic.max_outputs))
layer_network.stats.update(network_stats)
params = layer_network.layers[layer_to_clone.name].params
params[
"quantization_level"] = 'I8' if self._configuration.precision == 'INT8' else self._configuration.precision
layer_network.layers[layer_to_clone.name].params = params
exec_network = self._plugin.load(
network=layer_network,
config={"EXCLUSIVE_ASYNC_REQUESTS": "YES"})
if len(layer_network.inputs) != 1:
raise ValueError("created network has several inputs")
network_input_layer_name = next(
iter(layer_network.inputs.keys()))
single_layer_networks[
layer_to_clone.name] = SingleLayerNetwork(
network=layer_network,
exec_network=exec_network,
input_layer_name=network_input_layer_name,
layer_name=layer_to_clone.name,
output_layer_name=layer_to_clone.name + "_",
reference_output_layer_name=reference_output_layer_name
)
layer_index += 1
return single_layer_networks
finally:
del network
