def _create_network(self, input_shapes=None): assert self.plugin, "create_ie_plugin should be called before _create_network" self.network = ie.IENetwork(model=str(self._model), weights=str(self._weights)) self.original_outputs = self.network.outputs outputs = self.config.get('outputs') if outputs: def output_preprocessing(output_string): output_tuple = string_to_tuple(output_string, casting_type=None) if len(output_tuple) == 1: return output_string return tuple([output_tuple[0], int(output_tuple[1])]) preprocessed_outputs = [ output_preprocessing(output) for output in outputs ] self.network.add_outputs(preprocessed_outputs) if input_shapes is not None: self.network.reshape(input_shapes) self._batch = self.config.get('batch', self.network.batch_size) if self._batch != self.network.batch_size: self._set_batch_size(self._batch) affinity_map_path = self.config.get('affinity_map') if affinity_map_path and self._is_hetero(): self._set_affinity(affinity_map_path) elif affinity_map_path: warning('affinity_map config is applicable only for HETERO device')
def create_network(self) -> ie.IENetwork: network = ie.IENetwork(self._configuration.model, self._configuration.weights) if len(network.outputs) == 0: raise ValueError("no outputs") if len(network.inputs) == 0: raise ValueError("no inputs") return network
def benchmark_callback(self, network_inputs_data): latencies = list() if self._network: ie_network = self._network.ie_network else: ie_network = ie.IENetwork(self._configuration.model, self._configuration.weights) plugin = ie.IEPlugin(self._configuration.device) if self._configuration.cpu_extension: plugin.add_cpu_extension(self._configuration.cpu_extension) exec_network = plugin.load(ie_network) # warming up exec_network.infer(network_inputs_data) for i in range(self._iterations_count): start = datetime.datetime.now() exec_network.infer(network_inputs_data) latencies.append((datetime.datetime.now() - start).microseconds) self._latency = numpy.mean(latencies) / 1000000.0 del ie_network del exec_network del plugin
def read_network(self, model, weights): if 'read_network' in ie.IECore.__dict__: network = self.ie_core.read_network(model=str(model), weights=str(weights)) else: network = ie.IENetwork(model=str(model), weights=str(weights)) return network
def openvino_random_input(xml, weights, scale=255.): net = ie.IENetwork(model=xml, weights=weights) assert (len(net.inputs) == 1) i0 = [k for k in net.inputs.keys()][0] plugin = ie.IEPlugin(device="CPU") exec_net = plugin.load(network=net) input_shape = exec_net.requests[0].inputs[i0].shape input_array = np.random.random(input_shape).astype(np.float32) * scale return input_array
def main(): ####################### Device Initialization ######################## # Plugin initialization for specified device and load extensions library if specified plugin = ie.IEPlugin(device="MYRIAD") ######################################################################### ######################### Load Neural Network ######################### # Read in Graph file (IR) net = ie.IENetwork(model="cnn-mnist_inference.xml", weights="cnn-mnist_inference.bin") input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) # Load network to the plugin exec_net = plugin.load(network=net) del net ######################################################################## ######################### Obtain Input Tensor ######################## # Obtain and preprocess input tensor (image) # Read and pre-process input image maybe we don't need to show these details image_for_inference = cv2.imread("./1.JPG") image_for_inference = cv2.cvtColor(image_for_inference, cv2.COLOR_BGR2GRAY) image_for_inference = cv2.resize(image_for_inference, (28, 28)) image_for_inference = image_for_inference.astype(numpy.float32) image_for_inference[:] = 1 - ((image_for_inference[:]) * (1.0 / 255.0)) image_for_inference = image_for_inference.reshape(-1, 28, 28) # ######################################################################## # ########################## Start Inference ########################## # # Start synchronous inference and get inference result start_time = time.time() req_handle = exec_net.start_async(0, inputs={input_blob: image_for_inference}) # # ######################################################################## # res = exec_net.infer({input_blob:image_for_inference}) # # ######################## Get Inference Result ######################### status = req_handle.wait() res = req_handle.outputs[out_blob] # Do something with the results... (like print top 5) print("FPS:", 1 / (time.time() - start_time)) print((1 - res[0]).argsort()[:1]) # ############################### Clean Up ############################ del exec_net del plugin
def _create_network(self, input_shapes=None): model_path = Path(self._model) compiled_model = model_path.suffix == '.blob' if compiled_model: self.network = None self.exec_network = self.ie_core.import_network( str(self._model), self._device) self.original_outputs = list(self.exec_network.outputs.keys()) first_input = next(iter(self.exec_network.inputs)) input_info = self.exec_network.inputs[first_input] batch_pos = input_info.layout.find('N') self._batch = input_info.shape[batch_pos] if batch_pos != -1 else 1 return if self._weights is None: self._weights = model_path.parent / ( model_path.name.split(model_path.suffix)[0] + '.bin') if 'read_network' in ie.IECore.__dict__: self.network = self.ie_core.read_network(model=str(self._model), weights=str( self._weights)) else: self.network = ie.IENetwork(model=str(self._model), weights=str(self._weights)) self.original_outputs = self.network.outputs outputs = self.config.get('outputs') if outputs: def output_preprocessing(output_string): output_tuple = string_to_tuple(output_string, casting_type=None) if len(output_tuple) == 1: return output_string return tuple([output_tuple[0], int(output_tuple[1])]) preprocessed_outputs = [ output_preprocessing(output) for output in outputs ] self.network.add_outputs(preprocessed_outputs) if input_shapes is not None: self.network.reshape(input_shapes) self._batch = self.config.get('batch', self.network.batch_size) if self._batch != self.network.batch_size: self._set_batch_size(self._batch) affinity_map_path = self.config.get('affinity_map') if affinity_map_path and self._is_hetero(): self._set_affinity(affinity_map_path) elif affinity_map_path: warning('affinity_map config is applicable only for HETERO device')
def __init__(self, config_entry, adapter): super().__init__(config_entry, adapter) def fit_to_input(data, input_layer): shape_len = len(input_layer.shape) if shape_len == 4: return np.transpose(data, [0, 3, 1, 2]) if shape_len == 2: if len(np.shape(data)) == 1: return np.transpose([data]) return np.array(data) dlsdk_launcher_config = DLSDKLauncherConfig('DLSDK_Launcher') dlsdk_launcher_config.validate(self._config) self._device = self._config['device'].upper() self._set_variable = False self._prepare_bitstream_firmware(self._config) if dlsdk_launcher_config.need_conversion: self._model, self._weights = DLSDKLauncher.convert_model( self._config, dlsdk_launcher_config.framework) else: self._model = self._config['model'] self._weights = self._config['weights'] self._create_ie_plugin() self.network = ie.IENetwork(model=str(self._model), weights=str(self._weights)) self.original_outputs = self.network.outputs outputs = self._config.get('outputs') if outputs: self.network.add_outputs(outputs) self.input_feeder = InputFeeder(self._config.get('inputs') or [], self.network.inputs, prepare_input_data=fit_to_input) self._batch = self._config.get('batch', self.network.batch_size) if self._batch != self.network.batch_size: self._set_batch_size(self._batch) affinity_map_path = self._config.get('affinity_map') if affinity_map_path and self._is_hetero(): self._set_affinity(affinity_map_path) elif affinity_map_path: warning('affinity_map config is applicable only for HETERO device') self.exec_network = self.plugin.load(network=self.network) self.allow_reshape_input = self._config.get('allow_reshape_input', False)
def _create_network(self, input_shapes=None): assert self.plugin, "_create_ie_plugin should be called before _create_network" self.network = ie.IENetwork(model=str(self._model), weights=str(self._weights)) self.original_outputs = self.network.outputs outputs = self.config.get('outputs') if outputs: self.network.add_outputs(outputs) if input_shapes is not None: self.network.reshape(input_shapes) self._batch = self.config.get('batch', self.network.batch_size) if self._batch != self.network.batch_size: self._set_batch_size(self._batch) affinity_map_path = self.config.get('affinity_map') if affinity_map_path and self._is_hetero(): self._set_affinity(affinity_map_path) elif affinity_map_path: warning('affinity_map config is applicable only for HETERO device')
def create_ie_network(model_xml, model_bin): return ie.IENetwork(model_xml, model_bin)
def ie_network(self) -> ie.IENetwork: if not self._ie_network: self._ie_network = ie.IENetwork(self._model_path, self._weights_path) return self._ie_network
def __init__( self, device, face_detection_path, facenet_path=None, classifier=None, bg_remove_path=None, loaded_plugin=None, debug=False): self.use_classifiers = False self.debug = debug extensions = os.environ.get('INTEL_EXTENSIONS_PATH') if loaded_plugin is not None: plugin = loaded_plugin else: plugin = ie.IEPlugin(device=device) if extensions and "CPU" in device: for ext in extensions.split(':'): print("LOAD extension from {}".format(ext)) plugin.add_cpu_extension(ext) print('Load FACE DETECTION') face_path = face_detection_path weights_file = face_path[:face_path.rfind('.')] + '.bin' net = ie.IENetwork(face_path, weights_file) self.face_detect = nets.FaceDetect(plugin, net) if facenet_path: print('Load FACENET') model_file = facenet_path weights_file = model_file[:model_file.rfind('.')] + '.bin' net = ie.IENetwork(model_file, weights_file) self.facenet_input = list(net.inputs.keys())[0] outputs = list(iter(net.outputs)) self.facenet_output = outputs[0] self.face_net = plugin.load(net) if classifier and len(classifier) > 0: self.use_classifiers = bool(facenet_path) self.classifiers = [] self.classifier_names = [] self.embedding_sizes = [] self.class_names = None self.class_stats = None for clfi, clf in enumerate(classifier): # Load classifier with open(clf, 'rb') as f: print('Load CLASSIFIER %s' % clf) opts = {'file': f} if six.PY3: opts['encoding'] = 'latin1' (classifier, class_names, class_stats) = pickle.load(**opts) if isinstance(classifier, svm.SVC): embedding_size = classifier.shape_fit_[1] clfn = "SVM classifier" self.classifier_names.append("SVM") elif isinstance(classifier, neighbors.KNeighborsClassifier): embedding_size = classifier._fit_X.shape[1] clfn = "kNN (neighbors %d) classifier" % classifier.n_neighbors # self.classifier_names.append("kNN(%2d)" % classifier.n_neighbors) self.classifier_names.append("kNN") else: # try embedding_size = 512 embedding_size = 512 clfn = type(classifier) self.classifier_names.append("%d" % clfi) print('Loaded %s, embedding size: %d' % (clfn, embedding_size)) if self.class_names is None: self.class_names = class_names elif class_names != self.class_names: raise RuntimeError("Different class names in classifiers") if self.class_stats is None: self.class_stats = class_stats elif class_stats != self.class_stats: raise RuntimeError("Different class stats in classifiers") self.embedding_sizes.append(embedding_size) self.classifiers.append(classifier) self.bg_remove = bg_remove.get_driver(bg_remove_path)
def collect(self, statistics: dict(), full_network_result: InferenceResult) -> list: ''' Method get layers which can be quantized and affect on final accuracy. Separate network is created for each layer. ''' accuracy_drop_by_layer = list() network = ie.IENetwork(self._configuration.model, self._configuration.weights) # if self._configuration.batch_size: # # need to use reshape API # network.batch_size = self._configuration.batch_size try: network_info = NetworkInfo(self._configuration.model) # 2. go over all layers which affect accuracy and create network basing on it quantization_layers = list() index = 1 threads = list() for layer in network.layers.values(): if self._normalizer.is_quantization_supported(layer.type): layer_info = network_info.get_layer(layer.name) if (len(layer_info.outputs) == 1) and (len( layer_info.outputs[0].layer.inputs) == 1): quantization_layer = QuantizationLayer(index, layer) quantization_layers.append(quantization_layer) threads.append( SingleLayerNetworkThread(self, statistics, full_network_result, network, network_info, quantization_layer)) index += 1 it = iter(threads) threads_num = multiprocessing.cpu_count() * 2 active_threads = list() while True: active_threads.clear() for thread_num in range(threads_num): active_thread = next(it, None) if not active_thread: break active_threads.append(active_thread) active_thread.start() for active_thread in active_threads: active_thread.join() if not active_thread: debug("all layer networks were infered") break debug("all layer networks before #{} were infered".format( active_thread.quantization_layer.index)) for thread in threads: thread.join() accuracy_drop_by_layer.append(thread.result) accuracy_drop_by_layer.sort( key=lambda accuracy_drop: accuracy_drop.value, reverse=True) return accuracy_drop_by_layer finally: del network
def create_network_for_layer( self, weights: str, quantization_layer: ie.IENetLayer, quantization_layer_info: Layer, activation_layer: ie.IENetLayer): quantization_layer_weights = None quantization_layer_biases = None weights_file_path_has_to_be_removed = False try: if weights is None: weights_file_path_has_to_be_removed = True weights_file_path = os.path.join(tempfile._get_default_tempdir(), "model_" + next(tempfile._get_candidate_names()) + ".bin") with open(weights_file_path, 'wb') as bin_file: for blob_name, blob_content in quantization_layer.weights.items(): start = bin_file.tell() blob_content.tofile(bin_file) end = bin_file.tell() if blob_name == 'weights': quantization_layer_weights = Weights(start, end - start) if blob_name == 'biases': quantization_layer_biases = Biases(start, end - start) if quantization_layer_weights is None: raise ValueError("quantization layer '{}' doesn't contain weights".format(quantization_layer.name)) else: weights_file_path = weights quantization_layer_weights = quantization_layer_info.weights quantization_layer_biases = quantization_layer_info.biases if self.is_quantization_supported(quantization_layer.type): input_layer_info = quantization_layer_info.inputs[0].layer layers = [ Layer( 0, "Input", input_layer_info.name, quantization_layer.precision, {}, [], input_layer_info.outputs[0].port.dim), Layer( 1, quantization_layer.type, quantization_layer.name, quantization_layer.precision, quantization_layer.params, quantization_layer_info.inputs[0].port.dim, quantization_layer_info.outputs[0].port.dim, quantization_layer_weights, quantization_layer_biases) ] if activation_layer: activation_layer_info = quantization_layer_info.outputs[0].layer reference_output_layer_name = activation_layer_info.name outputs = activation_layer_info.outputs output_layer_outputs_dim = \ outputs[0].port.dim if outputs else activation_layer_info.inputs[0].port.dim layers.append(Layer( len(layers), activation_layer.type, activation_layer.name, activation_layer.precision, activation_layer.params, activation_layer_info.inputs[0].port.dim, output_layer_outputs_dim)) else: reference_output_layer_name = quantization_layer_info.name output_layer_outputs_dim = quantization_layer_info.outputs[0].port.dim layers.append(Layer( len(layers), "Power", quantization_layer.name + "_", quantization_layer.precision, {'power': 1.0, 'scale': 1.0, 'shift': 0.0}, output_layer_outputs_dim, output_layer_outputs_dim)) builder = NetworkBuilder().sequential(layers) else: raise ValueError("unsupported layer type '{}'".format(quantization_layer.type)) # filling weights and biases temporary_file = tempfile.NamedTemporaryFile(delete=False) try: builder_str = str(builder) network_content = str.encode(builder_str) temporary_file.write(network_content) temporary_file.close() network_for_layer_model = temporary_file.name network_for_layer = ie.IENetwork(network_for_layer_model, weights_file_path) network_for_layer.add_outputs([quantization_layer.name + "_"]) finally: if os.path.exists(temporary_file.name): temporary_file.close() os.remove(temporary_file.name) finally: if weights_file_path_has_to_be_removed and os.path.exists(weights_file_path): os.remove(weights_file_path) return network_for_layer, reference_output_layer_name
def _create_single_layer_networks(self, stat): ''' Method get layers which can be quantized and affect on final accuracy. Separate network is created for each layer. ''' network = ie.IENetwork(self._configuration.model, self._configuration.weights) # if self._configuration.batch_size: # # need to use reshape API # network.batch_size = self._configuration.batch_size try: network_info = NetworkInfo(self._configuration.model) # CVS-14302: IE Network INT8 Normalizer: scale factor calculation is incorrect # for layer_name, layer_statistics in stat.items(): # layer_info = network_info.get_layer(layer_name) # if layer_info.type == 'Convolution' and \ # layer_info.outputs and \ # layer_info.outputs[0].layer.type == 'ReLU' and \ # layer_info.outputs[0].layer.outputs[0] and \ # len(layer_statistics.max_outputs) > len(stat[layer_info.outputs[0].layer.name].max_outputs): # relu_max_outputs = stat[layer_info.outputs[0].layer.name].max_outputs # relu_min_outputs = stat[layer_info.outputs[0].layer.name].min_outputs # while len(layer_statistics.max_outputs) > len(relu_max_outputs): # relu_max_outputs.append(relu_max_outputs[-1]) # relu_min_outputs.append(relu_min_outputs[-1]) single_layer_networks = dict() layer_index = 1 for layer_to_clone in network.layers.values(): layer_to_clone_info = network_info.get_layer( layer_to_clone.name) if layer_to_clone.name in self._ignore_layer_names or \ not self._normalizer.is_quantization_supported(layer_to_clone.type) or \ len(layer_to_clone_info.outputs) != 1 or \ len(layer_to_clone_info.outputs[0].layer.inputs != 1): continue activation_layer = network.layers[ layer_to_clone_info.outputs[0].layer.name] if ( len(layer_to_clone_info.outputs) == 1 and self._normalizer.is_quantization_fusing_supported( layer_to_clone_info, layer_to_clone_info.outputs[0].layer)) else None if activation_layer: debug("create network #{} for layer {} ({}) -> {} ({})". format(layer_index, layer_to_clone.name, layer_to_clone.type, activation_layer.name, activation_layer.type)) else: debug("create network #{} for layer {} ({})".format( layer_index, layer_to_clone.name, layer_to_clone.type)) layer_network, reference_output_layer_name = self._normalizer.create_network_for_layer( self._configuration.weights, layer_to_clone, layer_to_clone_info, activation_layer) Network.reshape(layer_network, self._configuration.batch_size) network_stats = {} # TODO: initialize only neccessary statistic for layer_name, node_statistic in stat.items(): network_stats[layer_name] = ie.LayerStats( min=tuple(node_statistic.min_outputs), max=tuple(node_statistic.max_outputs)) layer_network.stats.update(network_stats) params = layer_network.layers[layer_to_clone.name].params params[ "quantization_level"] = 'I8' if self._configuration.precision == 'INT8' else self._configuration.precision layer_network.layers[layer_to_clone.name].params = params exec_network = self._plugin.load( network=layer_network, config={"EXCLUSIVE_ASYNC_REQUESTS": "YES"}) if len(layer_network.inputs) != 1: raise ValueError("created network has several inputs") network_input_layer_name = next( iter(layer_network.inputs.keys())) single_layer_networks[ layer_to_clone.name] = SingleLayerNetwork( network=layer_network, exec_network=exec_network, input_layer_name=network_input_layer_name, layer_name=layer_to_clone.name, output_layer_name=layer_to_clone.name + "_", reference_output_layer_name=reference_output_layer_name ) layer_index += 1 return single_layer_networks finally: del network