def pre_post_processing(model: Model, app_inputs_info, input_precision: str, output_precision: str, input_output_precision: str): pre_post_processor = PrePostProcessor(model) if input_precision: element_type = get_element_type(input_precision) for i in range(len(model.inputs)): pre_post_processor.input(i).tensor().set_element_type(element_type) app_inputs_info[i].element_type = element_type if output_precision: element_type = get_element_type(output_precision) for i in range(len(model.outputs)): pre_post_processor.output(i).tensor().set_element_type( element_type) user_precision_map = {} if input_output_precision: user_precision_map = parse_input_output_precision( input_output_precision) input_names = get_input_output_names(model.inputs) input_node_names = get_node_names(model.inputs) output_names = get_input_output_names(model.outputs) output_node_names = get_node_names(model.outputs) for node_name, precision in user_precision_map.items(): user_precision_map[node_name] = get_element_type(precision) for name, element_type in user_precision_map.items(): if name in input_names or name in input_node_names: input_index = input_names.index( name) if name in input_names else input_node_names.index( name) app_inputs_info[input_index].element_type = element_type pre_post_processor.input( input_index).tensor().set_element_type(element_type) elif name in output_names or name in output_node_names: if name in output_names: pre_post_processor.output(name).tensor().set_element_type( element_type) else: pre_post_processor.output(output_node_names.index( name)).tensor().set_element_type(element_type) else: raise Exception(f"Node '{name}' does not exist in model") # update app_inputs_info if not input_precision: inputs = model.inputs input_node_names = get_node_names(model.inputs) for i in range(len(inputs)): if app_inputs_info[i].name in user_precision_map: app_inputs_info[i].element_type = user_precision_map[ app_inputs_info[i].name] elif input_node_names[i] in user_precision_map: app_inputs_info[i].element_type = user_precision_map[ input_node_names[i]] elif app_inputs_info[i].is_image: app_inputs_info[i].element_type = Type.u8 pre_post_processor.input(i).tensor().set_element_type(Type.u8) # set layout for model input for info in app_inputs_info: pre_post_processor.input(info.name).model().set_layout(info.layout) model = pre_post_processor.build()
# Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 #! [import] from openvino.runtime import Core, set_batch from openvino.preprocess import PrePostProcessor #! [import] model_path = "model.xml" batch_size = 8 #! [ov_gna_read_model] core = Core() model = core.read_model(model=model_path) #! [ov_gna_read_model] #! [ov_gna_set_nc_layout] ppp = PrePostProcessor(model) for i in range(len(model.inputs)): input_name = model.input(i).get_any_name() ppp.input(i).model().set_layout("N?") model = ppp.build() #! [ov_gna_set_nc_layout] #! [ov_gna_set_batch_size] set_batch(model, batch_size) #! [ov_gna_set_batch_size]
# Copyright (C) 2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 #! [init_preproc] from openvino.runtime import Core, Type, Layout from openvino.preprocess import PrePostProcessor, ColorFormat core = Core() model = core.read_model("model.xml") p = PrePostProcessor(model) p.input().tensor().set_element_type(Type.u8) \ .set_color_format(ColorFormat.NV12_TWO_PLANES, ["y", "uv"]) \ .set_memory_type("GPU_SURFACE") p.input().preprocess().convert_color(ColorFormat.BGR) p.input().model().set_layout(Layout("NCHW")) model_with_preproc = p.build() #! [init_preproc]
def test_ngraph_preprocess_model(): model = bytes(b"""<net name="add_model" version="10"> <layers> <layer id="0" name="x" type="Parameter" version="opset1"> <data element_type="i32" shape="2,2,2"/> <output> <port id="0" precision="FP32"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="1" name="y" type="Parameter" version="opset1"> <data element_type="i32" shape="2,2,2"/> <output> <port id="0" precision="FP32"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="2" name="sum" type="Add" version="opset1"> <input> <port id="0"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> <port id="1"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </input> <output> <port id="2" precision="FP32"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="3" name="sum/sink_port_0" type="Result" version="opset1"> <input> <port id="0"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </input> </layer> </layers> <edges> <edge from-layer="0" from-port="0" to-layer="2" to-port="0"/> <edge from-layer="1" from-port="0" to-layer="2" to-port="1"/> <edge from-layer="2" from-port="2" to-layer="3" to-port="0"/> </edges> </net>""") core = Core() function = core.read_model(model=model) @custom_preprocess_function def custom_preprocess(output: Output): return ops.abs(output) p = PrePostProcessor(function) p.input(1).preprocess().convert_element_type(Type.f32).scale(0.5) p.input(0).preprocess().convert_element_type(Type.f32).mean(5.) p.output(0).postprocess().custom(custom_preprocess) function = p.build() input_data = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype(np.float32) expected_output = np.array([[[2, 1], [4, 7]], [[10, 13], [16, 19]]]).astype(np.float32) runtime = get_runtime() computation = runtime.computation(function) output = computation(input_data, input_data) assert np.equal(output, expected_output).all()
# Copyright (C) 2018-2022 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # from openvino.preprocess import ResizeAlgorithm, ColorFormat from openvino.runtime import Layout, Type xml_path = '' input_name = '' # ! [ov:preprocess:create] from openvino.preprocess import PrePostProcessor from openvino.runtime import Core core = Core() model = core.read_model(model=xml_path) ppp = PrePostProcessor(model) # ! [ov:preprocess:create] # ! [ov:preprocess:tensor] from openvino.preprocess import ColorFormat from openvino.runtime import Layout, Type ppp.input(input_name).tensor() \ .set_element_type(Type.u8) \ .set_shape([1, 480, 640, 3]) \ .set_layout(Layout('NHWC')) \ .set_color_format(ColorFormat.BGR) # ! [ov:preprocess:tensor] # ! [ov:preprocess:model] # `model's input` already `knows` it's shape and data type, no need to specify them here ppp.input(input_name).model().set_layout(Layout('NCHW')) # ! [ov:preprocess:model]
def test_infer_float16(device): model = bytes(b"""<net name="add_model" version="10"> <layers> <layer id="0" name="x" type="Parameter" version="opset1"> <data element_type="f16" shape="2,2,2"/> <output> <port id="0" precision="FP16"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="1" name="y" type="Parameter" version="opset1"> <data element_type="f16" shape="2,2,2"/> <output> <port id="0" precision="FP16"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="2" name="sum" type="Add" version="opset1"> <input> <port id="0"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> <port id="1"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </input> <output> <port id="2" precision="FP16"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </output> </layer> <layer id="3" name="sum/sink_port_0" type="Result" version="opset1"> <input> <port id="0"> <dim>2</dim> <dim>2</dim> <dim>2</dim> </port> </input> </layer> </layers> <edges> <edge from-layer="0" from-port="0" to-layer="2" to-port="0"/> <edge from-layer="1" from-port="0" to-layer="2" to-port="1"/> <edge from-layer="2" from-port="2" to-layer="3" to-port="0"/> </edges> </net>""") core = Core() func = core.read_model(model=model) p = PrePostProcessor(func) p.input(0).tensor().set_element_type(Type.f16) p.input(0).preprocess().convert_element_type(Type.f16) p.input(1).tensor().set_element_type(Type.f16) p.input(1).preprocess().convert_element_type(Type.f16) p.output(0).tensor().set_element_type(Type.f16) p.output(0).postprocess().convert_element_type(Type.f16) func = p.build() exec_net = core.compile_model(func, device) input_data = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]]).astype(np.float16) request = exec_net.create_infer_request() outputs = request.infer({0: input_data, 1: input_data}) assert np.allclose(list(outputs.values()), list(request.results.values())) assert np.allclose(list(outputs.values()), input_data + input_data)
def main(): args = parse_args() # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') core = Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- if args.model: log.info(f'Reading the model: {args.model}') # (.xml and .bin files) or (.onnx file) model = core.read_model(args.model) # --------------------------- Step 3. Apply preprocessing ------------------------------------------------------------- if args.output_layers: output_layer_names, output_layer_ports = parse_outputs_from_args( args) model.add_outputs(list(zip(output_layer_names, output_layer_ports))) ppp = PrePostProcessor(model) for i in range(len(model.inputs)): ppp.input(i).tensor() \ .set_element_type(Type.f32) \ .set_layout(Layout('NC')) # noqa: N400 ppp.input(i).model().set_layout(Layout('NC')) for i in range(len(model.outputs)): ppp.output(i).tensor().set_element_type(Type.f32) model = ppp.build() if args.context_window_left == args.context_window_right == 0: set_batch(model, args.batch_size) else: set_batch(model, 1) # ---------------------------Step 4. Configure plugin --------------------------------------------------------- devices = args.device.replace('HETERO:', '').split(',') plugin_config = {} if 'GNA' in args.device: gna_device_mode = devices[0] if '_' in devices[0] else 'GNA_AUTO' devices[0] = 'GNA' plugin_config['GNA_DEVICE_MODE'] = gna_device_mode plugin_config['GNA_PRECISION'] = f'I{args.quantization_bits}' plugin_config['GNA_EXEC_TARGET'] = args.exec_target plugin_config['GNA_PWL_MAX_ERROR_PERCENT'] = str(args.pwl_me) # Set a GNA scale factor if args.import_gna_model: if args.scale_factor: log.warning( f'Custom scale factor will be used for imported GNA model: {args.import_gna_model}' ) set_scale_factors(plugin_config, parse_scale_factors(args)) else: log.info( f'Using scale factor from the imported GNA model: {args.import_gna_model}' ) else: if args.scale_factor: set_scale_factors(plugin_config, parse_scale_factors(args)) else: scale_factors = [] for file_name in re.split(', |,', args.input): _, utterances = read_utterance_file(file_name) scale_factors.append(get_scale_factor(utterances[0])) log.info( 'Using scale factor(s) calculated from first utterance') set_scale_factors(plugin_config, scale_factors) if args.export_embedded_gna_model: plugin_config[ 'GNA_FIRMWARE_MODEL_IMAGE'] = args.export_embedded_gna_model plugin_config[ 'GNA_FIRMWARE_MODEL_IMAGE_GENERATION'] = args.embedded_gna_configuration if args.performance_counter: plugin_config['PERF_COUNT'] = 'YES' device_str = f'HETERO:{",".join(devices)}' if 'HETERO' in args.device else devices[ 0] # --------------------------- Step 5. Loading model to the device ----------------------------------------------------- log.info('Loading the model to the plugin') if args.model: compiled_model = core.compile_model(model, device_str, plugin_config) else: compiled_model = core.import_model(args.import_gna_model, device_str, plugin_config) # --------------------------- Exporting GNA model using InferenceEngine AOT API --------------------------------------- if args.export_gna_model: log.info(f'Writing GNA Model to {args.export_gna_model}') compiled_model.export_model(args.export_gna_model) return 0 if args.export_embedded_gna_model: log.info( f'Exported GNA embedded model to file {args.export_embedded_gna_model}' ) log.info( f'GNA embedded model export done for GNA generation {args.embedded_gna_configuration}' ) return 0 # --------------------------- Step 6. Set up input -------------------------------------------------------------------- if args.input_layers: input_layer_names = re.split(', |,', args.input_layers) else: input_layer_names = [ _input.any_name for _input in compiled_model.inputs ] input_file_names = re.split(', |,', args.input) if len(input_layer_names) != len(input_file_names): log.error( f'Number of network inputs ({len(compiled_model.inputs)}) is not equal ' f'to number of ark files ({len(input_file_names)})') sys.exit(-3) input_file_data = [ read_utterance_file(file_name) for file_name in input_file_names ] infer_data = [{ input_layer_names[j]: input_file_data[j].utterances[i] for j in range(len(input_layer_names)) } for i in range(len(input_file_data[0].utterances))] if args.output_layers: output_layer_names, output_layer_ports = parse_outputs_from_args(args) # If a name of output layer contains a port number then concatenate output_layer_names and output_layer_ports if ':' in compiled_model.outputs[0].any_name: output_layer_names = [ f'{output_layer_names[i]}:{output_layer_ports[i]}' for i in range(len(output_layer_names)) ] else: output_layer_names = [compiled_model.outputs[0].any_name] if args.output: output_file_names = re.split(', |,', args.output) if len(output_layer_names) != len(output_file_names): log.error( 'The number of output files is not equal to the number of network outputs.' ) sys.exit(-6) if args.reference: reference_file_names = re.split(', |,', args.reference) if len(output_layer_names) != len(reference_file_names): log.error( 'The number of reference files is not equal to the number of network outputs.' ) sys.exit(-5) reference_file_data = [ read_utterance_file(file_name) for file_name in reference_file_names ] references = [{ output_layer_names[j]: reference_file_data[j].utterances[i] for j in range(len(output_layer_names)) } for i in range(len(input_file_data[0].utterances))] # --------------------------- Step 7. Create infer request ------------------------------------------------------------ infer_request = compiled_model.create_infer_request() # --------------------------- Step 8. Do inference -------------------------------------------------------------------- log.info('Starting inference in synchronous mode') results = [] total_infer_time = 0 for i in range(len(infer_data)): start_infer_time = default_timer() # Reset states between utterance inferences to remove a memory impact for state in infer_request.query_state(): state.reset() results.append( do_inference( infer_data[i], infer_request, args.context_window_left, args.context_window_right, )) infer_time = default_timer() - start_infer_time total_infer_time += infer_time num_of_frames = infer_data[i][input_layer_names[0]].shape[0] avg_infer_time_per_frame = infer_time / num_of_frames # --------------------------- Step 9. Process output ------------------------------------------------------------------ log.info('') log.info(f'Utterance {i}:') log.info(f'Total time in Infer (HW and SW): {infer_time * 1000:.2f}ms') log.info(f'Frames in utterance: {num_of_frames}') log.info( f'Average Infer time per frame: {avg_infer_time_per_frame * 1000:.2f}ms' ) for name in output_layer_names: log.info('') log.info(f'Output blob name: {name}') log.info(f'Number scores per frame: {results[i][name].shape[1]}') if args.reference: log.info('') compare_with_reference(results[i][name], references[i][name]) if args.performance_counter: if 'GNA' in args.device: total_cycles = infer_request.profiling_info[ 0].real_time.total_seconds() stall_cycles = infer_request.profiling_info[ 1].real_time.total_seconds() active_cycles = total_cycles - stall_cycles frequency = 10**6 if args.arch == 'CORE': frequency *= GNA_CORE_FREQUENCY else: frequency *= GNA_ATOM_FREQUENCY total_inference_time = total_cycles / frequency active_time = active_cycles / frequency stall_time = stall_cycles / frequency log.info('') log.info('Performance Statistics of GNA Hardware') log.info( f' Total Inference Time: {(total_inference_time * 1000):.4f} ms' ) log.info(f' Active Time: {(active_time * 1000):.4f} ms') log.info(f' Stall Time: {(stall_time * 1000):.4f} ms') log.info('') log.info(f'Total sample time: {total_infer_time * 1000:.2f}ms') if args.output: for i, name in enumerate(output_layer_names): data = [ results[i][name] for i in range(len(input_file_data[0].utterances)) ] write_utterance_file(output_file_names[i], input_file_data[0].keys, data) log.info(f'File {output_file_names[i]} was created!') # ---------------------------------------------------------------------------------------------------------------------- log.info( 'This sample is an API example, ' 'for any performance measurements please use the dedicated benchmark_app tool\n' ) return 0
def _set_preprocess(self, preprocess): if preprocess.ie_processor is None: return if self.network is not None: self.disable_resize_to_input = False preprocess_steps = preprocess.ie_preprocess_steps if not preprocess_steps: return preprocessor = PrePostProcessor(self.network) for input_name in self.inputs: if input_name in self.const_inputs + self.image_info_inputs: continue input_id = self.input_to_index[input_name] for (name, value) in preprocess_steps: if name == 'resize_algorithm': preprocessor.input(input_id).tensor().set_spatial_dynamic_shape() preprocessor.input(input_id).preprocess().resize(value) self.need_dyn_resolving = False if name == 'convert_color_format': src, dst = value preprocessor.input(input_id).tensor().set_color_format(src) preprocessor.input(input_id).preprocess().convert_color(dst) if name == 'mean_variant': mean, scale = value if mean is not None: preprocessor.input(input_id).preprocess().mean(mean) if scale is not None: preprocessor.input(input_id).preprocess().scale(scale) self.network = preprocessor.build() self._preprocess_steps = preprocess_steps self.disable_resize_to_input = preprocess.ie_processor.has_resize()
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) # Parsing and validation of input arguments if len(sys.argv) != 4: log.info('Usage: <path_to_model> <path_to_image> <device_name>') return 1 model_path = sys.argv[1] image_path = sys.argv[2] device_name = sys.argv[3] # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') core = Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the network: {model_path}') # (.xml and .bin files) or (.onnx file) model = core.read_model(model_path) if len(model.inputs) != 1: log.error('Sample supports only single input topologies') return -1 if len(model.outputs) != 1: log.error('Sample supports only single output topologies') return -1 # --------------------------- Step 3. Set up input -------------------------------------------------------------------- # Read input image image = cv2.imread(image_path) # Add N dimension input_tensor = np.expand_dims(image, 0) log.info( 'Reshaping the network to the height and width of the input image') n, h, w, c = input_tensor.shape model.reshape({model.input().get_any_name(): PartialShape((n, c, h, w))}) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- ppp = PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ .set_element_type(Type.u8) \ .set_layout(Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input ppp.input().model().set_layout(Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' ppp.output().tensor().set_element_type(Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() # ---------------------------Step 4. Loading model to the device------------------------------------------------------- log.info('Loading the model to the plugin') compiled_model = core.compile_model(model, device_name) # --------------------------- Step 6. Create infer request and do inference synchronously ----------------------------- log.info('Starting inference in synchronous mode') results = compiled_model.infer_new_request({0: input_tensor}) # ---------------------------Step 6. Process output-------------------------------------------------------------------- predictions = next(iter(results.values())) # Change a shape of a numpy.ndarray with results ([1, 1, N, 7]) to get another one ([N, 7]), # where N is the number of detected bounding boxes detections = predictions.reshape(-1, 7) for detection in detections: confidence = detection[2] if confidence > 0.5: class_id = int(detection[1]) xmin = int(detection[3] * w) ymin = int(detection[4] * h) xmax = int(detection[5] * w) ymax = int(detection[6] * h) log.info( f'Found: class_id = {class_id}, confidence = {confidence:.2f}, ' f'coords = ({xmin}, {ymin}), ({xmax}, {ymax})') # Draw a bounding box on a output image cv2.rectangle(image, (xmin, ymin), (xmax, ymax), (0, 255, 0), 2) cv2.imwrite('out.bmp', image) if os.path.exists('out.bmp'): log.info('Image out.bmp was created!') else: log.error('Image out.bmp was not created. Check your permissions.') # ---------------------------------------------------------------------------------------------------------------------- log.info( 'This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n' ) return 0
def main() -> int: log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = parse_args() # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') core = Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the network: {args.model}') # (.xml and .bin files) or (.onnx file) model = core.read_model(args.model) if len(model.inputs) != 1: log.error('Sample supports only single input topologies') return -1 if len(model.outputs) != 1: log.error('Sample supports only single output topologies') return -1 # --------------------------- Step 3. Set up input -------------------------------------------------------------------- # Read input images images = [cv2.imread(image_path) for image_path in args.input] # Resize images to model input dims _, _, h, w = model.input().shape resized_images = [cv2.resize(image, (w, h)) for image in images] # Add N dimension input_tensors = [np.expand_dims(image, 0) for image in resized_images] # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- ppp = PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ .set_element_type(Type.u8) \ .set_layout(Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input ppp.input().model().set_layout(Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' ppp.output().tensor().set_element_type(Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() # --------------------------- Step 5. Loading model to the device ----------------------------------------------------- log.info('Loading the model to the plugin') compiled_model = core.compile_model(model, args.device) # --------------------------- Step 6. Create infer request queue ------------------------------------------------------ log.info('Starting inference in asynchronous mode') infer_queue = AsyncInferQueue(compiled_model, len(input_tensors)) infer_queue.set_callback(completion_callback) # --------------------------- Step 7. Do inference -------------------------------------------------------------------- for i, input_tensor in enumerate(input_tensors): infer_queue.start_async({0: input_tensor}, args.input[i]) infer_queue.wait_all() # ---------------------------------------------------------------------------------------------------------------------- log.info( 'This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n' ) return 0
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) # Parsing and validation of input arguments if len(sys.argv) != 4: log.info(f'Usage: {sys.argv[0]} <path_to_model> <path_to_image> <device_name>') return 1 model_path = sys.argv[1] image_path = sys.argv[2] device_name = sys.argv[3] # --------------------------- Step 1. Initialize OpenVINO Runtime Core ------------------------------------------------ log.info('Creating OpenVINO Runtime Core') core = Core() # --------------------------- Step 2. Read a model -------------------------------------------------------------------- log.info(f'Reading the model: {model_path}') # (.xml and .bin files) or (.onnx file) model = core.read_model(model_path) if len(model.inputs) != 1: log.error('Sample supports only single input topologies') return -1 if len(model.outputs) != 1: log.error('Sample supports only single output topologies') return -1 # --------------------------- Step 3. Set up input -------------------------------------------------------------------- # Read input image image = cv2.imread(image_path) # Add N dimension input_tensor = np.expand_dims(image, 0) # --------------------------- Step 4. Apply preprocessing ------------------------------------------------------------- ppp = PrePostProcessor(model) _, h, w, _ = input_tensor.shape # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' # - set static spatial dimensions to input tensor to resize from ppp.input().tensor() \ .set_element_type(Type.u8) \ .set_layout(Layout('NHWC')) \ .set_spatial_static_shape(h, w) # noqa: ECE001, N400 # 2) Adding explicit preprocessing steps: # - apply linear resize from tensor spatial dims to model spatial dims ppp.input().preprocess().resize(ResizeAlgorithm.RESIZE_LINEAR) # 3) Here we suppose model has 'NCHW' layout for input ppp.input().model().set_layout(Layout('NCHW')) # 4) Set output tensor information: # - precision of tensor is supposed to be 'f32' ppp.output().tensor().set_element_type(Type.f32) # 5) Apply preprocessing modifing the original 'model' model = ppp.build() # --------------------------- Step 5. Loading model to the device ----------------------------------------------------- log.info('Loading the model to the plugin') compiled_model = core.compile_model(model, device_name) # --------------------------- Step 6. Create infer request and do inference synchronously ----------------------------- log.info('Starting inference in synchronous mode') results = compiled_model.infer_new_request({0: input_tensor}) # --------------------------- Step 7. Process output ------------------------------------------------------------------ predictions = next(iter(results.values())) # Change a shape of a numpy.ndarray with results to get another one with one dimension probs = predictions.reshape(-1) # Get an array of 10 class IDs in descending order of probability top_10 = np.argsort(probs)[-10:][::-1] header = 'class_id probability' log.info(f'Image path: {image_path}') log.info('Top 10 results: ') log.info(header) log.info('-' * len(header)) for class_id in top_10: probability_indent = ' ' * (len('class_id') - len(str(class_id)) + 1) log.info(f'{class_id}{probability_indent}{probs[class_id]:.7f}') log.info('') # ---------------------------------------------------------------------------------------------------------------------- log.info('This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n') return 0
def test_pre_post_process_build(): p = PrePostProcessor(model) p.input().model().set_layout(Layout("NC")) check_gil_released_safe(p.build)
def apply_preprocessing(ov_function: Model, argv: argparse.Namespace): """ Applies pre-processing of model inputs by adding appropriate operations On return, 'ov_function' object will be updated Expected 'argv.mean_scale_values' formats examples: a) Dict: {'inputName': {'mean': [1., 2., 3.], 'scale': [2., 4., 8.]}} b) List: list(np.array([(np.array([1., 2., 3.]), np.array([2., 4., 6.])), (np.array([7., 8., 9.]), np.array([5., 6., 7.]))) Expected 'argv.layout_values' format examples: a) Specific layouts for inputs and outputs { 'input1': { 'source_layout': 'nchw', 'target_layout': 'nhwc' }, 'output2': { 'source_layout': 'nhwc' } } b) Layout for single input: {'': {'source_layout': 'nchw'}} :param: ov_function OV function for applying mean/scale pre-processing :param: argv Parsed command line arguments """ prep = PrePostProcessor(ov_function) if 'mean_scale_values' in argv and argv.mean_scale_values: mean_scale_values = argv.mean_scale_values else: mean_scale_values = {} mean_scale_values = update_mean_scale_to_dict( input_nodes=ov_function.inputs, mean_scale_val=mean_scale_values, scale=argv.scale) # On return, mean_scale_values is a dictionary with input names as key and mean/scale pair as value # {'inputName': {'mean': [1., 2., 3.], 'scale': [2.]}} layout_values = {} if 'layout_values' in argv and argv.layout_values: layout_values = argv.layout_values if '' in layout_values: if len(ov_function.inputs) > 1: input_names = [ list(ov_input.get_tensor().get_names())[0] for ov_input in ov_function.inputs ] raise Error( 'Layout without name can be specified for models with only one input, ' 'but provided model has {} inputs: \'{}\'. ' 'Please specify explicitly input/output name for --layout option' .format(len(input_names), input_names)) layout_values = { list(ov_function.input().get_tensor().get_names())[0]: { 'source_layout': layout_values[''].get('source_layout'), 'target_layout': layout_values[''].get('target_layout') } } check_keys_valid(ov_function=ov_function, dict_to_validate=mean_scale_values, search_outputs=False) check_keys_valid(ov_function=ov_function, dict_to_validate=layout_values, search_outputs=True) layout_values = update_layout_is_input_flag(ov_function, layout_values) layout_values = guess_source_layouts_by_mean_scale(ov_function, layout_values, mean_scale_values) need_reverse = 'reverse_input_channels' in argv and argv.reverse_input_channels suitable_params_ric = [] if need_reverse: suitable_params_ric = guess_source_layouts_for_reverse_channels( ov_function=ov_function, layout_values=layout_values) for node_name, layout_value in layout_values.items(): if layout_value.get('source_layout'): if layout_value.get('is_input'): prep.input(node_name).model().set_layout( Layout(layout_value['source_layout'])) else: prep.output(node_name).model().set_layout( Layout(layout_value['source_layout'])) if layout_value.get('target_layout'): if layout_value.get('is_input'): prep.input(node_name).tensor().set_layout( Layout(layout_value['target_layout'])) else: prep.output(node_name).tensor().set_layout( Layout(layout_value['target_layout'])) # Apply reverse_input_channels if need_reverse: for name, _ in suitable_params_ric: prep.input(name).preprocess().reverse_channels() log.debug( 'reverse_input_channels pre-processing applied to {}'.format( name)) for node_name, node_mean_scale_values in mean_scale_values.items(): # Apply mean first, then scale if node_mean_scale_values['mean'] is not None: prep.input(node_name).preprocess().mean( node_mean_scale_values['mean']) if node_mean_scale_values['scale'] is not None: prep.input(node_name).preprocess().scale( node_mean_scale_values['scale']) log.debug('Mean/Scale pre-processing applied to {}'.format(node_name)) # Apply pre-processing builder to a function ov_function = prep.build() # Remove guessed layout values from ov_function (these values shall not be serialized to IR for node_name, layout_value in layout_values.items(): if layout_value.get('source_guessed') and \ not layout_value.get('target_layout'): # search for parameter object for idx, ov_input in enumerate(ov_function.inputs): if node_name in ov_input.get_tensor().get_names(): log.debug('Clearing guessed layout {} for {}'.format( layout_value['source_layout'], node_name)) ov_function.get_parameters()[idx].layout = Layout()
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) # Parsing and validation of input arguments if len(sys.argv) != 3: log.info(f'Usage: {sys.argv[0]} <path_to_model> <device_name>') return 1 model_path = sys.argv[1] device_name = sys.argv[2] labels = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] number_top = 1 # ---------------------------Step 1. Initialize OpenVINO Runtime Core-------------------------------------------------- log.info('Creating OpenVINO Runtime Core') core = Core() # ---------------------------Step 2. Read a model in OpenVINO Intermediate Representation------------------------------ log.info( f'Loading the model using ngraph function with weights from {model_path}' ) model = create_ngraph_function(model_path) # ---------------------------Step 3. Apply preprocessing---------------------------------------------------------- # Get names of input and output blobs ppp = PrePostProcessor(model) # 1) Set input tensor information: # - input() provides information about a single model input # - precision of tensor is supposed to be 'u8' # - layout of data is 'NHWC' ppp.input().tensor() \ .set_element_type(Type.u8) \ .set_layout(Layout('NHWC')) # noqa: N400 # 2) Here we suppose model has 'NCHW' layout for input ppp.input().model().set_layout(Layout('NCHW')) # 3) Set output tensor information: # - precision of tensor is supposed to be 'f32' ppp.output().tensor().set_element_type(Type.f32) # 4) Apply preprocessing modifing the original 'model' model = ppp.build() # Set a batch size equal to number of input images set_batch(model, digits.shape[0]) # ---------------------------Step 4. Loading model to the device------------------------------------------------------- log.info('Loading the model to the plugin') compiled_model = core.compile_model(model, device_name) # ---------------------------Step 5. Prepare input--------------------------------------------------------------------- n, c, h, w = model.input().shape input_data = np.ndarray(shape=(n, c, h, w)) for i in range(n): image = digits[i].reshape(28, 28) image = image[:, :, np.newaxis] input_data[i] = image # ---------------------------Step 6. Do inference---------------------------------------------------------------------- log.info('Starting inference in synchronous mode') results = compiled_model.infer_new_request({0: input_data}) # ---------------------------Step 7. Process output-------------------------------------------------------------------- predictions = next(iter(results.values())) log.info(f'Top {number_top} results: ') for i in range(n): probs = predictions[i] # Get an array of number_top class IDs in descending order of probability top_n_idexes = np.argsort(probs)[-number_top:][::-1] header = 'classid probability' header = header + ' label' if labels else header log.info(f'Image {i}') log.info('') log.info(header) log.info('-' * len(header)) for class_id in top_n_idexes: probability_indent = ' ' * (len('classid') - len(str(class_id)) + 1) label_indent = ' ' * (len('probability') - 8) if labels else '' label = labels[class_id] if labels else '' log.info( f'{class_id}{probability_indent}{probs[class_id]:.7f}{label_indent}{label}' ) log.info('') # ---------------------------------------------------------------------------------------------------------------------- log.info( 'This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool\n' ) return 0