def get_layout(self, input_node=None): if self._layout is not None: if 'C' not in self._layout or 'H' not in self._layout or 'W' not in self._layout: raise ValueError('Unexpected {} layout'.format(self._layout)) if self._shape is not None and 'N' in self._layout and len( self._shape) == 3: self._layout = self._layout[1:] self._layout = Layout(self._layout) return if input_node and hasattr(input_node.graph, 'meta_data') \ and input_node.graph.meta_data.get('layout', None) not in [None, '()']: layout_from_ir = get_layout_values( input_node.graph.meta_data.get('layout', None)) if layout_from_ir is not None: layout_from_ir = layout_from_ir[next( iter(layout_from_ir))].get('source_layout', None) self._layout = Layout(layout_from_ir) return image_colors_dim = (Dimension(3), Dimension(1)) num_dims = len(self._shape) if num_dims == 4: if self._shape[1] in image_colors_dim: self._layout = Layout("NCHW") elif self._shape[3] in image_colors_dim: self._layout = Layout("NHWC") elif num_dims == 3: if self._shape[0] in image_colors_dim: self._layout = Layout("CHW") elif self._shape[2] in image_colors_dim: self._layout = Layout("HWC") logger.info(f'Layout value is set {self._layout}')
def get_layout(self, input_node=None): if self._layout is not None: if 'C' not in self._layout or 'H' not in self._layout or 'W' not in self._layout: raise ValueError('Unexpected {} layout'.format(self._layout)) if self._shape is not None and 'N' in self._layout and len( self._shape) == 3: self._layout = self._layout[1:] self._layout = Layout(self._layout) return if input_node: layout_from_ir = input_node.graph.graph.get('layout', None) if layout_from_ir is not None: if self._shape is not None and 'N' in layout_from_ir and len( self._shape) == 3: layout_from_ir = layout_from_ir[1:] self._layout = Layout(layout_from_ir) return image_colors_dim = (Dimension(3), Dimension(1)) num_dims = len(self._shape) if num_dims == 4: if self._shape[1] in image_colors_dim: self._layout = Layout("NCHW") elif self._shape[3] in image_colors_dim: self._layout = Layout("NHWC") elif num_dims == 3: if self._shape[0] in image_colors_dim: self._layout = Layout("CHW") elif self._shape[2] in image_colors_dim: self._layout = Layout("HWC")
def reshape_network(network, shapes): partial_shapes = {} for name, shape in shapes.items(): p_shape = PartialShape( [Dimension(d) if not isinstance(d, tuple) else Dimension(d[0], d[1]) for d in shape]) partial_shapes[name] = p_shape network.reshape(partial_shapes) return network
def reshape_model(self, new_shape): new_shape = { name: PartialShape([ Dimension(dim) if not isinstance(dim, tuple) else Dimension( dim[0], dim[1]) for dim in shape ]) for name, shape in new_shape.items() } self.model.reshape(new_shape)
def partial_shape_from_tuple(shape: tuple): new_shape = [] for dim in shape: if isinstance(dim, tuple): assert len(dim) == 2, "Incorrect boundaries of dimension {} in shape {}".format(dim, shape) assert dim[0] >= 0, "Incorrect min value of dimension {} in shape".format(dim, shape) new_shape.append(Dimension(dim[0], dim[1])) else: assert isinstance(dim, np.int64), "Incorrect type of dimension {} in shape".format(dim, shape) new_shape.append(Dimension(dim)) return PartialShape(new_shape)
def parse_partial_shape(shape_str): dims = [] for dim in shape_str.split(','): if '.. ' in dim: range = list(int(d) for d in dim.split('..')) assert len(range) == 2 dims.append(Dimension(range)) elif dim == '?': dims.append(Dimension()) else: dims.append(Dimension(int(dim))) return PartialShape(dims)
def get_network_batch_size(inputs_info): null_dimension = Dimension(0) batch_size = null_dimension for info in inputs_info: batch_index = info.layout.get_index_by_name('N') if info.layout.has_name('N') else -1 if batch_index != -1: if batch_size == null_dimension: batch_size = info.partial_shape[batch_index] elif batch_size != info.partial_shape[batch_index]: raise Exception("Can't deterimine batch size: batch is different for different inputs!") if batch_size == null_dimension: batch_size = Dimension(1) return batch_size
def test_non_max_suppression(): boxes_shape = [1, 1000, 4] scores_shape = [1, 1, 1000] boxes_parameter = ov.parameter(boxes_shape, name="Boxes", dtype=np.float32) scores_parameter = ov.parameter(scores_shape, name="Scores", dtype=np.float32) node = ov.non_max_suppression(boxes_parameter, scores_parameter, make_constant_node(1000, np.int64)) assert node.get_type_name() == "NonMaxSuppression" assert node.get_output_size() == 3 assert node.get_output_partial_shape(0) == PartialShape([Dimension(0, 1000), Dimension(3)]) assert node.get_output_partial_shape(1) == PartialShape([Dimension(0, 1000), Dimension(3)]) assert list(node.get_output_shape(2)) == [1]
def test_input_shape_read_only(): shape = Shape([1, 10]) param = ov.parameter(shape, dtype=np.float32) model = Model(ov.relu(param), [param]) ref_shape = model.input().shape ref_shape[0] = Dimension(3) assert model.input().shape == shape
def is_image_info(self): if str(self.layout) != "[N,C]": return False return len( self.channels ) >= 2 if self.channels.is_static else self.channels.relaxes( Dimension(2))
def _reshape_input(self, shapes, make_dynamic=False): if hasattr(self, 'exec_network'): del self.exec_network if self.infer_request is not None: del self.infer_request self.infer_request = None partial_shapes = {} for name, shape in shapes.items(): p_shape = PartialShape( [Dimension(d) if not isinstance(d, tuple) else Dimension(d[0], d[1]) for d in shape]) partial_shapes[self.input_to_index[name]] = p_shape self.network.reshape(partial_shapes) self.dyn_input_layers, self._partial_shapes = self.get_dynamic_inputs(self.network) if self.dyn_input_layers and make_dynamic: return self.exec_network = self.ie_core.compile_model(self.network, self.device) self.infer_request = self.exec_network.create_infer_request()
def parse_batch_size(batch_size_str): if batch_size_str: error_message = f"Can't parse batch size '{batch_size_str}'" dims = batch_size_str.split("..") if len(dims) > 2: raise Exception(error_message) elif len(dims) == 2: range = [] for d in dims: if d.isnumeric(): range.append(int(d)) else: raise Exception(error_message) return Dimension(*range) else: if dims[0].lstrip("-").isnumeric(): return Dimension(int(dims[0])) elif dims[0] == "?": return Dimension() else: raise Exception(error_message) else: return Dimension(0)
def test_set_batch_dimension(): model = create_test_model() model_param1 = model.get_parameters()[0] model_param2 = model.get_parameters()[1] # batch == 2 model_param1.set_layout(Layout("NC")) assert get_batch(model) == 2 # set batch to 1 set_batch(model, Dimension(1)) assert get_batch(model) == 1 # check if shape of param 1 has changed assert model_param1.get_output_shape(0) == PartialShape([1, 1]) # check if shape of param 2 has not changed assert model_param2.get_output_shape(0) == PartialShape([2, 1])
def test_set_batch_dimension(): param1 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data1") param2 = ops.parameter(Shape([2, 1]), dtype=np.float32, name="data2") add = ops.add(param1, param2) func = Model(add, [param1, param2], "TestFunction") func_param1 = func.get_parameters()[0] func_param2 = func.get_parameters()[1] # batch == 2 func_param1.set_layout(Layout("NC")) assert get_batch(func) == 2 # set batch to 1 set_batch(func, Dimension(1)) assert get_batch(func) == 1 # check if shape of param 1 has changed assert str(func_param1.get_output_shape(0) == {1, 1}) # check if shape of param 2 has not changed assert str(func_param2.get_output_shape(0) == {2, 1})
def test_partial_shape(): ps = PartialShape([1, 2, 3, 4]) assert ps.is_static assert not ps.is_dynamic assert ps.rank == 4 assert repr(ps) == "<PartialShape: {1,2,3,4}>" assert ps.get_dimension(0) == Dimension(1) assert ps.get_dimension(1) == Dimension(2) assert ps.get_dimension(2) == Dimension(3) assert ps.get_dimension(3) == Dimension(4) shape = Shape([1, 2, 3]) ps = PartialShape(shape) assert ps.is_static assert not ps.is_dynamic assert ps.all_non_negative assert ps.rank == 3 assert list(ps.get_shape()) == [1, 2, 3] assert list(ps.get_max_shape()) == [1, 2, 3] assert list(ps.get_min_shape()) == [1, 2, 3] assert list(ps.to_shape()) == [1, 2, 3] assert repr(shape) == "<Shape: {1, 2, 3}>" assert repr(ps) == "<PartialShape: {1,2,3}>" ps = PartialShape( [Dimension(1), Dimension(2), Dimension(3), Dimension.dynamic()]) assert not ps.is_static assert ps.is_dynamic assert ps.all_non_negative assert ps.rank == 4 assert list(ps.get_min_shape()) == [1, 2, 3, 0] assert list(ps.get_max_shape())[3] > 1000000000 assert repr(ps) == "<PartialShape: {1,2,3,?}>" assert ps.get_dimension(0) == Dimension(1) assert ps.get_dimension(1) == Dimension(2) assert ps.get_dimension(2) == Dimension(3) assert ps.get_dimension(3) == Dimension.dynamic() ps = PartialShape([1, 2, 3, -1]) assert not ps.is_static assert ps.is_dynamic assert ps.all_non_negative assert ps.rank == 4 assert list(ps.get_min_shape()) == [1, 2, 3, 0] assert list(ps.get_max_shape())[3] > 1000000000 assert repr(ps) == "<PartialShape: {1,2,3,?}>" ps = PartialShape.dynamic() assert not ps.is_static assert ps.is_dynamic assert ps.rank == Dimension.dynamic() assert list(ps.get_min_shape()) == [] assert list(ps.get_max_shape()) == [] assert repr(ps) == "<PartialShape: ...>" ps = PartialShape.dynamic(rank=Dimension(2)) assert not ps.is_static assert ps.is_dynamic assert ps.rank == 2 assert 2 == ps.rank assert list(ps.get_min_shape()) == [0, 0] assert list(ps.get_max_shape())[0] > 1000000000 assert repr(ps) == "<PartialShape: {?,?}>"
def getDimentionByLayout(self, character): if self.layout.has_name(character): return self.partial_shape[self.layout.get_index_by_name(character)] else: return Dimension(0)
def is_image_info(self): if str(self.layout) != "[N,C]": return False return self.channels.relaxes(Dimension(2))
def run(args): statistics = None try: if args.number_streams is None: logger.warning(" -nstreams default value is determined automatically for a device. " "Although the automatic selection usually provides a reasonable performance, " "but it still may be non-optimal for some cases, for more information look at README. ") command_line_arguments = get_command_line_arguments(sys.argv) if args.report_type: statistics = StatisticsReport(StatisticsReport.Config(args.report_type, args.report_folder)) statistics.add_parameters(StatisticsReport.Category.COMMAND_LINE_PARAMETERS, command_line_arguments) def is_flag_set_in_command_line(flag): return any(x.strip('-') == flag for x, y in command_line_arguments) device_name = args.target_device devices = parse_devices(device_name) device_number_streams = parse_nstreams_value_per_device(devices, args.number_streams) config = {} if args.load_config: load_config(args.load_config, config) is_network_compiled = False _, ext = os.path.splitext(args.path_to_model) if ext == BLOB_EXTENSION: is_network_compiled = True print("Model is compiled") # ------------------------------ 2. Loading OpenVINO --------------------------------------------------- next_step(step_id=2) benchmark = Benchmark(args.target_device, args.number_infer_requests, args.number_iterations, args.time, args.api_type, args.inference_only) ## CPU (MKLDNN) extensions if CPU_DEVICE_NAME in device_name and args.path_to_extension: benchmark.add_extension(path_to_extension=args.path_to_extension) ## GPU (clDNN) Extensions if GPU_DEVICE_NAME in device_name and args.path_to_cldnn_config: if GPU_DEVICE_NAME not in config.keys(): config[GPU_DEVICE_NAME] = {} config[GPU_DEVICE_NAME]['CONFIG_FILE'] = args.path_to_cldnn_config if GPU_DEVICE_NAME in config.keys() and 'CONFIG_FILE' in config[GPU_DEVICE_NAME].keys(): cldnn_config = config[GPU_DEVICE_NAME]['CONFIG_FILE'] benchmark.add_extension(path_to_cldnn_config=cldnn_config) if not args.perf_hint: for device in devices: supported_config_keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS') if 'PERFORMANCE_HINT' in supported_config_keys: logger.warning(f"-hint default value is determined as 'THROUGHPUT' automatically for {device} device" + "For more detailed information look at README.") args.perf_hint = "throughput" version = benchmark.get_version_info() logger.info(version) # --------------------- 3. Setting device configuration -------------------------------------------------------- next_step() def get_device_type_from_name(name) : new_name = str(name) new_name = new_name.split(".", 1)[0] new_name = new_name.split("(", 1)[0] return new_name ## Set default values from dumped config default_devices = set() for device in devices: device_type = get_device_type_from_name(device) if device_type in config and device not in config: config[device] = config[device_type].copy() default_devices.add(device_type) for def_device in default_devices: config.pop(def_device) perf_counts = False for device in devices: if device not in config.keys(): config[device] = {} ## Set performance counter if is_flag_set_in_command_line('pc'): ## set to user defined value config[device]['PERF_COUNT'] = 'YES' if args.perf_counts else 'NO' elif 'PERF_COUNT' in config[device].keys() and config[device]['PERF_COUNT'] == 'YES': logger.warning(f"Performance counters for {device} device is turned on. " + "To print results use -pc option.") elif args.report_type in [ averageCntReport, detailedCntReport ]: logger.warning(f"Turn on performance counters for {device} device " + f"since report type is {args.report_type}.") config[device]['PERF_COUNT'] = 'YES' elif args.exec_graph_path is not None: logger.warning(f"Turn on performance counters for {device} device " + "due to execution graph dumping.") config[device]['PERF_COUNT'] = 'YES' else: ## set to default value config[device]['PERF_COUNT'] = 'YES' if args.perf_counts else 'NO' perf_counts = True if config[device]['PERF_COUNT'] == 'YES' else perf_counts ## high-level performance hints if is_flag_set_in_command_line('hint') or args.perf_hint: config[device]['PERFORMANCE_HINT'] = args.perf_hint.upper() if is_flag_set_in_command_line('nireq'): config[device]['PERFORMANCE_HINT_NUM_REQUESTS'] = str(args.number_infer_requests) ## the rest are individual per-device settings (overriding the values the device will deduce from perf hint) def set_throughput_streams(): key = get_device_type_from_name(device) + "_THROUGHPUT_STREAMS" if device in device_number_streams.keys(): ## set to user defined value supported_config_keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS') if key not in supported_config_keys: raise Exception(f"Device {device} doesn't support config key '{key}'! " + "Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>") config[device][key] = device_number_streams[device] elif key not in config[device].keys() and args.api_type == "async" and not is_flag_set_in_command_line('hint'): ## set the _AUTO value for the #streams logger.warning(f"-nstreams default value is determined automatically for {device} device. " + "Although the automatic selection usually provides a reasonable performance, " "but it still may be non-optimal for some cases, for more information look at README.") if device != MYRIAD_DEVICE_NAME: ## MYRIAD sets the default number of streams implicitly config[device][key] = get_device_type_from_name(device) + "_THROUGHPUT_AUTO" if key in config[device].keys(): device_number_streams[device] = config[device][key] if CPU_DEVICE_NAME in device: # CPU supports few special performance-oriented keys # limit threading for CPU portion of inference if args.number_threads and is_flag_set_in_command_line("nthreads"): config[device]['CPU_THREADS_NUM'] = str(args.number_threads) if is_flag_set_in_command_line("enforcebf16") or is_flag_set_in_command_line("enforce_bfloat16"): config[device]['ENFORCE_BF16'] = 'YES' if args.enforce_bfloat16 else 'NO' if is_flag_set_in_command_line('pin'): ## set to user defined value config[device]['CPU_BIND_THREAD'] = args.infer_threads_pinning elif 'CPU_BIND_THREAD' not in config[device].keys(): if MULTI_DEVICE_NAME in device_name and GPU_DEVICE_NAME in device_name: logger.warning(f"Turn off threads pinning for {device} " + "device since multi-scenario with GPU device is used.") config[device]['CPU_BIND_THREAD'] = 'NO' ## for CPU execution, more throughput-oriented execution via streams set_throughput_streams() elif GPU_DEVICE_NAME in device: ## for GPU execution, more throughput-oriented execution via streams set_throughput_streams() if MULTI_DEVICE_NAME in device_name and CPU_DEVICE_NAME in device_name: logger.warning("Turn on GPU throttling. Multi-device execution with the CPU + GPU performs best with GPU throttling hint, " + "which releases another CPU thread (that is otherwise used by the GPU driver for active polling)") config[device]['GPU_PLUGIN_THROTTLE'] = '1' elif MYRIAD_DEVICE_NAME in device: set_throughput_streams() config[device]['LOG_LEVEL'] = 'LOG_INFO' elif GNA_DEVICE_NAME in device: if is_flag_set_in_command_line('qb'): if args.qb == 8: config[device]['GNA_PRECISION'] = 'I8' else: config[device]['GNA_PRECISION'] = 'I16' else: supported_config_keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS') if 'CPU_THREADS_NUM' in supported_config_keys and args.number_threads and is_flag_set_in_command_line("nthreads"): config[device]['CPU_THREADS_NUM'] = str(args.number_threads) if 'CPU_THROUGHPUT_STREAMS' in supported_config_keys and args.number_streams and is_flag_set_in_command_line("streams"): config[device]['CPU_THROUGHPUT_STREAMS'] = args.number_streams if 'CPU_BIND_THREAD' in supported_config_keys and args.infer_threads_pinning and is_flag_set_in_command_line("pin"): config[device]['CPU_BIND_THREAD'] = args.infer_threads_pinning perf_counts = perf_counts benchmark.set_config(config) if args.cache_dir: benchmark.set_cache_dir(args.cache_dir) topology_name = "" load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile') if load_from_file_enabled and not is_network_compiled: next_step() print("Skipping the step for loading model from file") next_step() print("Skipping the step for loading model from file") next_step() print("Skipping the step for loading model from file") # --------------------- 7. Loading the model to the device ------------------------------------------------- next_step() start_time = datetime.utcnow() compiled_model = benchmark.core.compile_model(args.path_to_model, benchmark.device) duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Compile model took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('load network time (ms)', duration_ms) ]) app_inputs_info, _ = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.input_scale, args.input_mean, compiled_model.inputs) batch_size = get_network_batch_size(app_inputs_info) elif not is_network_compiled: # --------------------- 4. Read the Intermediate Representation of the network ----------------------------- next_step() start_time = datetime.utcnow() model = benchmark.read_model(args.path_to_model) topology_name = model.get_name() duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Read model took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('read network time (ms)', duration_ms) ]) # --------------------- 5. Resizing network to match image sizes and given batch --------------------------- next_step() app_inputs_info, reshape = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.input_scale, args.input_mean, model.inputs) if reshape: start_time = datetime.utcnow() shapes = { info.name : info.partial_shape for info in app_inputs_info } logger.info( 'Reshaping model: {}'.format(', '.join("'{}': {}".format(k, str(v)) for k, v in shapes.items()))) model.reshape(shapes) duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Reshape model took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('reshape network time (ms)', duration_ms) ]) # use batch size according to provided layout and shapes batch_size = get_network_batch_size(app_inputs_info) logger.info(f'Network batch size: {batch_size}') # --------------------- 6. Configuring inputs and outputs of the model -------------------------------------------------- next_step() pre_post_processing(model, app_inputs_info, args.input_precision, args.output_precision, args.input_output_precision) print_inputs_and_outputs_info(model) # --------------------- 7. Loading the model to the device ------------------------------------------------- next_step() start_time = datetime.utcnow() compiled_model = benchmark.core.compile_model(model, benchmark.device) duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Compile model took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('load network time (ms)', duration_ms) ]) else: next_step() print("Skipping the step for compiled network") next_step() print("Skipping the step for compiled network") next_step() print("Skipping the step for compiled network") # --------------------- 7. Loading the model to the device ------------------------------------------------- next_step() start_time = datetime.utcnow() compiled_model = benchmark.core.import_model(args.path_to_model) duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Import model took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('import network time (ms)', duration_ms) ]) app_inputs_info, _ = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.input_scale, args.input_mean, compiled_model.inputs) batch_size = get_network_batch_size(app_inputs_info) # --------------------- 8. Querying optimal runtime parameters -------------------------------------------------- next_step() ## actual device-deduced settings for device in devices: keys = benchmark.core.get_property(device, 'SUPPORTED_CONFIG_KEYS') logger.info(f'DEVICE: {device}') for k in keys: try: logger.info(f' {k} , {benchmark.core.get_property(device, k)}') except: pass # Update number of streams for device in device_number_streams.keys(): key = get_device_type_from_name(device) + '_THROUGHPUT_STREAMS' device_number_streams[device] = benchmark.core.get_property(device, key) # ------------------------------------ 9. Creating infer requests and preparing input data ---------------------- next_step() # Create infer requests start_time = datetime.utcnow() requests = benchmark.create_infer_requests(compiled_model) duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}" logger.info(f"Create {benchmark.nireq} infer requests took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('create infer requests time (ms)', duration_ms) ]) # Prepare input data paths_to_input = list() if args.paths_to_input: for path in args.paths_to_input: if ":" in next(iter(path), ""): paths_to_input.extend(path) else: paths_to_input.append(os.path.abspath(*path)) data_queue = get_input_data(paths_to_input, app_inputs_info) static_mode = check_for_static(app_inputs_info) allow_inference_only_or_sync = can_measure_as_static(app_inputs_info) if not allow_inference_only_or_sync and benchmark.api_type == 'sync': raise Exception("Benchmarking of the model with dynamic shapes is available for async API only." "Please use -api async -nstreams 1 -nireq 1 to emulate sync behavior.") if benchmark.inference_only == None: if static_mode: benchmark.inference_only = True else: benchmark.inference_only = False elif benchmark.inference_only and not allow_inference_only_or_sync: raise Exception("Benchmarking dynamic model available with input filling in measurement loop only!") if benchmark.inference_only: logger.info("Benchmarking in inference only mode (inputs filling are not included in measurement loop).") else: logger.info("Benchmarking in full mode (inputs filling are included in measurement loop).") # update batch size in case dynamic network with one data_shape if benchmark.inference_only and batch_size.is_dynamic: batch_size = Dimension(data_queue.batch_sizes[data_queue.current_group_id]) benchmark.latency_groups = get_latency_groups(app_inputs_info) if len(benchmark.latency_groups) > 1: logger.info(f"Defined {len(benchmark.latency_groups)} tensor groups:") for group in benchmark.latency_groups: print(f"\t{str(group)}") # Iteration limit benchmark.niter = get_number_iterations(benchmark.niter, benchmark.nireq, max(len(info.shapes) for info in app_inputs_info), benchmark.api_type) # Set input tensors before first inference for request in requests: data_tensors = data_queue.get_next_input() for port, data_tensor in data_tensors.items(): input_tensor = request.get_input_tensor(port) if not static_mode: input_tensor.shape = data_tensor.shape input_tensor.data[:] = data_tensor.data if statistics: statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ ('topology', topology_name), ('target device', device_name), ('API', args.api_type), ('inference_only', benchmark.inference_only), ('precision', "UNSPECIFIED"), ('batch size', str(batch_size)), ('number of iterations', str(benchmark.niter)), ('number of parallel infer requests', str(benchmark.nireq)), ('duration (ms)', str(get_duration_in_milliseconds(benchmark.duration_seconds))), ]) for nstreams in device_number_streams.items(): statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG, [ (f"number of {nstreams[0]} streams", str(nstreams[1])), ]) # ------------------------------------ 10. Measuring performance ----------------------------------------------- output_string = process_help_inference_string(benchmark, device_number_streams) next_step(additional_info=output_string) progress_bar_total_count = 10000 if benchmark.niter and not benchmark.duration_seconds: progress_bar_total_count = benchmark.niter progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) if args.progress else None duration_ms = f"{benchmark.first_infer(requests):.2f}" logger.info(f"First inference took {duration_ms} ms") if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('first inference time (ms)', duration_ms) ]) pcseq = args.pcseq if static_mode or len(benchmark.latency_groups) == 1: pcseq = False fps, median_latency_ms, avg_latency_ms, min_latency_ms, max_latency_ms, total_duration_sec, iteration = benchmark.main_loop(requests, data_queue, batch_size, args.latency_percentile, progress_bar, pcseq) # ------------------------------------ 11. Dumping statistics report ------------------------------------------- next_step() if args.dump_config: dump_config(args.dump_config, config) logger.info(f"OpenVINO configuration settings were dumped to {args.dump_config}") if args.exec_graph_path: dump_exec_graph(compiled_model, args.exec_graph_path) if perf_counts: perfs_count_list = [] for request in requests: perfs_count_list.append(request.profiling_info) if args.perf_counts: print_perf_counters(perfs_count_list) if statistics: statistics.dump_performance_counters(perfs_count_list) if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('total execution time (ms)', f'{get_duration_in_milliseconds(total_duration_sec):.2f}'), ('total number of iterations', str(iteration)), ]) if MULTI_DEVICE_NAME not in device_name: latency_prefix = None if args.latency_percentile == 50 and static_mode: #latency_prefix = 'median latency (ms)' latency_prefix = 'latency (ms)' elif args.latency_percentile != 50: latency_prefix = 'latency (' + str(args.latency_percentile) + ' percentile) (ms)' if latency_prefix: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ (latency_prefix, f'{median_latency_ms:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("avg latency", f'{avg_latency_ms:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("min latency", f'{min_latency_ms:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("max latency", f'{max_latency_ms:.2f}'), ]) if pcseq: for group in benchmark.latency_groups: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("group", str(group)), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("avg latency", f'{group.avg:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("min latency", f'{group.min:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ("max latency", f'{group.max:.2f}'), ]) statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('throughput', f'{fps:.2f}'), ]) statistics.dump() print(f'Count: {iteration} iterations') print(f'Duration: {get_duration_in_milliseconds(total_duration_sec):.2f} ms') if MULTI_DEVICE_NAME not in device_name: print('Latency:') if args.latency_percentile == 50 and static_mode: print(f' Median: {median_latency_ms:.2f} ms') elif args.latency_percentile != 50: print(f'({args.latency_percentile} percentile): {median_latency_ms:.2f} ms') print(f' AVG: {avg_latency_ms:.2f} ms') print(f' MIN: {min_latency_ms:.2f} ms') print(f' MAX: {max_latency_ms:.2f} ms') if pcseq: print("Latency for each data shape group: ") for group in benchmark.latency_groups: print(f" {str(group)}") print(f' AVG: {group.avg:.2f} ms') print(f' MIN: {group.min:.2f} ms') print(f' MAX: {group.max:.2f} ms') print(f'Throughput: {fps:.2f} FPS') del compiled_model next_step.step_id = 0 except Exception as e: logger.exception(e) if statistics: statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS, [ ('error', str(e)), ]) statistics.dump() sys.exit(1)
def test_reshape_with_python_types(device): model = create_test_model() def check_shape(new_shape): for input in model.inputs: assert input.partial_shape == new_shape shape1 = [1, 4] new_shapes = {input: shape1 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape(shape1)) shape2 = [1, 6] new_shapes = {input.any_name: shape2 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape(shape2)) shape3 = [1, 8] new_shapes = {i: shape3 for i, input in enumerate(model.inputs)} model.reshape(new_shapes) check_shape(PartialShape(shape3)) shape4 = [1, -1] new_shapes = {input: shape4 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape([Dimension(1), Dimension(-1)])) shape5 = [1, (1, 10)] new_shapes = {input: shape5 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape([Dimension(1), Dimension(1, 10)])) shape6 = [Dimension(3), Dimension(3, 10)] new_shapes = {input: shape6 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape(shape6)) shape7 = "1..10, ?" new_shapes = {input: shape7 for input in model.inputs} model.reshape(new_shapes) check_shape(PartialShape(shape7)) # reshape mixed keys shape8 = [(1, 20), -1] new_shapes = {"data1": shape8, 1: shape8} model.reshape(new_shapes) check_shape(PartialShape([Dimension(1, 20), Dimension(-1)])) # reshape with one input param = ops.parameter([1, 3, 28, 28]) model = Model(ops.relu(param), [param]) shape9 = [-1, 3, (28, 56), (28, 56)] model.reshape(shape9) check_shape( PartialShape([ Dimension(-1), Dimension(3), Dimension(28, 56), Dimension(28, 56) ])) shape10 = "?,3,..224,..224" model.reshape(shape10) check_shape( PartialShape([ Dimension(-1), Dimension(3), Dimension(-1, 224), Dimension(-1, 224) ])) # check exceptions shape10 = [1, 1, 1, 1] with pytest.raises(TypeError) as e: model.reshape({model.input().node: shape10}) assert "Incorrect key type <class 'openvino.pyopenvino.op.Parameter'> to reshape a model, " \ "expected keys as openvino.runtime.Output, int or str." in str(e.value) with pytest.raises(TypeError) as e: model.reshape({0: range(1, 9)}) assert "Incorrect value type <class 'range'> to reshape a model, " \ "expected values as openvino.runtime.PartialShape, str, list or tuple." in str(e.value)
def main(): args = build_argparser().parse_args() # load vocabulary file for model vocab = load_vocab_file(args.vocab) log.debug("Loaded vocab file from {}, get {} tokens".format( args.vocab, len(vocab))) # create tokenizer tokenizer = Tokenizer(BPE(str(args.vocab), str(args.merges))) tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tokenizer.decoder = decoders.ByteLevel() log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) ie = Core() # read IR model_path = args.model log.info('Reading model {}'.format(args.model)) model = ie.read_model(model_path) # check number inputs and outputs if len(model.inputs) != 1: raise RuntimeError( 'The demo expects model with single input, while provided {}'. format(len(model.inputs))) if len(model.outputs) != 1: raise RuntimeError( 'The demo expects model with single output, while provided {}'. format(len(model.outputs))) input_tensor = model.inputs[0].any_name if not args.dynamic_shape and ( model.inputs[0].partial_shape.is_dynamic or model.inputs[0].shape[1] != args.max_seq_len): model.reshape({ input_tensor: PartialShape([Dimension(1), Dimension(args.max_seq_len)]) }) if args.dynamic_shape: model.reshape({ input_tensor: PartialShape([Dimension(1), Dimension(0, args.max_seq_len)]) }) # load model to the device compiled_model = ie.compile_model(model, args.device) infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(args.model, args.device)) if args.input: def prompts(): for prompt in args.input: log.info("Input prompt: {}".format(prompt)) yield prompt else: def prompts(): while True: yield input('Type input prompt (empty string to exit):') # loop on user's or prepared prompts for prompt in prompts(): if not prompt.strip(): break # encode input tokens = tokenizer.encode_batch([prompt])[0].ids input_ids = np.array([tokens], dtype=np.int32) # maximum number of tokens that can be processed by network at once max_length = args.max_seq_len eos_token_id = len(vocab) - 1 cur_input_len = input_ids.shape[-1] # maximum number of tokens that will be generated max_sample_token_num = args.max_sample_token_num + cur_input_len t0 = time.perf_counter() t_count = 0 while True: model_input = input_ids if not args.dynamic_shape: # pad the rest of the request pad_len = max_length - cur_input_len model_input = np.concatenate( (input_ids, [[eos_token_id] * pad_len]), axis=-1) # create numpy inputs for IE inputs = { input_tensor: model_input, } # infer by IE t_start = time.perf_counter() res = infer_request.infer(inputs) t_end = time.perf_counter() t_count += 1 log.info( "Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)" .format(model_input.shape[1], 1 / (t_end - t_start), t_end - t_start)) outputs = next(iter(res.values())) next_token_logits = outputs[:, cur_input_len - 1, :] # pre-process distribution next_token_scores = process_logits(input_ids, next_token_logits, eos_token_id) if args.top_k > 0: next_token_scores = get_top_k_logits(next_token_scores, args.top_k) if args.top_p < 1.0: next_token_scores = get_top_p_logits(next_token_scores, args.top_p) # get next token id probs = softmax(next_token_scores) next_tokens = np.random.choice(probs.shape[-1], 1, p=probs[0], replace=True) # update info for the next step input_ids = np.concatenate((input_ids, [next_tokens]), axis=-1) cur_input_len = input_ids.shape[-1] if stop_criteria(input_ids, min(max_length, max_sample_token_num), eos_token_id): break t1 = time.perf_counter() text = tokenizer.decode_batch(input_ids)[0] log.info( "{} requests were processed in {:0.2f}sec ({:0.2}sec per request)". format(t_count, t1 - t0, (t1 - t0) / t_count)) # print result log.info("GENERATED SEQUENCE: {}".format(text))
def test_partial_shape(): ps = PartialShape([1, 2, 3, 4]) assert ps.is_static assert not ps.is_dynamic assert ps.rank == 4 assert repr(ps) == "<PartialShape: {1,2,3,4}>" assert ps.get_dimension(0) == Dimension(1) assert ps.get_dimension(1) == Dimension(2) assert ps.get_dimension(2) == Dimension(3) assert ps.get_dimension(3) == Dimension(4) shape = Shape([1, 2, 3]) ps = PartialShape(shape) assert ps.is_static assert not ps.is_dynamic assert ps.all_non_negative assert ps.rank == 3 assert list(ps.get_shape()) == [1, 2, 3] assert list(ps.get_max_shape()) == [1, 2, 3] assert list(ps.get_min_shape()) == [1, 2, 3] assert list(ps.to_shape()) == [1, 2, 3] assert repr(shape) == "<Shape: {1, 2, 3}>" assert repr(ps) == "<PartialShape: {1,2,3}>" ps = PartialShape( [Dimension(1), Dimension(2), Dimension(3), Dimension.dynamic()]) assert not ps.is_static assert ps.is_dynamic assert ps.all_non_negative assert ps.rank == 4 assert list(ps.get_min_shape()) == [1, 2, 3, 0] assert list(ps.get_max_shape())[3] > 1000000000 assert repr(ps) == "<PartialShape: {1,2,3,?}>" assert ps.get_dimension(0) == Dimension(1) assert ps.get_dimension(1) == Dimension(2) assert ps.get_dimension(2) == Dimension(3) assert ps.get_dimension(3) == Dimension.dynamic() ps = PartialShape([1, 2, 3, -1]) assert not ps.is_static assert ps.is_dynamic assert ps.all_non_negative assert ps.rank == 4 assert list(ps.get_min_shape()) == [1, 2, 3, 0] assert list(ps.get_max_shape())[3] > 1000000000 assert repr(ps) == "<PartialShape: {1,2,3,?}>" ps = PartialShape.dynamic() assert not ps.is_static assert ps.is_dynamic assert ps.rank == Dimension.dynamic() assert list(ps.get_min_shape()) == [] assert list(ps.get_max_shape()) == [] assert repr(ps) == "<PartialShape: ...>" ps = PartialShape.dynamic(rank=Dimension(2)) assert not ps.is_static assert ps.is_dynamic assert ps.rank == 2 assert 2 == ps.rank assert list(ps.get_min_shape()) == [0, 0] assert list(ps.get_max_shape())[0] > 1000000000 assert repr(ps) == "<PartialShape: {?,?}>" shape_list = [(1, 10), [2, 5], 4, Dimension(2), "..10"] ref_ps = PartialShape([ Dimension(1, 10), Dimension(2, 5), Dimension(4), Dimension(2), Dimension(-1, 10) ]) assert PartialShape(shape_list) == ref_ps assert PartialShape(tuple(shape_list)) == ref_ps with pytest.raises(TypeError) as e: PartialShape([(1, 2, 3)]) assert "Two elements are expected in tuple(lower, upper) " \ "for dynamic dimension, but 3 elements were given." in str(e.value) with pytest.raises(TypeError) as e: PartialShape([("?", "?")]) assert "Incorrect pair of types (<class 'str'>, <class 'str'>) " \ "for dynamic dimension, ints are expected." in str(e.value) with pytest.raises(TypeError) as e: PartialShape([range(10)]) assert "Incorrect type <class 'range'> for dimension. Expected types are: " \ "int, str, openvino.runtime.Dimension, list/tuple with lower " \ "and upper values for dynamic dimension." in str(e.value) ps = PartialShape("...") assert ps == PartialShape.dynamic() ps = PartialShape("?, 3, ..224, 28..224") assert ps == PartialShape( [Dimension(-1), Dimension(3), Dimension(-1, 224), Dimension(28, 224)]) with pytest.raises(RuntimeError) as e: ps = PartialShape("?,,3") assert 'Cannot get vector of dimensions! "?,,3" is incorrect' in str( e.value) shape = Shape() assert len(shape) == 0
def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd): """ Load input model and convert it to nGraph function :param: argv: parsed command line arguments :param: moc_front_end: Loaded Frontend for converting input model :return: converted nGraph function ready for serialization """ input_model = moc_front_end.load(argv.input_model) user_shapes, outputs, freeze_placeholder = fe_user_data_repack( input_model, argv.placeholder_shapes, argv.placeholder_data_types, argv.output, argv.freeze_placeholder_with_value, moc_front_end.get_name()) def check_places_are_same(places_original: List[Place], places_new: List[Place]): """ Check if set of new places is same as original or not. :param places_original: List[Place] Original model places :param places_new: List[Place] New list of places :return: True if new list of places is same as original """ return len(places_original) == len(places_new) and len([ item for item in places_original if any([item.is_equal(item2['node']) for item2 in places_new]) ]) == len(places_original) def add_names_to_tensors(model: InputModel, places: List[Place]): """ Adds additional names to some model input tensors. This helper should be used when a model modification is going to happen. :param model The input model loaded by a given frontend :param places An object containing Places and names that will be used for model modification """ for new_input in places: if not hasattr(new_input, 'input_name'): continue try: model.add_name_for_tensor(new_input['node'], new_input['input_name']) except NotImplementedFailure as e: # some frontends might not implement this method log.warn( 'Could not add an additional name to a tensor pointed to by \'{}\'. Details: {}' .format(new_input['input_name'], str(e))) enabled_transforms, disabled_transforms = get_enabled_and_disabled_transforms( ) if 'ANALYSIS_JSON_PRINT' in enabled_transforms: # NOTE that model analysis is performed before applying user's settings (inputs's shapes etc.) framework_model = moc_front_end.decode(input_model) json_model_analysis_dump(framework_model) # a model is not processed further in json analysis mode sys.exit(0) inputs_equal = True if user_shapes: inputs_equal = check_places_are_same(input_model.get_inputs(), user_shapes) outputs_equal = True if outputs: outputs_equal = check_places_are_same(input_model.get_outputs(), outputs) log.debug('Inputs are same: {}, outputs are same: {}'.format( inputs_equal, outputs_equal)) if not inputs_equal and not outputs_equal: log.debug('Using extract subgraph') new_input_places = [x['node'] for x in user_shapes] new_output_places = [x['node'] for x in outputs] add_names_to_tensors(input_model, user_shapes) input_model.extract_subgraph(new_input_places, new_output_places) # invalidation of existing Place objects could have happened in the operation above if user_shapes: user_shapes, outputs, freeze_placeholder = fe_user_data_repack( input_model, argv.placeholder_shapes, argv.placeholder_data_types, argv.output, argv.freeze_placeholder_with_value, moc_front_end.get_name()) elif not inputs_equal: log.debug('Using override_all_inputs') add_names_to_tensors(input_model, user_shapes) new_input_places = [x['node'] for x in user_shapes] input_model.override_all_inputs(new_input_places) # invalidation of existing Place objects could have happened in the operation above names = [place.get_names()[0] for place in new_input_places] shapes = [shape for shape in argv.placeholder_shapes.values()] # we have to update names used to find nodes, since the original # ones where cut off the graph placeholder_shapes = dict(zip(names, shapes)) if user_shapes: user_shapes, outputs, freeze_placeholder = fe_user_data_repack( input_model, placeholder_shapes, argv.placeholder_data_types, argv.output, argv.freeze_placeholder_with_value, moc_front_end.get_name()) elif not outputs_equal: log.debug('Using override_all_outputs') add_names_to_tensors(input_model, user_shapes) new_output_places = [x['node'] for x in outputs] input_model.override_all_outputs(new_output_places) if user_shapes: for user_shape in user_shapes: if user_shape.get('shape') is not None: input_model.set_partial_shape( user_shape['node'], partial_shape_from_tuple(user_shape['shape'])) if user_shape.get('data_type') is not None: data_type = get_element_type(user_shape['data_type']) log.debug('Set data type: {}'.format(data_type)) input_model.set_element_type(user_shape['node'], data_type) def shape_to_array(shape: PartialShape): return [shape.get_dimension(i) for i in range(shape.rank.get_length())] # Set batch size if argv.batch is not None and argv.batch > 0: log.debug('Setting batch size to {}'.format(argv.batch)) for place in input_model.get_inputs(): old_partial_shape = input_model.get_partial_shape(place) old_shape_array = shape_to_array( old_partial_shape) if old_partial_shape.rank.is_static else [] joined_name = ' '.join(place.get_names()) validate_batch_in_shape(old_shape_array, joined_name) # Assume batch size is always 1-st dimension in shape # Keep other dimensions unchanged new_shape = [ old_partial_shape.get_dimension(i) for i in range(old_partial_shape.rank.get_length()) ] new_shape[0] = Dimension(argv.batch) new_partial_shape = PartialShape(new_shape) log.debug('Input: {}, Old shape: {}, New shape: {}'.format( joined_name, old_shape_array, new_shape)) input_model.set_partial_shape(place, new_partial_shape) ngraph_function = moc_front_end.convert(input_model) return ngraph_function
def test_dimension_comparisons(): d1 = Dimension.dynamic() d2 = Dimension.dynamic() assert d1 == d2 assert d1 == -1 assert d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert d2.relaxes(d1) assert d2.compatible(d1) assert d2.same_scheme(d1) d1 = Dimension.dynamic() d2 = Dimension(3) assert d1 != d2 assert d2 == 3 assert not d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert not d2.relaxes(d1) assert d2.compatible(d1) assert not d2.same_scheme(d1) d1 = Dimension(3) d2 = Dimension(3) assert d1 == d2 assert d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert d2.relaxes(d1) assert d2.compatible(d1) assert d2.same_scheme(d1) d1 = Dimension(4) d2 = Dimension(3) assert d1 != d2 assert not d1.refines(d2) assert not d1.relaxes(d2) assert not d2.refines(d1) assert not d2.relaxes(d1) assert not d2.compatible(d1) assert not d2.same_scheme(d1) d = Dimension("?") assert d == Dimension() d = Dimension("1") assert d == Dimension(1) d = Dimension("..10") assert d == Dimension(-1, 10) d = Dimension("10..") assert d == Dimension(10, -1) d = Dimension("5..10") assert d == Dimension(5, 10) with pytest.raises(RuntimeError) as e: d = Dimension("C") assert 'Cannot parse dimension: "C"' in str(e.value) with pytest.raises(RuntimeError) as e: d = Dimension("?..5") assert 'Cannot parse min bound: "?"' in str(e.value) with pytest.raises(RuntimeError) as e: d = Dimension("5..?") assert 'Cannot parse max bound: "?"' in str(e.value)
def test_dimension_comparisons(): d1 = Dimension.dynamic() d2 = Dimension.dynamic() assert d1 == d2 assert d1 == -1 assert d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert d2.relaxes(d1) assert d2.compatible(d1) assert d2.same_scheme(d1) d1 = Dimension.dynamic() d2 = Dimension(3) assert d1 != d2 assert d2 == 3 assert not d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert not d2.relaxes(d1) assert d2.compatible(d1) assert not d2.same_scheme(d1) d1 = Dimension(3) d2 = Dimension(3) assert d1 == d2 assert d1.refines(d2) assert d1.relaxes(d2) assert d2.refines(d1) assert d2.relaxes(d1) assert d2.compatible(d1) assert d2.same_scheme(d1) d1 = Dimension(4) d2 = Dimension(3) assert d1 != d2 assert not d1.refines(d2) assert not d1.relaxes(d2) assert not d2.refines(d1) assert not d2.relaxes(d1) assert not d2.compatible(d1) assert not d2.same_scheme(d1)
def get_inputs_info(shape_string, data_shape_string, layout_string, batch_size, scale_string, mean_string, inputs): input_names = get_input_output_names(inputs) shape_map = parse_input_parameters(shape_string, input_names) data_shape_map = get_data_shapes_map(data_shape_string, input_names) layout_map = parse_input_parameters(layout_string, input_names) batch_size = parse_batch_size(batch_size) reshape = False batch_found = False input_info = [] for i in range(len(inputs)): info = AppInputInfo() # Input name info.name = input_names[i] # Input precision info.element_type = inputs[i].element_type # Shape info.original_shape = inputs[i].partial_shape if info.name in shape_map.keys(): info.partial_shape = parse_partial_shape(shape_map[info.name]) reshape = True else: info.partial_shape = inputs[i].partial_shape # Layout if info.name in layout_map.keys(): info.layout = Layout(layout_map[info.name]) elif inputs[i].node.layout != Layout(): info.layout = inputs[i].node.layout else: image_colors_dim = Dimension(3) shape = info.partial_shape num_dims = len(shape) if num_dims == 4: if(shape[1]) == image_colors_dim: info.layout = Layout("NCHW") elif(shape[3] == image_colors_dim): info.layout = Layout("NHWC") elif num_dims == 3: if(shape[0]) == image_colors_dim: info.layout = Layout("CHW") elif(shape[2] == image_colors_dim): info.layout = Layout("HWC") # Update shape with batch if needed if batch_size != 0: if batch_size.is_static and data_shape_map: logger.warning(f"Batch size will be ignored. Provide batch deminsion in data_shape parameter.") else: batch_index = -1 if info.layout.has_name('N'): batch_index = info.layout.get_index_by_name('N') elif info.layout == Layout(): supposed_batch = info.partial_shape[0] if supposed_batch.is_dynamic or supposed_batch in [0, 1]: logger.warning(f"Batch dimension is not specified for input '{info.name}'. " "The first dimension will be interpreted as batch size.") batch_index = 0 info.layout = Layout("N...") if batch_index != -1 and info.partial_shape[batch_index] != batch_size: info.partial_shape[batch_index] = batch_size reshape = True batch_found = True elif batch_index == -1 and not batch_found and i == len(inputs) - 1: raise Exception(f"Batch dimension is not specified for this model!") # Data shape if info.name in data_shape_map.keys() and info.is_dynamic: for p_shape in data_shape_map[info.name]: if p_shape.is_dynamic: raise Exception(f"Data shape always should be static, {str(p_shape)} is dynamic.") elif info.partial_shape.compatible(p_shape): info.data_shapes.append(p_shape.to_shape()) else: raise Exception(f"Data shape '{str(p_shape)}' provided for input '{info.name}' " f"is not compatible with partial shape '{str(info.partial_shape)}' for this input.") elif info.name in data_shape_map.keys(): logger.warning(f"Input '{info.name}' has static shape. Provided data shapes for this input will be ignored.") input_info.append(info) # Update scale, mean scale_map = parse_scale_or_mean(scale_string, input_info) mean_map = parse_scale_or_mean(mean_string, input_info) for input in input_info: if input.name in scale_map: input.scale = scale_map[input.name] if input.name in mean_map: input.mean = mean_map[input.name] return input_info, reshape
def test_dimension(): dim = Dimension() assert dim.is_dynamic assert not dim.is_static assert repr(dim) == "<Dimension: ?>" dim = Dimension.dynamic() assert dim.is_dynamic assert not dim.is_static assert repr(dim) == "<Dimension: ?>" dim = Dimension(10) assert dim.is_static assert len(dim) == 10 assert dim.get_length() == 10 assert dim.get_min_length() == 10 assert dim.get_max_length() == 10 assert repr(dim) == "<Dimension: 10>" dim = Dimension(5, 15) assert dim.is_dynamic assert dim.get_min_length() == 5 assert dim.get_max_length() == 15 assert repr(dim) == "<Dimension: 5..15>"
def moc_pipeline(argv: argparse.Namespace, moc_front_end: FrontEnd): """ Load input model and convert it to nGraph function :param: argv: parsed command line arguments :param: moc_front_end: Loaded Frontend for converting input model :return: converted nGraph function ready for serialization """ input_model = moc_front_end.load(argv.input_model) user_shapes, outputs, freeze_placeholder = fe_user_data_repack( input_model, argv.placeholder_shapes, argv.placeholder_data_types, argv.output, argv.freeze_placeholder_with_value) def check_places_are_same(places_original: List[Place], places_new: List[Place]): """ Check if set of new places is same as original or not. :param places_original: List[Place] Original model places :param places_new: List[Place] New list of places :return: True if new list of places is same as original """ return len(places_original) == len(places_new) and len([ item for item in places_original if any([item.is_equal(item2['node']) for item2 in places_new]) ]) == len(places_original) inputs_equal = True if user_shapes: inputs_equal = check_places_are_same(input_model.get_inputs(), user_shapes) outputs_equal = True if outputs: outputs_equal = check_places_are_same(input_model.get_outputs(), outputs) log.debug('Inputs are same: {}, outputs are same: {}'.format( inputs_equal, outputs_equal)) if not inputs_equal and not outputs_equal: # Use ExtractSubgraph new_input_places = [x['node'] for x in user_shapes] new_output_places = [x['node'] for x in outputs] log.debug('Using extract subgraph') input_model.extract_subgraph(new_input_places, new_output_places) elif not inputs_equal: new_input_places = [x['node'] for x in user_shapes] log.debug('Using override_all_inputs') input_model.override_all_inputs(new_input_places) elif not outputs_equal: new_output_places = [x['node'] for x in outputs] log.debug('Using override_all_outputs') input_model.override_all_outputs(new_output_places) if user_shapes: for user_shape in user_shapes: if user_shape.get('shape') is not None: input_model.set_partial_shape( user_shape['node'], PartialShape(user_shape['shape'])) if user_shape.get('data_type') is not None: data_type = get_element_type(user_shape['data_type']) log.debug('Set data type: {}'.format(data_type)) input_model.set_element_type(user_shape['node'], data_type) def shape_to_array(shape: PartialShape): return [shape.get_dimension(i) for i in range(shape.rank.get_length())] # Set batch size if argv.batch is not None and argv.batch > 0: log.debug('Setting batch size to {}'.format(argv.batch)) for place in input_model.get_inputs(): old_partial_shape = input_model.get_partial_shape(place) old_shape_array = shape_to_array( old_partial_shape) if old_partial_shape.rank.is_static else [] joined_name = ' '.join(place.get_names()) validate_batch_in_shape(old_shape_array, joined_name) # Assume batch size is always 1-st dimension in shape # Keep other dimensions unchanged new_shape = [ old_partial_shape.get_dimension(i) for i in range(old_partial_shape.rank.get_length()) ] new_shape[0] = Dimension(argv.batch) new_partial_shape = PartialShape(new_shape) log.debug('Input: {}, Old shape: {}, New shape: {}'.format( joined_name, old_shape_array, new_shape)) input_model.set_partial_shape(place, new_partial_shape) ngraph_function = moc_front_end.convert(input_model) return ngraph_function