def init(): """Initialize model Returns: model """ model_xml = "/usr/local/ev_sdk/model/openvino/yolov3.xml" if not os.path.isfile(model_xml): log.error(f'{model_xml} does not exist') return None model_bin = pathlib.Path(model_xml).with_suffix('.bin').as_posix() log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # Load Inference Engine log.info('Loading Inference Engine') global ie ie = IECore() global exec_net exec_net = ie.load_network(network=net, device_name=device) log.info('Device info:') versions = ie.get_versions(device) print("{}".format(device)) print("MKLDNNPlugin version ......... {}.{}".format(versions[device].major, versions[device].minor)) print("Build ........... {}".format(versions[device].build_number)) input_blob = next(iter(net.inputs)) n, c, h, w = net.inputs[input_blob].shape global input_h, input_w, input_c, input_n input_h, input_w, input_c, input_n = h, w, c, n return net
def boot_myriad(): myriad = {} model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # ------------ Load Inference Engine and network files ------------- log.info("Loading Inference Engine") ie = IECore() log.info("IR Engine info:") log.info(" Available IR Devices: {}".format(ie.available_devices)) if 'MYRIAD' in ie.available_devices: verinfo = ie.get_versions('MYRIAD')['MYRIAD'] log.info(" Myriad Plugin version: {}.{}".format( verinfo.major, verinfo.minor)) log.info(" Myriad Build: {}".format(verinfo.build_number)) devices = ie.get_metric(metric_name='AVAILABLE_DEVICES', device_name='MYRIAD') if (len(devices) > 1): ir_device = "MULTI:" for device in devices: ir_device += "MYRIAD." + device + "," ir_device = ir_device[:-1] else: ir_device = "MYRIAD" else: ir_device = "CPU" log.warn("!! MYRIAD DEVICE NOT FOUND. USING CPU !!") log.info(" Selected Device(s): {}".format(ir_device)) log.info("Loading network files XML [{}] and BIN [{}]".format( model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) net_input_layer_name = next(iter(net.inputs)) n, c, h, w = net.inputs[net_input_layer_name].shape log.info(" Network shape: [{},{},{},{}]".format(n, c, h, w)) log.info("Loading labels file [{}]".format(args.labels.name)) labels_map = [x.strip() for x in args.labels] log.info("Loading network to device... Please wait..") exec_net = ie.load_network(network=net, device_name=ir_device, num_requests=4) ir_num_requests = exec_net.get_metric('OPTIMAL_NUMBER_OF_INFER_REQUESTS') log.info(" Optimal Number of Infer Requests: {}".format(ir_num_requests)) log.info("Done loading network") myriad = { 'n': n, 'c': c, 'h': h, 'w': w, 'labels_map': labels_map, 'net': net, 'exec_net': exec_net, 'net_input_layer_name': net_input_layer_name, } return myriad
def test_get_version(device): ie = IECore() version = ie.get_versions(device) assert isinstance(version, dict), "Returned version must be a dictionary" assert device in version, "{} plugin version wasn't found in versions" assert hasattr(version[device], "major"), "Returned version has no field 'major'" assert hasattr(version[device], "minor"), "Returned version has no field 'minor'" assert hasattr(version[device], "description"), "Returned version has no field 'description'" assert hasattr(version[device], "build_number"), "Returned version has no field 'build_number'"
def __init__(self, model_xml, model_bin, input_names, output_names, vocab_file, device='MYRIAD'): self.context = None self.input_names = input_names self.output_names = output_names log.info("Loading vocab file:\t{}".format(vocab_file)) with open(vocab_file, "r", encoding="utf-8") as r: self.vocab = { t.rstrip("\n"): i for i, t in enumerate(r.readlines()) } log.info("{} tokens loaded".format(len(self.vocab))) log.info("Initializing Inference Engine") ie = IECore() ie.set_config({'VPU_HW_STAGES_OPTIMIZATION': 'NO'}, "MYRIAD") version = ie.get_versions(device)[device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) log.info("Plugin version is {}".format(version_str)) # read IR log.info("Loading network files:\n\t{}\n\t{}".format( model_xml, model_bin)) ie_encoder = ie.read_network(model=model_xml, weights=model_bin) # maximum number of tokens that can be processed by network at once self.max_length = ie_encoder.input_info[ self.input_names[0]].input_data.shape[1] # load model to the device log.info("Loading model to the {}".format(device)) self.ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=device)
def intelNeuralStick_init(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) ie = IECore() log.info("Device info:") versions = ie.get_versions(DEVICE_NAME) print("{}{}".format(" " * 10, DEVICE_NAME)) print("{}Plugin version ......... {}.{}".format( " " * 10, versions[DEVICE_NAME].major, versions[DEVICE_NAME].minor)) print("{}Build ........... {}".format(" " * 10, versions[DEVICE_NAME].build_number)) # Read IR objectDetection = intelNeuralStick_modelRead(MODEL_OBJECT_DETECTION, DEVICE_NAME, ie) faceDetection = intelNeuralStick_modelRead(MODEL_FACE_DETECTION, DEVICE_NAME, ie) genderAgeDetection = intelNeuralStick_modelRead( MODEL_AGE_GENDER_PREDICTION, DEVICE_NAME, ie) return objectDetection, faceDetection, genderAgeDetection
class LandmarkDetector: ''' Class for the Face Detection Model. ''' def __init__( self, model_name='models/intel/facial-landmarks-35-adas-0002/FP32-INT8/facial-landmarks-35-adas-0002', device='CPU', extensions=None): self.model_name = model_name self.model_w = model_name + '.bin' self.model_s = model_name + '.xml' self.device = device self.extension = extensions self.inference_results = None self.pre_image = None self.post_image = None self.network = None self.input_name = None self.input_shape = None self.output_shape = None self.output_name = None self.plugin = None self.exec_network = None def load_model(self): self.plugin = IECore() log.info("Attempting to load network for model:") log.info(self.model_name) self.network = self.plugin.read_network(model=self.model_s, weights=self.model_w) self.exec_network = self.plugin.load_network(self.network, self.device) self.input_name = next(iter(self.network.inputs)) self.output_name = next(iter(self.network.outputs)) self.input_shape = self.network.inputs[self.input_name].shape self.output_shape = self.network.outputs[self.output_name].shape def predict(self, image): self.pre_image = self.preprocess_input(image) self.inference_results = self.exec_network.infer( {self.input_name: self.pre_image}) landmarks = self.preprocess_output(self.inference_results, image) return landmarks def check_model(self): self.plugin = IECore() log.info("Checking model layers for model:") log.info(self.model_name) self.network = self.plugin.read_network(model=self.model_s, weights=self.model_w) # double check supported network layers # code taken from Project-01 (ND131) if "CPU" in self.device: supported_layers = self.plugin.query_network(self.network, "CPU") not_supported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(self.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") versions = self.plugin.get_versions(self.device) log.info("{}{}".format(" " * 8, self.device)) log.info("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[self.device].major, versions[self.device].minor)) log.info("{}Build ........... {}".format( " " * 8, versions[self.device].build_number)) def preprocess_input(self, image): post_frame = cv2.resize(image, (self.input_shape[3], self.input_shape[2])) post_frame = post_frame.transpose((2, 0, 1)) post_frame = post_frame.reshape(1, *post_frame.shape) input_width = post_frame.shape[1] input_height = post_frame.shape[0] log.debug("input width x height: %d x %d", self.input_shape[3], self.input_shape[2]) return post_frame def preprocess_output(self, outputs, image): ''' We only care about the eye coordinates for this project, parse the eye's (X,Y) positions and repackage for further processing ''' width = image.shape[1] height = image.shape[0] outputs = outputs[self.output_name][0] ##Left Eye Position (X,Y) eye_l_x = int(outputs[0] * width) eye_l_y = int(outputs[1] * height) ##Right Eye Position (X,Y) eye_r_x = int(outputs[2] * width) eye_r_y = int(outputs[3] * height) nose_x = int(outputs[4] * width) nose_y = int(outputs[5] * height) lip_l_x = int(outputs[6] * width) lip_l_y = int(outputs[7] * height) lip_r_x = int(outputs[8] * width) lip_r_y = int(outputs[9] * height) ##used for later drawing result = { 'eye_left': [eye_l_x, eye_l_y], 'eye_right': [eye_r_x, eye_r_y], 'nose': [nose_x, nose_y], 'lip_left': [lip_l_x, lip_l_y], 'lip_right': [lip_r_x, lip_r_y] } return result
class InferenceEngine: def __init__(self, model_xml, device, stride): self.device = device self.stride = stride model_bin = os.path.splitext(model_xml)[0] + '.bin' log.info("Loading network files:\n\t{}\n\t{}".format( model_xml, model_bin)) self.net = IENetwork(model=model_xml, weights=model_bin) log.info("Loading Inference Engine") self.ie = IECore() log.info("Device info:") versions = self.ie.get_versions(device) log.info("{}{}".format(" " * 8, device)) log.info("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[device].major, versions[device].minor)) log.info("{}Build ........... {}".format( " " * 8, versions[device].build_number)) self.input_blob = next(iter(self.net.inputs)) log.info( f"Input blob: {self.input_blob} - shape: {self.net.inputs[self.input_blob].shape}" ) for o in self.net.outputs.keys(): log.info(f"Output blob: {o} - shape: {self.net.outputs[o].shape}") if o == "Mconv7_stage2_L2": self.heatmaps_blob = "Mconv7_stage2_L2" self.pafs_blob = "Mconv7_stage2_L1" elif o == "heatmaps": self.heatmaps_blob = "heatmaps" self.pafs_blob = "pafs" log.info( f"Heatmaps blob: {self.heatmaps_blob} - PAFs blob: {self.pafs_blob}" ) log.info("Loading model to the plugin") self.exec_net = self.ie.load_network(network=self.net, num_requests=1, device_name=device) def infer(self, img): img = img[0:img.shape[0] - (img.shape[0] % self.stride), 0:img.shape[1] - (img.shape[1] % self.stride)] n, c, h, w = self.net.inputs[self.input_blob].shape if h != img.shape[0] or w != img.shape[1]: log.info(f"Reshaping of network") self.net.reshape( {self.input_blob: (n, c, img.shape[0], img.shape[1])}) log.info( f"Input blob: {self.input_blob} - new shape: {self.net.inputs[self.input_blob].shape}" ) for o in self.net.outputs.keys(): log.info( f"Output blob: {o} - new shape: {self.net.outputs[o].shape}" ) self.exec_net = self.ie.load_network(network=self.net, num_requests=1, device_name=self.device) img = np.transpose(img, (2, 0, 1))[None, ] inference_result = self.exec_net.infer(inputs={self.input_blob: img}) inference_result = (inference_result[self.heatmaps_blob][0], inference_result[self.pafs_blob][0]) return inference_result
class Classification: def __init__(self, device: str, number_infer_requests, number_iterations, duration_seconds, api_type): self.device = device self.ie = IECore() self.nireq = number_infer_requests self.niter = number_iterations self.duration_seconds = get_duration_seconds(duration_seconds, self.niter, self.device) self.api_type = api_type self.device_number_streams = {} def __del__(self): del self.ie def add_extension(self, path_to_extension: str=None, path_to_cldnn_config: str=None): if GPU_DEVICE_NAME in self.device: if path_to_cldnn_config: self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config}, GPU_DEVICE_NAME) logger.info('GPU extensions is loaded {}'.format(path_to_cldnn_config)) if CPU_DEVICE_NAME in self.device or MYRIAD_DEVICE_NAME in self.device: if path_to_extension: self.ie.add_extension(extension_path=path_to_extension, device_name=CPU_DEVICE_NAME) logger.info('CPU extensions is loaded {}'.format(path_to_extension)) def get_version_info(self) -> str: logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format('', 'API version', get_version())) version_string = 'Device info\n' for device, version in self.ie.get_versions(self.device).items(): version_string += '{: <9}{}\n'.format('', device) version_string += '{: <9}{:.<24}{} {}.{}\n'.format('', version.description, ' version', version.major, version.minor) version_string += '{: <9}{:.<24} {}\n'.format('', 'Build', version.build_number) return version_string @staticmethod def reshape(ie_network: IENetwork, batch_size: int): new_shapes = {} for input_layer_name, input_layer in ie_network.inputs.items(): new_shapes[input_layer_name] = get_blob_shape(input_layer, batch_size) if new_shapes: logger.info('Resizing network to batch = {}'.format(batch_size)) ie_network.reshape(new_shapes) def set_config(self, number_streams: int, api_type: str = 'async', number_threads: int = None, infer_threads_pinning: int = None): devices = parse_devices(self.device) self.device_number_streams = parse_nstreams_value_per_device(devices, number_streams) for device_name in self.device_number_streams.keys(): key = device_name + "_THROUGHPUT_STREAMS" supported_config_keys = self.ie.get_metric(device_name, 'SUPPORTED_CONFIG_KEYS') if key not in supported_config_keys: raise Exception("Device " + device_name + " doesn't support config key '" + key + "'! " + "Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>"); for device in devices: if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys # limit threading for CPU portion of inference if number_threads: self.ie.set_config({'CPU_THREADS_NUM': str(number_threads)}, device) if MULTI_DEVICE_NAME in self.device and GPU_DEVICE_NAME in self.device: self.ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME) else: # pin threads for CPU portion of inference self.ie.set_config({'CPU_BIND_THREAD': infer_threads_pinning}, device) # for CPU execution, more throughput-oriented execution via streams # for pure CPU execution, more throughput-oriented execution via streams if api_type == 'async': cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'} if device in self.device_number_streams.keys(): cpu_throughput['CPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device)) self.ie.set_config(cpu_throughput, device) self.device_number_streams[device] = self.ie.get_config(device, 'CPU_THROUGHPUT_STREAMS') elif device == GPU_DEVICE_NAME: if api_type == 'async': gpu_throughput = {'GPU_THROUGHPUT_STREAMS': 'GPU_THROUGHPUT_AUTO'} if device in self.device_number_streams.keys(): gpu_throughput['GPU_THROUGHPUT_STREAMS'] = str(self.device_number_streams.get(device)) self.ie.set_config(gpu_throughput, device) self.device_number_streams[device] = self.ie.get_config(device, 'GPU_THROUGHPUT_STREAMS') if MULTI_DEVICE_NAME in self.device and CPU_DEVICE_NAME in self.device: # multi-device execution with the CPU+GPU performs best with GPU trottling hint, # which releases another CPU thread (that is otherwise used by the GPU driver for active polling) self.ie.set_config({'CLDNN_PLUGIN_THROTTLE': '1'}, device) elif device == MYRIAD_DEVICE_NAME: self.ie.set_config({'LOG_LEVEL': 'LOG_INFO'}, MYRIAD_DEVICE_NAME) def read_network(self, path_to_model: str): xml_filename = os.path.abspath(path_to_model) head, tail = os.path.splitext(xml_filename) bin_filename = os.path.abspath(head + BIN_EXTENSION) ie_network = self.ie.read_network(xml_filename, bin_filename) input_info = ie_network.inputs if not input_info: raise AttributeError('No inputs info is provided') return ie_network def load_network(self, ie_network: IENetwork, perf_counts: bool): config = {'PERF_COUNT': ('YES' if perf_counts else 'NO')} exe_network = self.ie.load_network(ie_network, self.device, config=config, num_requests=1 if self.api_type == 'sync' else self.nireq or 0) # Number of requests self.nireq = len(exe_network.requests) return exe_network def infer(self, exe_network, batch_size, progress_bar=None): progress_count = 0 infer_requests = exe_network.requests # warming up - out of scope if self.api_type == 'sync': infer_requests[0].infer() else: infer_requests[0].async_infer() status = exe_network.wait() if status != StatusCode.OK: raise Exception("Wait for all requests is failed with status code {}!".format(status)) out_blob = next(iter(exe_network.outputs)) start_time = datetime.utcnow() exec_time = 0 iteration = 0 times = [] in_fly = set() # Start inference & calculate performance # to align number if iterations to guarantee that last infer requests are executed in the same conditions **/ #(self.duration_seconds and exec_time < self.duration_seconds) or \ while (self.niter and iteration < self.niter) or \ (len(in_fly) == iteration*self.nireq) or \ (self.api_type == 'async' and iteration % self.nireq): if self.api_type == 'sync': infer_requests[0].infer() times.append(infer_requests[0].latency) else: infer_request_id = exe_network.get_idle_request_id() if infer_request_id < 0: status = exe_network.wait(num_requests=1) if status != StatusCode.OK: raise Exception("Wait for idle request failed!") infer_request_id = exe_network.get_idle_request_id() if infer_request_id < 0: raise Exception("Invalid request id!") if infer_request_id in in_fly: times.append(infer_requests[infer_request_id].latency) else: in_fly.add(infer_request_id) infer_requests[infer_request_id].async_infer() #times.append(infer_requests[infer_request_id].latency) iteration += 1 exec_time = (datetime.utcnow() - start_time).total_seconds() if progress_bar: if self.duration_seconds: # calculate how many progress intervals are covered by current iteration. # depends on the current iteration time and time of each progress interval. # Previously covered progress intervals must be skipped. progress_interval_time = self.duration_seconds / progress_bar.total_num new_progress = int(exec_time / progress_interval_time - progress_count) progress_bar.add_progress(new_progress) progress_count += new_progress elif self.niter: progress_bar.add_progress(1) # wait the latest inference executions inference_output = [] status = exe_network.wait() for infer_request in infer_requests: output = infer_request.outputs[out_blob] inference_output.append(output.tolist()) if status != StatusCode.OK: raise Exception("Wait for all requests is failed with status code {}!".format(status)) total_duration_sec = (datetime.utcnow() - start_time).total_seconds() for infer_request_id in in_fly: times.append(infer_requests[infer_request_id].latency) times.sort() latency_ms = median(times) fps = batch_size * 1000 / latency_ms if self.api_type == 'sync' else batch_size * iteration / total_duration_sec if progress_bar: progress_bar.finish() return inference_output,fps, latency_ms, total_duration_sec, iteration
def setup(url): global vocab global ie_encoder global input_names global output_names global model global c_tokens_id global ie_encoder_exec global args global c_tokens_se global context global COLOR_RED global COLOR_RESET log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() if args.colors: COLOR_RED = "\033[91m" COLOR_RESET = "\033[0m" else: COLOR_RED = "" COLOR_RESET = "" # load vocabulary file for model log.info("Loading vocab file:\t{}".format(args.vocab)) vocab = load_vocab_file(args.vocab) log.info("{} tokens loaded".format(len(vocab))) # get context as a string (as we might need it's length for the sequence reshape) p = url paragraphs = get_paragraphs([p]) context = '\n'.join(paragraphs) log.info("Size: {} chars".format(len(context))) log.info("Context: " + COLOR_RED + context + COLOR_RESET) # encode context into token ids list c_tokens_id, c_tokens_se = text_to_tokens(context.lower(), vocab) log.info("Initializing Inference Engine") ie = IECore() version = ie.get_versions(args.device)[args.device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) log.info("Plugin version is {}".format(version_str)) # read IR model_xml = args.model model_bin = model_xml.with_suffix(".bin") log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) ie_encoder = ie.read_network(model=model_xml, weights=model_bin) if args.reshape: # reshape the sequence length to the context + maximum question length (in tokens) first_input_layer = next(iter(ie_encoder.inputs)) c = ie_encoder.inputs[first_input_layer].shape[1] # find the closest multiple of 64, if it is smaller than current network's sequence length, let' use that seq = min(c, int(np.ceil((len(c_tokens_id) + args.max_question_token_num) / 64) * 64)) if seq < c: input_info = list(ie_encoder.inputs) new_shapes = dict([]) for i in input_info: n, c = ie_encoder.inputs[i].shape new_shapes[i] = [n, seq] log.info("Reshaped input {} from {} to the {}".format(i, ie_encoder.inputs[i].shape, new_shapes[i])) log.info("Attempting to reshape the network to the modified inputs...") try: ie_encoder.reshape(new_shapes) log.info("Successful!") except RuntimeError: log.error("Failed to reshape the network, please retry the demo without '-r' option") sys.exit(-1) else: log.info("Skipping network reshaping," " as (context length + max question length) exceeds the current (input) network sequence length") # check input and output names input_names = list(i.strip() for i in args.input_names.split(',')) output_names = list(o.strip() for o in args.output_names.split(',')) if ie_encoder.inputs.keys() != set(input_names) or ie_encoder.outputs.keys() != set(output_names): log.error("Input or Output names do not match") log.error(" The demo expects input->output names: {}->{}. " "Please use the --input_names and --output_names to specify the right names " "(see actual values below)".format(input_names, output_names)) log.error(" Actual network input->output names: {}->{}".format(list(ie_encoder.inputs.keys()), list(ie_encoder.outputs.keys()))) log.error(" Actual network input->output values: {}->{}".format(list(ie_encoder.inputs.values()), list(ie_encoder.outputs.values()))) raise Exception("Unexpected network input or output names") # load model to the device log.info("Loading model to the {}".format(args.device)) ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device)
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info("Loading Inference Engine") ie = IECore() log.info("Initializing plugin for {} device...".format(args.device)) plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir) model_xml = args.model #'face-detection-retail-0004.xml' model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = ie.read_network(model=model_xml, weights=model_bin) log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) log.info("Loading IR to the plugin...") exec_net = plugin.load(network=net, num_requests=2) if args.labels: with open(args.labels, 'r') as f: labels_map = [x.strip() for x in f] else: labels_map = None n, c, h, w = net.inputs[input_blob].shape del net #s3client = boto3.resource('s3') expiresIn = 5 * 24 * 3600 #expires recorded voice after 5 days cur_request_id = 0 next_request_id = 1 render_time = 0 last_epoch = time.time() cap = cv2.VideoCapture(0) cap.set(3, 640) cap.set(4, 480) ret, frame = cap.read() log.info("To close the application, press 'CTRL+C'") while cap.isOpened(): ret, frame = cap.read() if not ret: break initial_w = cap.get(3) initial_h = cap.get(4) # Main sync point: # in the truly Async mode we start the NEXT infer request, while waiting for the CURRENT to complete # in the regular mode we start the CURRENT request and immediately wait for it's completion inf_start = time.time() in_frame = cv2.resize(frame, (w, h)) in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((n, c, h, w)) exec_net.start_async(request_id=cur_request_id, inputs={input_blob: in_frame}) if exec_net.requests[cur_request_id].wait(-1) == 0: inf_end = time.time() det_time = inf_end - inf_start # Parse detection results of the current request res = exec_net.requests[cur_request_id].outputs[out_blob] #log.info("Processing output blobs") #print(res) for obj in res[0][0]: # Draw only objects when probability more than specified threshold if obj[2] > args.prob_threshold: xmin = int(obj[3] * initial_w) ymin = int(obj[4] * initial_h) xmax = int(obj[5] * initial_w) ymax = int(obj[6] * initial_h) class_id = int(obj[1]) # Draw box and label\class_id #color = (min(class_id * 12.5, 255), min(class_id * 7, 255), min(class_id * 5, 255)) color = (232, 35, 244) cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2) det_label = labels_map[class_id] if labels_map else str( class_id) cv2.putText( frame, det_label + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) diff = (time.time() - last_epoch) log.info("Object detected {:.1f} s".format( round(obj[2] * 100, 1))) if diff > 60: last_epoch = time.time() t = TempImage() cv2.imwrite(t.path, frame) #s3client.meta.client.upload_file(t.path, args.bucket, t.key ) #log.info("upload image to {}".format(signedUrl)) play_mp3('alert.wav') t.cleanup() inf_time_message = "Inference time: {:.3f} ms".format(det_time * 1000) render_time_message = "OpenCV rendering time: {:.3f} ms".format( render_time * 1000) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) render_start = time.time() cv2.imshow("Detection Results", frame) render_end = time.time() render_time = render_end - render_start key = cv2.waitKey(1) if key == 27: break cv2.destroyAllWindows() cap.release()
class facial_landmarks_detection: ''' Class for the facial_landmarks_detection. ''' def __init__(self, model, device='CPU', extensions=None): ''' TODO: Use this to set your instance variables. ''' self.model = model self.device = device self.extension = extensions self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None self.device_list = [] self.print_once = True self.network_input_shape = None self.filtered_result = [[]] # ====== Load model files, verify model, and get the input shap parameters at start ====== log.info( "<---------- class facial_landmark_detection model --------->") self.load_model() self.check_model() self.network_input_shape = self.get_input_shape() def load_model(self): ''' TODO: You will need to complete this method. This method is for loading the model to the device specified by the user. If your model requires any Plugins, this is where you can load them. ''' self.plugin = IECore() log.info("------device avaibility--------") for available_devices in self.plugin.available_devices: #Dont use device variable, conflicts. self.device_list.append(available_devices) log.info("Available device: " + str(self.device_list)) #get name of available devices log.info("---------Plugin version--------") ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info log.info("descr: maj.min.num" + str(ver.description) + "." + str(ver.major) + "." + str(ver.minor) + "." + str(ver.build_number)) ### Load IR files into their related class model_xml = self.model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Check if path is not given or model name doesnt ends with .xml if model_xml is not None and model_xml.find('.xml') != -1: f, s = model_xml.rsplit( ".", 1 ) #check from last "." and "r"split only one element from last model_bin = f + ".bin" else: log.error( "Error! Model files are not found or invalid, check paths") log.error("Program stopped") sys.exit() #exit program no further execution log.info("-------------Model path----------") log.info("XML: " + str(model_xml)) log.info("bin: " + str(model_bin)) self.network = IENetwork(model=model_xml, weights=model_bin) log.info("ModelFiles are successfully loaded into IENetwork") return def check_model(self): ''' This function will check the input model for compatibility with hardware, It will check for supported and unsuported layers if any. No return required. ''' ### Check for supported layers ### log.info("Checking for supported Network layers...") # Query network will return all the layer, required all the time if device changes. supported_layers = self.plugin.query_network(network=self.network, device_name="CPU") log.info("------Status of default Network layers--------") log.info("No. of Layers in network: " + str(len(self.network.layers))) log.info("No. of supported layers:" + str(len(supported_layers))) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.info("Unsupported layers found:" + str(unsupported_layers)) log.info("CPU extension required and adding...") #exit(1) ### Adding any necessary extensions ### if self.extension and "CPU" in self.device: self.plugin.add_extension(self.extension, self.device) log.info("Checking for CPU extension compatibility...") # Again Query network will return fresh list of supported layers. supported_layers = self.plugin.query_network( network=self.network, device_name="CPU") log.info( "------Status of Network layers with CPU Extension--------" ) log.info("No. of Layers in network:" + str(len(self.network.layers))) log.info("No. of supported layers:" + str(len(supported_layers))) log.info("CPU extension added sucessfully!") unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: log.error("Unsupported layers found: " + str((unsupported_layers))) log.error("Error! Model not supported, Program stopped") exit() else: log.error("Error! cpu extension not found") log.error("Program stopped") exit() else: log.info("All the layers are supported, No CPU extension required") # This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference self.exec_network = self.plugin.load_network(self.network, "CPU") print("IR successfully loaded into Inference Engine") return def get_input_shape(self): ''' Get dimensions of inputs ''' ### Get the input layer informations ### log.info("-----Accessing input layer information-----") log.info('facial_landmarks_detection model Network input layers = ' + str(list(self.network.inputs))) log.info('facial_landmarks_detection Network input layers type: ' + str(type(self.network.inputs))) self.input_blob = next(iter(self.network.inputs)) #Origional log.info("-------------------------------") return self.network.inputs[self.input_blob].shape #Origional def wait(self): ### Wait for the Async request to be complete. ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### Extract and return the output results self.output_blob = next(iter(self.network.outputs)) # First return the name of blob as dictionary and second output of first blob as Nd array return self.exec_network.requests[ 0].outputs, self.exec_network.requests[0].outputs[self.output_blob] def preprocess_input(self, input_frames_raw, network_input_shape_height, network_input_shape_width): ''' Before feeding the data into the model for inference, you might have to preprocess it. This function is where you can do that. ''' p_frame = cv2.resize( input_frames_raw, (network_input_shape_height, network_input_shape_width)) #Resize as per network input spec. p_frame = p_frame.transpose((2, 0, 1)) #swap channel cxhxw p_frame = p_frame.reshape( 1, *p_frame.shape) #add one axis 1 to make 4D shape for network input #print(p_frame.shape) #Debug output return p_frame def predict(self, input_frames_raw, input_frame_raw_height, input_frame_raw_width): ''' TODO: You will need to complete this method. This method is meant for running predictions on the input image or video frame. input: RBG image in jpg format or video frame. ''' # pre-process origional frame to match network inputs. p_frame = self.preprocess_input(input_frames_raw, self.network_input_shape[3], self.network_input_shape[2]) # Run Async inference self.exec_network.start_async( request_id=0, #Origional inputs={self.input_blob: p_frame}) # run inference # # Run sync inference # self.exec_network.infer({self.input_blob: p_frame}) # run inference # wait until result available if self.wait() == 0: ### the results of the inference request ### blob, result = self.get_output( ) # origioinal for single blob outputs # Print available blob infirmation blob_list = [] if self.print_once: # Print only Once self.print_once = False for name, output_ in blob.items( ): #Find the possible BLOBS for name, blob_list.append(name) log.info( "The name of available blob of facial_landmarks_detection model is: " + str(blob_list)) return result
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### TODO: Initialize any class variables desired ### self.plugin = None self.network = None self.input_blob = None self.output_blob = None self.exec_network = None self.infer_request = None self.count_input_layers = 0 self.first_input_layer = None self.second_input_layer = None def load_model(self, model, device="CPU", cpu_extension=None): ### TODO: Load the model ### self.plugin = IECore() print("------device avaibility--------") for devices in self.plugin.available_devices: #Dont use device variable, conflicts. print("Available device:", devices) #get name of available devices print("---------Plugin version--------") ver = self.plugin.get_versions("CPU")["CPU"] # get plugin info print("{descr}: {maj}.{min}.{num}".format(descr=ver.description, maj=ver.major, min=ver.minor, num=ver.build_number)) ### Load IR files into their related class model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" # Check if path is not given or model name doesnt ends with .xml if model_xml is not None and model_xml.find('.xml') != -1: f, s = model_xml.rsplit( ".", 1 ) #check from last "." and "r"split only one element from last model_bin = f + ".bin" else: print("Error! Model files are not found or invalid, check paths") print("Program stopped") sys.exit() #exit program no further execution print("-------------Model path----------") print("XML:", model_xml) print("bin:", model_bin) self.network = IENetwork(model=model_xml, weights=model_bin) print("ModelFiles are successfully loaded into IENetwork") ### TODO: Check for supported layers ### print("Checking for supported Network layers...") # Query network will return all the layer, required all the time if device changes. supported_layers = self.plugin.query_network(network=self.network, device_name="CPU") print("------Status of default Network layers--------") print("No. of Layers in network", len(self.network.layers)) print("No. of supported layers:", len(supported_layers)) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: print("Unsupported layers found: {}".format(unsupported_layers)) print("CPU extension required and adding...") #exit(1) ### TODO: Add any necessary extensions ### #print(cpu_extension) #print(device) if cpu_extension and "CPU" in device: self.plugin.add_extension(cpu_extension, device) print("Checking for CPU extension compatibility...") # Again Query network will return fresh list of supported layers. supported_layers = self.plugin.query_network( network=self.network, device_name="CPU") print( "------Status of Network layers with CPU Extension--------" ) print("No. of Layers in network", len(self.network.layers)) print("No. of supported layers:", len(supported_layers)) unsupported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(unsupported_layers) != 0: print("Unsupported layers found: {}".format( unsupported_layers)) print("Error! Model not supported, Program stopped") exit() else: print("Error! cpu extension not found") print("Program stopped") exit() else: print("All the layers are supported, No CPU extension required") ### TODO: Return the loaded inference plugin ### # This will enable all following four functions ref:intel doc. ie_api.IECore Class Reference self.exec_network = self.plugin.load_network(self.network, "CPU") print("IR successfully loaded into Inference Engine") ### Note: You may need to update the function parameters. ### return def get_input_shape(self): ### TODO: Return the shape of the input layer ### # Get the input layer # To avoid unnecessory conversion in realtime in exec_net() and get_input_shape() self.count_input_layers = len(list(self.network.inputs)) print("-----Accessing input layer information-----") print('Network input layers = ' + str(list(self.network.inputs))) print('Network input layers type: ', type(self.network.inputs)) # FasterRcnn model Fix for Openvino V2019R3 if self.count_input_layers > 1: # check if more than 1 input layers # Model:TF-faster_rcnn_inception_v2_coco_2018_01_28 # Network input layers name = ['image_info', 'image_tensor'] # Access it by dictionary element # To avoid unnecessory conversion in realtime in exec_net() self.first_input_layer = list(self.network.inputs)[0] self.second_input_layer = list(self.network.inputs)[1] #print("Var:",self.first_input_layer,self.second_input_layer) #Debug print("More than one input layers found") print("### Warning!!! Manual data feed may require... ###") print("Read respective model documentation for more info.") self.input_blob = self.network.inputs[self.second_input_layer] print("Manually selected input layer: ", self.second_input_layer) print("Fixed data applied to other input layer: ", self.first_input_layer, ":", [600, 1024, 1]) print("in function exec_net()") print("-------------------------------") return self.input_blob.shape else: # If regular model self.input_blob = next(iter(self.network.inputs)) #Origional print("-------------------------------") return self.network.inputs[self.input_blob].shape #Origional def exec_net(self, image): ### TODO: Start an asynchronous request ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### # FasterRcnn model Fix for Openvino V2019R3 if self.count_input_layers > 1: # check if more than 1 input layers # Model:TF-faster_rcnn_inception_v2_coco_2018_01_28 # Network input layers name = ['image_info', 'image_tensor'] # Access it by dictionary element # Manually feed data according to model documentation # inputs layes names={'image_info':[600,1024,1],'image_tensor': image}) # image_info:[HxWxS], height, width and scale factor usually 1 self.exec_network.start_async(request_id=0, inputs={ self.first_input_layer: [600, 1024, 1], self.second_input_layer: image }) else: # If regular model self.exec_network.start_async( request_id=0, #Origional inputs={self.input_blob: image}) return def wait(self): ### TODO: Wait for the request to be complete. ### ### TODO: Return any necessary information ### ### Note: You may need to update the function parameters. ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### TODO: Extract and return the output results ### Note: You may need to update the function parameters. ### self.output_blob = next(iter(self.network.outputs)) # First return the name of blob as dictionary and second output of first blob as Nd array return self.exec_network.requests[ 0].outputs, self.exec_network.requests[0].outputs[self.output_blob]
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info("Loading Inference Engine") ie = IECore() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_name = splitext(basename(model_xml))[0] model_bin = model_xml[:-3] + 'bin' log.info("Loading network files:\n\t{}\n".format(model_xml)) net = ie.read_network(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- tab = pd.DataFrame(data=d) tab.to_csv(globalize(f"{args.tab_base}_{model_name}.csv", "C:/frames/")) # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Device info:") device = 'CPU' versions = ie.get_versions(device) print("{}{}".format(" " * 8, device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[device].major, versions[device].minor)) print("{}Build ........... {}".format(" " * 8, versions[device].build_number)) supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- infos = [*net.input_info] print("inputs number: " + str(len(infos))) print("input shape: " + str(net.input_info[infos[0]].input_data.shape)) print("input key: " + infos[0]) n, c, h, w = net.input_info[infos[0]].input_data.shape log.info("Preparing input blobs") out_blob = next(iter(net.outputs)) input_name = infos[0] log.info("Batch size is {}".format(net.batch_size)) net.input_info[infos[0]].precision = 'U8' log.info('Preparing output blobs') output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))] for output_key in net.outputs: if net.layers[output_key].type == "DetectionOutput": output_name, output_info = output_key, net.outputs[output_key] if output_name == "": log.error("Can't find a DetectionOutput layer in the topology") output_dims = output_info.shape if len(output_dims) != 4: log.error("Incorrect output dimensions for SSD model") max_proposal_count, object_size = output_dims[2], output_dims[3] if object_size != 7: log.error("Output item should have 7 as a last dimension") output_info.precision = "FP32" log.info("Loading model to the device") exec_net = ie.load_network(network=net, device_name=device) log.info("Creating infer request and starting inference") # ----------------------------------------------------------------------------------------------------- if args.video is not None and args.input is not None: raise RuntimeError('Either use video or images input') if args.video is None and args.input is None: raise RuntimeError('Need an input: video or images') has_video = False if args.video is not None: has_video = True log.info('Processing and ' + ('sav' if args.save else 'show') + 'ing ' + ('video' if has_video else 'images')) if args.save: if has_video: outdir = dirname(args.video) + sep + 'results' + sep else: outdir = args.input + sep + 'results' + sep makedirs(outdir, exist_ok=True) if has_video: cap = cv2.VideoCapture(args.video) fps = cap.get(cv2.CAP_PROP_FPS) video_length = cap.get(cv2.CAP_PROP_FRAME_COUNT) ret, image = cap.read() # first frame just to read size (...) size = (image.shape[1], image.shape[0]) if args.save: writer = cv2.VideoWriter(outdir + basename(args.video), cv2.VideoWriter_fourcc(*'mp4v'), fps, size) else: filenames = sorted(glob.glob(args.input + '/*.jpg'), key=getmtime) network_ratio = w / h count = -1 x1, y1, ws, hs, paths, objects, probas = [], [], [], [], [], [], [] while True: count += 1 if has_video: ret, image = cap.read() name = f'frame{count}' if count % 100 == 0: print('Progress: %.2f%%' % (100.0 * count / video_length), end='\r', flush=True) if count % 1000 == 0: log.info('Progress: %.2f%%' % (100.0 * count / video_length)) if not ret: break else: if count == len(filenames): break name = filenames[count] image = cv2.imread(name) output = [] ih, iw = image.shape[:-1] input_ratio = iw / ih if input_ratio < network_ratio: new_h = int(floor(w / input_ratio)) new_w = w scale_ratio = iw / w off_h = int(floor((new_h - h) / 2)) off_w = 0 else: new_h = h new_w = int(floor(h * input_ratio)) scale_ratio = ih / h off_h = 0 off_w = int(floor((new_w - w) / 2)) crop = cv2.resize(image, (new_w, new_h)) crop = crop[off_h:off_h + h, off_w:off_w + w, :] images_hw = crop.shape[:-1] data = { input_name: crop.transpose((2, 0, 1)) } # Change data layout from HWC to CHW res = exec_net.infer(inputs=data) res = res[out_blob][0][0] for number, proposal in enumerate(res): if proposal[2] > 0: ih, iw = images_hw label = np.int(proposal[1]) confidence = proposal[2] xmin = np.int(scale_ratio * (off_w + iw * proposal[3])) ymin = np.int(scale_ratio * (off_h + ih * proposal[4])) xmax = np.int(scale_ratio * (off_w + iw * proposal[5])) ymax = np.int(scale_ratio * (off_h + ih * proposal[6])) if confidence > args.confidence: output.append((xmin, ymin, xmax, ymax, label, basename(name), confidence)) """if not args.save: print("[{},{}] element, prob = {:.6} ({},{})-({},{})" \ .format(number, label, confidence, xmin, ymin, xmax, ymax))""" for box in output: if box[5] == 1: cl = (255, 0, 0) else: cl = (0, 0, 255) x, y, width, height = box[0], box[ 1], box[2] - box[0], box[3] - box[1] cv2.rectangle(image, (box[0], box[1]), (box[2], box[3]), cl, 2) multiple_append([x1, y1, ws, hs, objects, paths, probas], [x, y, width, height, box[4], box[5], box[6]]) if args.save: if has_video: writer.write(image) else: base = basename(name) log.info(f'Write to {outdir + base}') cv2.imwrite(outdir + 'detection_' + model_name + '_' + base, image) else: if args.hide: cv2.imshow('result', image) cv2.waitKey(0) d = { 'file': paths, 'object': objects, 'x1': x1, 'y1': y1, 'w': ws, 'h': hs, 'p': probas } tab = pd.DataFrame(data=d) tab.to_csv(args.tab_base, index_label='#') # ----------------------------------------------------------------------------------------------------- if has_video: cap.release() if args.save: writer.release() log.info("Execution successful\n")
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info('Loading Inference Engine') ie = IECore() # ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format --- model = args.model log.info(f'Loading network:') log.info(f' {model}') net = ie.read_network(model=model) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info('Device info:') versions = ie.get_versions(args.device) log.info(f' {args.device}') log.info( f' MKLDNNPlugin version ......... {versions[args.device].major}.{versions[args.device].minor}' ) log.info(f' Build ........... {versions[args.device].build_number}') # ----------------------------------------------------------------------------------------------------- # --------------------------- 3. Read and preprocess input -------------------------------------------- log.info(f'Inputs number: {len(net.input_info.keys())}') assert len(net.input_info.keys() ) == 1, 'Sample supports clean SSD network with one input' assert len(net.outputs.keys() ) == 1, 'Sample supports clean SSD network with one output' input_name = list(net.input_info.keys())[0] input_info = net.input_info[input_name] supported_input_dims = 4 # Supported input layout - NHWC log.info(f' Input name: {input_name}') log.info(f' Input shape: {str(input_info.input_data.shape)}') if len(input_info.input_data.layout) == supported_input_dims: n, c, h, w = input_info.input_data.shape assert n == 1, 'Sample supports topologies with one input image only' else: raise AssertionError('Sample supports input with NHWC shape only') image = cv2.imread(args.input) h_new, w_new = image.shape[:-1] images = np.ndarray(shape=(n, c, h_new, w_new)) log.info('File was added: ') log.info(f' {args.input}') image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW images[0] = image log.info( 'Reshaping the network to the height and width of the input image') net.reshape({input_name: [n, c, h_new, w_new]}) log.info( f'Input shape after reshape: {str(net.input_info["data"].input_data.shape)}' ) # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info('Preparing input blobs') if len(input_info.layout) == supported_input_dims: input_info.precision = 'U8' data = {} data[input_name] = images # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') func = ng.function_from_cnn(net) ops = func.get_ordered_ops() output_name, output_info = '', net.outputs[next(iter(net.outputs.keys()))] output_ops = {op.friendly_name : op for op in ops \ if op.friendly_name in net.outputs and op.get_type_name() == 'DetectionOutput'} if len(output_ops) == 1: output_name, output_info = output_ops.popitem() assert output_name != '', 'Can' 't find a DetectionOutput layer in the topology' output_dims = output_info.shape assert output_dims != 4, 'Incorrect output dimensions for SSD model' assert output_dims[3] == 7, 'Output item should have 7 as a last dimension' output_info.precision = 'FP32' # ----------------------------------------------------------------------------------------------------- # --------------------------- Performing inference ---------------------------------------------------- log.info('Loading model to the device') exec_net = ie.load_network(network=net, device_name=args.device) log.info('Creating infer request and starting inference') exec_result = exec_net.infer(inputs=data) # ----------------------------------------------------------------------------------------------------- # --------------------------- Read and postprocess output --------------------------------------------- log.info('Processing output blobs') result = exec_result[output_name] boxes = {} detections = result[0][0] # [0][0] - location of detections in result blob for number, proposal in enumerate(detections): imid, label, confidence, coords = np.int(proposal[0]), np.int( proposal[1]), proposal[2], proposal[3:] if confidence > 0.5: # correcting coordinates to actual image resolution xmin, ymin, xmax, ymax = w_new * coords[0], h_new * coords[ 1], w_new * coords[2], h_new * coords[3] log.info( f' [{number},{label}] element, prob = {confidence:.6f}, ' f'bbox = ({xmin:.3f},{ymin:.3f})-({xmax:.3f},{ymax:.3f}), batch id = {imid}' ) if not imid in boxes.keys(): boxes[imid] = [] boxes[imid].append([xmin, ymin, xmax, ymax]) imid = 0 # as sample supports one input image only, imid in results will always be 0 tmp_image = cv2.imread(args.input) for box in boxes[imid]: # drawing bounding boxes on the output image cv2.rectangle(tmp_image, (np.int(box[0]), np.int(box[1])), (np.int(box[2]), np.int(box[3])), color=(232, 35, 244), thickness=2) cv2.imwrite('out.bmp', tmp_image) log.info('Image out.bmp created!') # ----------------------------------------------------------------------------------------------------- log.info('Execution successful\n') log.info( 'This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool' )
return parser #log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() #log.info("Loading Inference Engine") ie = IECore() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" #log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = ie.read_network(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- #log.info("Device info:") versions = ie.get_versions(args.device) #print("{}{}".format(" " * 8, args.device)) #print("{}MKLDNNPlugin version ......... {}.{}".format(" " * 8, versions[args.device].major, versions[args.device].minor)) #print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) # CPU를 device로 선택했을때 if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers] if len(not_supported_layers) != 0: log.error("Following layers are not supported by the plugin for specified device {}:\n {}".
def main(): total_timer = Timer(name='total') total_timer.tic() log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 ), "Sample supports topologies only with 1 input" input_name = next(iter(net.inputs.keys())) input_info = net.inputs[input_name] input_info.precision = 'FP32' log.info('input shape: {}'.format(input_info.shape)) # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') assert (len(net.outputs.keys()) == 2 ), "Sample supports topologies only with 2 output" loc_out_name = "797" class_out_name = "741" assert (loc_out_name in net.outputs.keys()) and (class_out_name in net.outputs.keys()) loc_out_info = net.outputs[loc_out_name] class_out_info = net.outputs[class_out_name] loc_out_info.precision = "FP32" class_out_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------- log.info("Loading model to the device") # cpu_throughput = {'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'} ie.set_config({'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO'}, args.device) ie.set_config({'CPU_BIND_THREAD': 'YES'}, args.device) exec_net = ie.load_network(network=net, device_name=args.device, num_requests=0) infer_requests = exec_net.requests request_queue = InferRequestsQueue(infer_requests) log.info('nreqs: {}, nstream:{}'.format( len(infer_requests), ie.get_config(args.device, 'CPU_THROUGHPUT_STREAMS'))) # --------------------------- 3. Read and preprocess input -------------------------------------------- # ----------------------------------------------------------------------------------------------------- if not os.path.exists(args.result_dir): os.makedirs(args.result_dir) if args.voc_res_file and os.path.exists(args.voc_res_file): os.remove(args.voc_res_file) load_data_timer = Timer(name='load_data') post_process_timer = Timer(name='post_process') adapter = RetinaNetAdapter(input_shape=args.patch_size) # --------------------------- Performing inference ---------------------------------------------------- result_all_images = defaultdict(list) data_loader = DataLoader(args.image_dir, args.strides, args.patch_size) while True: load_data_timer.tic() input_data = data_loader.next() load_data_timer.toc() if input_data == None: break infer_request = request_queue.get_idle_request() if not infer_request: raise Exception('No idle Infer Requests!') if infer_request.cur_meta == None: infer_request.start_async(input_name, input_data) continue # get result post_process_timer.tic() image_name = infer_request.cur_meta['image_name'] x = infer_request.cur_meta['x'] y = infer_request.cur_meta['y'] loc_out = infer_request.request.outputs[loc_out_name][0] class_out = infer_request.request.outputs[class_out_name][0] ## start infer infer_request.start_async(input_name, input_data) ## post-process result = adapter.process(loc_out, class_out) result, _ = nms(result, thresh=0.5, keep_top_k=100) result[:, 0] += x result[:, 1] += y result[:, 2] += x result[:, 3] += y result_all_images[image_name].append(result) post_process_timer.toc() # wait the latest inference executions request_queue.wait_all() post_process_timer.tic() for infer_request in request_queue.requests: # get result image_name = infer_request.cur_meta['image_name'] x = infer_request.cur_meta['x'] y = infer_request.cur_meta['y'] loc_out = infer_request.request.outputs[loc_out_name][0] class_out = infer_request.request.outputs[class_out_name][0] ## post-process result = adapter.process(loc_out, class_out) result, _ = nms(result, thresh=0.5, keep_top_k=100) result[:, 0] += x result[:, 1] += y result[:, 2] += x result[:, 3] += y result_all_images[image_name].append(result) post_process_timer.toc() post_process_timer.tic() ## process total image result for image_name, result_per_image in result_all_images.items(): result_per_image = np.concatenate(result_per_image, axis=0) nms_result, _ = nms(result_per_image, thresh=0.5) voc_format = '{} {:.4f} {} {} {} {}' pos_all = [] voc_all = [] for i in range(nms_result.shape[0]): x = int(nms_result[i, 0]) y = int(nms_result[i, 1]) w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1) h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1) p = float(nms_result[i, 4]) pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p} pos_all.append(pos) if args.voc_res_file: xmin = x ymin = y xmax = int(nms_result[i, 2]) ymax = int(nms_result[i, 3]) voc_str = voc_format.format( os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax) voc_all.append(voc_str) file_name = os.path.splitext(image_name)[0] + '.json' with open(os.path.join(args.result_dir, file_name), 'w') as f: json.dump(pos_all, f) if args.voc_res_file: with open(args.voc_res_file, 'a') as f: for voc_str in voc_all: f.write(voc_str + '\n') post_process_timer.toc() total_timer.toc() # ----------------------------------------------------------------------------------------------------- all_timers = [] # all_timers.extend([create_anchor_timer, # read_img_timer, # preprocess_timer, # infer_timer, # adapter_timer, # patch_img_nms_timer, # whole_img_nms_timer, # add_offset_timer, # write_result_timer, # total_timer]) all_timers.extend([load_data_timer, post_process_timer, total_timer]) for timer in all_timers: log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format( timer.name, timer.avg * 1000, timer.total)) log.info('infer: {:2f}s'.format(request_queue.get_duration_in_seconds())) log.info("Execution successful\n")
def main(): total_timer = Timer(name='total') total_timer.tic() log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 ), "Sample supports topologies only with 1 input" input_name = next(iter(net.inputs.keys())) input_info = net.inputs[input_name] # input_info.precision = 'FP32' input_info.precision = 'U8' # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') assert (len(net.outputs.keys()) == 2 ), "Sample supports topologies only with 2 output" loc_out_name = args.loc_out_name class_out_name = args.class_out_name assert (loc_out_name in net.outputs.keys()) and (class_out_name in net.outputs.keys()) loc_out_info = net.outputs[loc_out_name] class_out_info = net.outputs[class_out_name] loc_out_info.precision = "FP32" class_out_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------- log.info("Loading model to the device") # ie.set_config({'CPU_THREADS_NUM': str(12)}, 'CPU') exec_net = ie.load_network(network=net, device_name=args.device) # # --------------------------- 3. Read and preprocess input -------------------------------------------- # # ----------------------------------------------------------------------------------------------------- if not os.path.exists(args.result_dir): os.makedirs(args.result_dir) if args.voc_res_file and os.path.exists(args.voc_res_file): os.remove(args.voc_res_file) # create_anchor_timer = Timer(name='create_anchor') # read_img_timer = Timer(name='read_img') # preprocess_timer = Timer(name='preprocess') # infer_timer = Timer(name='infer') # adapter_timer = Timer(name='adapter') # patch_img_nms_timer = Timer(name='patch_img_nms') # whole_img_nms_timer = Timer(name='whole_img_nms') # add_offset_timer = Timer(name='add_offset') # write_result_timer = Timer(name='write_result') queue = Queue() producer = Producer('Producer', queue, args) consumer = Consumer('Consumer', queue, exec_net, input_name, args) producer.start() consumer.start() producer.join() consumer.join() log.info('All threads finished!') total_timer.toc() # # ----------------------------------------------------------------------------------------------------- all_timers = [] # all_timers.extend([create_anchor_timer, # read_img_timer, # preprocess_timer, # infer_timer, # adapter_timer, # patch_img_nms_timer, # whole_img_nms_timer, # add_offset_timer, # write_result_timer, # total_timer]) all_timers.extend([total_timer]) for timer in all_timers: log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format( timer.name, timer.avg * 1000, timer.total)) log.info("Execution successful\n")
class Benchmark: def __init__(self, device: str, number_infer_requests, number_iterations, duration_seconds, api_type): self.device = device.upper() self.ie = IECore() self.nireq = number_infer_requests self.niter = number_iterations self.duration_seconds = get_duration_seconds(duration_seconds, self.niter, self.device) self.api_type = api_type self.device_number_streams = {} def __del__(self): del self.ie def add_extension(self, path_to_extension: str = None, path_to_cldnn_config: str = None): if GPU_DEVICE_NAME in self.device: if path_to_cldnn_config: self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config}, GPU_DEVICE_NAME) logger.info( 'GPU extensions is loaded {}'.format(path_to_cldnn_config)) if CPU_DEVICE_NAME in self.device or MYRIAD_DEVICE_NAME in self.device: if path_to_extension: self.ie.add_extension(extension_path=path_to_extension, device_name=CPU_DEVICE_NAME) logger.info( 'CPU extensions is loaded {}'.format(path_to_extension)) def get_version_info(self) -> str: logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format( '', 'API version', get_version())) version_string = 'Device info\n' for device, version in self.ie.get_versions(self.device).items(): version_string += '{: <9}{}\n'.format('', device) version_string += '{: <9}{:.<24}{} {}.{}\n'.format( '', version.description, ' version', version.major, version.minor) version_string += '{: <9}{:.<24} {}\n'.format( '', 'Build', version.build_number) return version_string @staticmethod def reshape(ie_network: IENetwork, batch_size: int): new_shapes = {} for input_layer_name, input_layer in ie_network.inputs.items(): shape = input_layer.shape layout = input_layer.layout try: batch_index = layout.index('N') except ValueError: batch_index = 1 if layout == 'C' else -1 if batch_index != -1 and shape[batch_index] != batch_size: shape[batch_index] = batch_size new_shapes[input_layer_name] = shape if new_shapes: logger.info('Resizing network to batch = {}'.format(batch_size)) ie_network.reshape(new_shapes) def set_config(self, number_streams: int, api_type: str = 'async', number_threads: int = None, infer_threads_pinning: int = None): devices = parse_devices(self.device) self.device_number_streams = parse_value_per_device( devices, number_streams) for device in devices: if device == CPU_DEVICE_NAME: # CPU supports few special performance-oriented keys # limit threading for CPU portion of inference if number_threads: self.ie.set_config( {'CPU_THREADS_NUM': str(number_threads)}, device) if MULTI_DEVICE_NAME in self.device and GPU_DEVICE_NAME in self.device: self.ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME) else: # pin threads for CPU portion of inference self.ie.set_config( {'CPU_BIND_THREAD': infer_threads_pinning}, device) # for CPU execution, more throughput-oriented execution via streams # for pure CPU execution, more throughput-oriented execution via streams if api_type == 'async': cpu_throughput = { 'CPU_THROUGHPUT_STREAMS': 'CPU_THROUGHPUT_AUTO' } if device in self.device_number_streams.keys(): cpu_throughput['CPU_THROUGHPUT_STREAMS'] = str( self.device_number_streams.get(device)) self.ie.set_config(cpu_throughput, device) self.device_number_streams[device] = self.ie.get_config( device, 'CPU_THROUGHPUT_STREAMS') elif device == GPU_DEVICE_NAME: if api_type == 'async': gpu_throughput = { 'GPU_THROUGHPUT_STREAMS': 'GPU_THROUGHPUT_AUTO' } if device in self.device_number_streams.keys(): gpu_throughput['GPU_THROUGHPUT_STREAMS'] = str( self.device_number_streams.get(device)) self.ie.set_config(gpu_throughput, device) self.device_number_streams[device] = self.ie.get_config( device, 'GPU_THROUGHPUT_STREAMS') if MULTI_DEVICE_NAME in self.device and CPU_DEVICE_NAME in self.device: # multi-device execution with the CPU+GPU performs best with GPU trottling hint, # which releases another CPU thread (that is otherwise used by the GPU driver for active polling) self.ie.set_config({'CLDNN_PLUGIN_THROTTLE': '1'}, device) elif device == MYRIAD_DEVICE_NAME: self.ie.set_config({'LOG_LEVEL': 'LOG_INFO'}, MYRIAD_DEVICE_NAME) def load_network(self, ie_network: IENetwork, perf_counts: bool, number_infer_requests: int = None): config = {'PERF_COUNT': ('YES' if perf_counts else 'NO')} exe_network = self.ie.load_network(ie_network, self.device, config=config, num_requests=number_infer_requests or 0) return exe_network def infer(self, request_queue, requests_input_data, batch_size, progress_bar): progress_count = 0 # warming up - out of scope infer_request = request_queue.get_idle_request() if not infer_request: raise Exception('No idle Infer Requests!') if self.api_type == 'sync': infer_request.infer(requests_input_data[infer_request.req_id]) else: infer_request.start_async( requests_input_data[infer_request.req_id]) request_queue.wait_all() request_queue.reset_times() start_time = datetime.now() exec_time = (datetime.now() - start_time).total_seconds() iteration = 0 # Start inference & calculate performance # to align number if iterations to guarantee that last infer requests are executed in the same conditions **/ while (self.niter and iteration < self.niter) or \ (self.duration_seconds and exec_time < self.duration_seconds) or \ (self.api_type == 'async' and iteration % self.nireq): infer_request = request_queue.get_idle_request() if not infer_request: raise Exception('No idle Infer Requests!') if self.api_type == 'sync': infer_request.infer(requests_input_data[infer_request.req_id]) else: infer_request.start_async( requests_input_data[infer_request.req_id]) iteration += 1 exec_time = (datetime.now() - start_time).total_seconds() if self.duration_seconds: # calculate how many progress intervals are covered by current iteration. # depends on the current iteration time and time of each progress interval. # Previously covered progress intervals must be skipped. progress_interval_time = self.duration_seconds / progress_bar.total_num new_progress = int(exec_time / progress_interval_time - progress_count) progress_bar.add_progress(new_progress) progress_count += new_progress elif self.niter: progress_bar.add_progress(1) # wait the latest inference executions request_queue.wait_all() total_duration_sec = request_queue.get_duration_in_seconds() times = request_queue.times times.sort() latency_ms = median(times) fps = batch_size * 1000 / latency_ms if self.api_type == 'async': fps = batch_size * iteration / total_duration_sec progress_bar.finish() return fps, latency_ms, total_duration_sec, iteration
def main(args=None): try: # ------------------------------ 1. Parsing and validating input arguments ------------------------------------- next_step() if not args: args = parse_args() # ------------------------------ 2. Loading Inference Engine --------------------------------------------------- next_step() device_name = args.target_device.upper() ie = IECore() if CPU_DEVICE_NAME in device_name: if args.path_to_extension: ie.add_cpu_extension(extension_path=args.path_to_extension, device_name=CPU_DEVICE_NAME) if GPU_DEVICE_NAME in device_name: if args.path_to_cldnn_config: ie.set_config({'CONFIG_FILE': args.path_to_cldnn_config}, GPU_DEVICE_NAME) logger.info("GPU extensions is loaded {}".format( args.path_to_cldnn_config)) logger.info("InferenceEngine:\n{: <9}{}".format("", get_version())) version_string = "Device is {}\n".format(device_name) for device, version in ie.get_versions(device_name).items(): version_string += "{: <9}{}\n".format("", device) version_string += "{: <9}{:.<24}{} {}.{}\n".format( "", version.description, " version", version.major, version.minor) version_string += "{: <9}{:.<24} {}\n".format( "", "Build", version.build_number) logger.info(version_string) # --------------------- 3. Read the Intermediate Representation of the network --------------------------------- next_step() xml_filename = os.path.abspath(args.path_to_model) head, tail = os.path.splitext(xml_filename) bin_filename = os.path.abspath(head + BIN_EXTENSION) ie_network = IENetwork(xml_filename, bin_filename) input_info = ie_network.inputs if len(input_info) == 0: raise AttributeError('No inputs info is provided') # --------------------- 4. Resizing network to match image sizes and given batch ------------------------------- next_step() batch_size = ie_network.batch_size precision = ie_network.precision if args.batch_size and args.batch_size != ie_network.batch_size: new_shapes = {} for key in input_info.keys(): shape = input_info[key].shape layout = input_info[key].layout batchIndex = -1 if ((layout == 'NCHW') or (layout == 'NCDHW') or (layout == 'NHWC') or (layout == 'NDHWC') or (layout == 'NC')): batchIndex = 0 elif (layout == 'CN'): batchIndex = 1 if ((batchIndex != -1) and (shape[batchIndex] != args.batch_size)): shape[batchIndex] = args.batch_size new_shapes[key] = shape if (len(new_shapes) > 0): logger.info("Resizing network to batch = {}".format( args.batch_size)) ie_network.reshape(new_shapes) batch_size = args.batch_size logger.info("Network batch size: {}, precision {}".format( batch_size, precision)) # --------------------- 5. Configuring input of the model ------------------------------------------------------ next_step() for key in input_info.keys(): if (isImage(input_info[key])): # Set the precision of input data provided by the user # Should be called before load of the network to the plugin input_info[key].precision = 'U8' # --------------------- 6. Setting device configuration -------------------------------------------------------- next_step() devices = parseDevices(device_name) device_nstreams = parseValuePerDevice(devices, args.number_streams) for device in devices: if device == CPU_DEVICE_NAME: ## CPU supports few special performance-oriented keys ## limit threading for CPU portion of inference if args.number_threads: ie.set_config( {'CPU_THREADS_NUM': str(args.number_threads)}, device) if MULTI_DEVICE_NAME in device_name and GPU_DEVICE_NAME in device_name: ie.set_config({'CPU_BIND_THREAD': 'NO'}, CPU_DEVICE_NAME) else: # pin threads for CPU portion of inference ie.set_config( {'CPU_BIND_THREAD': args.infer_threads_pinning}, device) ## for CPU execution, more throughput-oriented execution via streams # for pure CPU execution, more throughput-oriented execution via streams if args.api_type == 'async': ie.set_config( { 'CPU_THROUGHPUT_STREAMS': str(device_nstreams.get(device)) if device in device_nstreams.keys() else 'CPU_THROUGHPUT_AUTO' }, device) device_nstreams[device] = int( ie.get_config(device, 'CPU_THROUGHPUT_STREAMS')) elif device == GPU_DEVICE_NAME: if args.api_type == 'async': ie.set_config( { 'GPU_THROUGHPUT_STREAMS': str(device_nstreams.get(device)) if device in device_nstreams.keys() else 'GPU_THROUGHPUT_AUTO' }, device) device_nstreams[device] = int( ie.get_config(device, 'GPU_THROUGHPUT_STREAMS')) if MULTI_DEVICE_NAME in device_name and CPU_DEVICE_NAME in device_name: ## multi-device execution with the CPU+GPU performs best with GPU trottling hint, ## which releases another CPU thread (that is otherwise used by the GPU driver for active polling) ie.set_config({'CLDNN_PLUGIN_THROTTLE': str(1)}, device) elif device == MYRIAD_DEVICE_NAME: ie.set_config( { 'LOG_LEVEL': 'LOG_INFO', 'VPU_LOG_LEVEL': 'LOG_WARNING' }, MYRIAD_DEVICE_NAME) # --------------------- 7. Loading the model to the device ----------------------------------------------------- next_step() config = {'PERF_COUNT': ('YES' if args.perf_counts else 'NO')} exe_network = ie.load_network(ie_network, device_name, config=config, num_requests=args.number_infer_requests if args.number_infer_requests else 0) # --------------------- 8. Setting optimal runtime parameters -------------------------------------------------- next_step() ## Number of requests infer_requests = exe_network.requests nireq = len(infer_requests) ## Iteration limit niter = args.number_iterations if niter and args.api_type == 'async': niter = (int)((niter + nireq - 1) / nireq) * nireq if (args.number_iterations != niter): logger.warn( "Number of iterations was aligned by request number " "from {} to {} using number of requests {}".format( args.number_iterations, niter, nireq)) ## Time limit duration_seconds = 0 if args.time: ## time limit duration_seconds = args.time elif not args.number_iterations: ## default time limit duration_seconds = get_duration_in_secs(device) # ------------------------------------ 8. Creating infer requests and filling input blobs ---------------------- next_step() request_queue = InferRequestsQueue(infer_requests) path_to_input = os.path.abspath( args.path_to_input) if args.path_to_input else None requests_input_data = getInputs(path_to_input, batch_size, ie_network.inputs, infer_requests) # ------------------------------------ 9. Measuring performance ------------------------------------------------ progress_count = 0 progress_bar_total_count = 10000 output_string = "Start inference {}ronously".format(args.api_type) if (args.api_type == "async"): if output_string != "": output_string += ", " output_string += str(nireq) + " inference requests" device_ss = '' for device, nstreams in device_nstreams.items(): if device_ss != '': device_ss += ', ' device_ss += "{} streams for {}".format(str(nstreams), device) if device_ss != '': output_string += " using " + device_ss output_string += ", limits: " if niter: if not duration_seconds: progress_bar_total_count = niter output_string += str(niter) + " iterations" if duration_seconds: if niter: output_string += ", " output_string += str( getDurationInMilliseconds(duration_seconds)) + " ms duration" next_step(output_string) ## warming up - out of scope infer_request = request_queue.getIdleRequest() if not infer_request: raise Exception("No idle Infer Requests!") if (args.api_type == 'sync'): infer_request.infer(requests_input_data[infer_request.id]) else: infer_request.startAsync(requests_input_data[infer_request.id]) request_queue.waitAll() request_queue.resetTimes() start_time = datetime.now() exec_time = (datetime.now() - start_time).total_seconds() iteration = 0 progress_bar = ProgressBar(progress_bar_total_count, args.stream_output, args.progress) ## Start inference & calculate performance ## to align number if iterations to guarantee that last infer requests are executed in the same conditions **/ while ((niter and iteration < niter) or (duration_seconds and exec_time < duration_seconds) or (args.api_type == "async" and iteration % nireq != 0)): infer_request = request_queue.getIdleRequest() if not infer_request: raise Exception("No idle Infer Requests!") if (args.api_type == 'sync'): infer_request.infer(requests_input_data[infer_request.id]) else: infer_request.startAsync(requests_input_data[infer_request.id]) iteration += 1 exec_time = (datetime.now() - start_time).total_seconds() if niter: progress_bar.add_progress(1) else: ## calculate how many progress intervals are covered by current iteration. ## depends on the current iteration time and time of each progress interval. ## Previously covered progress intervals must be skipped. progress_interval_time = duration_seconds / progress_bar_total_count new_progress = (int)(exec_time / progress_interval_time - progress_count) progress_bar.add_progress(new_progress) progress_count += new_progress ## wait the latest inference executions request_queue.waitAll() total_duration_sec = request_queue.getDurationInSeconds() times = request_queue.times times.sort() latency_ms = median(times) fps = batch_size * 1000 / latency_ms if args.api_type == 'sync' else batch_size * iteration / total_duration_sec progress_bar.finish() # ------------------------------------ 10. Dumping statistics report ------------------------------------------- next_step() if args.exec_graph_path: try: exec_graph_info = exe_network.get_exec_graph_info() exec_graph_info.serialize(args.exec_graph_path) logger.info("Executable graph is stored to {}".format( args.exec_graph_path)) del exec_graph_info except Exception as e: logging.exception(e) if args.perf_counts: for ni in range(int(nireq)): perf_counts = exe_network.requests[ni].get_perf_counts() logger.info( "Pefrormance counts for {}-th infer request".format(ni)) for layer, stats in perf_counts.items(): max_layer_name = 30 print("{:<30}{:<15}{:<30}{:<20}{:<20}{:<20}".format( layer[:max_layer_name - 4] + '...' if (len(layer) >= max_layer_name) else layer, stats['status'], 'layerType: ' + str(stats['layer_type']), 'realTime: ' + str(stats['real_time']), 'cpu: ' + str(stats['cpu_time']), 'execType: ' + str(stats['exec_type']))) print("Count: {} iterations".format(iteration)) print("Duration: {:.2f} ms".format( getDurationInMilliseconds(total_duration_sec))) if not MULTI_DEVICE_NAME in device_name: print("Latency: {:.4f} ms".format(latency_ms)) print("Throughput: {:.2f} FPS".format(fps)) del exe_network del ie next_step.step_id = 0 except Exception as e: logging.exception(e)
def main(args): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) n, w, h, c = net.inputs[input_blob].shape # net.inputs[input_blob].precision = 'U8' # input_blob.precision = 'I8' log.info('shape: [{},{},{},{}]'.format(n, w, h, c)) exec_net = ie.load_network(network=net, device_name=args.device) handler = SignalHandler() print('model loaded, starting detection loop', flush=True) while not handler.interrupted: sleep(0.25) r = requests.get(args.motion_url, timeout=0.5) if not r: log.error( 'error getting motion from url, trying again after a minute') sleep(60) dat = np.frombuffer(r.content, dtype=np.int8) dat = np.reshape(dat, [1, w, h, 4]) dat = dat[:, :, :, 0:2] dat = dat.astype(np.float32) res = exec_net.infer(inputs={input_blob: dat}) res = res[out_blob] res = np.reshape(res, []) if res > args.total_motion: requests.post(args.post_url, data='"on"') print('motion detected: {}'.format(res), flush=True) sleep(600) print('exited.')
class HandPoseOpenvino(HandPose): def __init__(self, pd_xml=PALM_DETECTION_OPENVINO_FP32, lm_xml=LANDMARK_OPENVINO_FP32, pd_device="CPU", lm_device="CPU", **kwargs): """ pd_xml : palm detection model xml filepath pd_device : device on which palm detection model is run lm_xml : landmark model xml filepath lm_device : device on which landmark model is run kwargs : args inherited from HandPose class """ from openvino.inference_engine import IENetwork, IECore super().__init__(**kwargs) log.info("Loading Inference Engine") self.ie = IECore() log.info("Device info:") versions = self.ie.get_versions(pd_device) log.info("{}{}".format(" " * 8, pd_device)) log.info("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[pd_device].major, versions[pd_device].minor)) log.info("{}Build ........... {}".format( " " * 8, versions[pd_device].build_number)) # Palm detection model self.pd_name = os.path.splitext(pd_xml)[0] pd_bin = self.pd_name + '.bin' log.info( "Palm detection model - Loading network files:\n\t{}\n\t{}".format( pd_xml, pd_bin)) self.pd_net = IENetwork(model=pd_xml, weights=pd_bin) # [ INFO ] Input blob: input - shape: [1, 3, 256, 256] # [ INFO ] Output blob: classificators - shape: [1, 2944, 1] : tensor scores # [ INFO ] Output blob: regressors - shape: [1, 2944, 18] : tensor bboxes self.pd_input_blob = next(iter(self.pd_net.inputs)) log.info( f"Input blob: {self.pd_input_blob} - shape: {self.pd_net.inputs[self.pd_input_blob].shape}" ) _, _, self.pd_h, self.pd_w = self.pd_net.inputs[ self.pd_input_blob].shape for o in self.pd_net.outputs.keys(): log.info( f"Output blob: {o} - shape: {self.pd_net.outputs[o].shape}") self.pd_scores = "classificators" self.pd_bboxes = "regressors" log.info("Loading palm detection model to the plugin") self.pd_exec_net = self.ie.load_network(network=self.pd_net, num_requests=1, device_name=pd_device) # Landmarks model if self.use_landmarks: if lm_device != pd_device: log.info("Device info:") versions = self.ie.get_versions(pd_device) log.info("{}{}".format(" " * 8, pd_device)) log.info("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[pd_device].major, versions[pd_device].minor)) log.info("{}Build ........... {}".format( " " * 8, versions[pd_device].build_number)) self.lm_name = os.path.splitext(lm_xml)[0] lm_bin = self.lm_name + '.bin' log.info( "Landmark model - Loading network files:\n\t{}\n\t{}".format( lm_xml, lm_bin)) self.lm_net = IENetwork(model=lm_xml, weights=lm_bin) # [ INFO ] Input blob: input - shape: [1, 3, 256, 256] # [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid/Sigmoid - shape: [1, 1, 1, 1]: score # [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid_1/Sigmoid_1 - shape: [1, 1, 1, 1] : handedness # [ INFO ] Output blob: StatefulPartitionedCall/functional_1/tf_op_layer_ld_21_3d/ld_21_3d - shape: [1, 63] : landmarks self.lm_input_blob = next(iter(self.lm_net.inputs)) log.info( f"Input blob: {self.lm_input_blob} - shape: {self.lm_net.inputs[self.lm_input_blob].shape}" ) _, _, self.lm_h, self.lm_w = self.lm_net.inputs[ self.lm_input_blob].shape for o in self.lm_net.outputs.keys(): log.info( f"Output blob: {o} - shape: {self.lm_net.outputs[o].shape}" ) self.lm_score = "StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid/Sigmoid" self.lm_handedness = "StatefulPartitionedCall/functional_1/tf_op_layer_Sigmoid_1/Sigmoid_1" self.lm_landmarks = "StatefulPartitionedCall/functional_1/tf_op_layer_ld_21_3d/ld_21_3d" log.info("Loading landmark model to the plugin") self.lm_exec_net = self.ie.load_network(network=self.lm_net, num_requests=1, device_name=lm_device) def palm_detection_infer(self, img_nn): """ img_nn: normalized and resized image, can be directly fed into the palm detection model """ inference_result = self.pd_exec_net.infer( inputs={self.pd_input_blob: img_nn}) scores = np.squeeze(inference_result[self.pd_scores]) bboxes = inference_result[self.pd_bboxes][0] return scores, bboxes def landmarks_regression_infer(self, img_nn): """ img_nn: normalized and resized image, can be directly fed into the landmarks model Returns : - lm_score : probability between 0 and 1 that img_nn contains a handArgumentParser - lm handedness : probability between 0 and 1 that the hand is a right handArgumentParser - lm_array : array of size 63 (=3x21) containing 21 (x, y, z) coordinates of 21 landmarks """ inference_result = self.lm_exec_net.infer( inputs={self.lm_input_blob: img_nn}) lm_score = np.squeeze(inference_result[self.lm_score]) handedness = np.squeeze(inference_result[self.lm_handedness]) lm_array = np.squeeze(inference_result[self.lm_landmarks]) return lm_score, handedness, lm_array
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info("Loading Inference Engine") ie = IECore() # ---1. Read a model in OpenVINO Intermediate Representation (.xml and .bin files) or ONNX (.onnx file) format --- model = args.model log.info(f"Loading network:\n\t{model}") net = ie.read_network(model=model) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) # ----------------------------------------------------------------------------------------------------- # --------------------------- 3. Read and preprocess input -------------------------------------------- print("inputs number: " + str(len(net.input_info.keys()))) assert len( net.input_info.keys()) == 1, 'Sample supports networks with one input' for input_key in net.input_info: print("input shape: " + str(net.input_info[input_key].input_data.shape)) print("input key: " + input_key) if len(net.input_info[input_key].input_data.layout) == 4: n, c, h, w = net.input_info[input_key].input_data.shape images = np.ndarray(shape=(n, c, h, w)) images_hw = [] for i in range(n): image = cv2.imread(args.input) ih, iw = image.shape[:-1] images_hw.append((ih, iw)) log.info("File was added: ") log.info(" {}".format(args.input)) if (ih, iw) != (h, w): log.warning("Image {} is resized from {} to {}".format( args.input, image.shape[:-1], (h, w))) image = cv2.resize(image, (w, h)) image = image.transpose( (2, 0, 1)) # Change data layout from HWC to CHW images[i] = image # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.input_info.keys()) == 1 or len(net.input_info.keys()) == 2), "Sample supports topologies only with 1 or 2 inputs" out_blob = next(iter(net.outputs)) input_name, input_info_name = "", "" for input_key in net.input_info: if len(net.input_info[input_key].layout) == 4: input_name = input_key net.input_info[input_key].precision = 'U8' elif len(net.input_info[input_key].layout) == 2: input_info_name = input_key net.input_info[input_key].precision = 'FP32' if net.input_info[input_key].input_data.shape[1] != 3 and net.input_info[input_key].input_data.shape[1] != 6 or \ net.input_info[input_key].input_data.shape[0] != 1: log.error( 'Invalid input info. Should be 3 or 6 values length.') data = {} data[input_name] = images if input_info_name != "": infos = np.ndarray(shape=(n, c), dtype=float) for i in range(n): infos[i, 0] = h infos[i, 1] = w infos[i, 2] = 1.0 data[input_info_name] = infos # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') func = ng.function_from_cnn(net) ops = func.get_ordered_ops() output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))] output_ops = {op.friendly_name : op for op in ops \ if op.friendly_name in net.outputs and op.get_type_name() == "DetectionOutput"} if len(output_ops) != 0: output_name, output_info = output_ops.popitem() if output_name == "": log.error("Can't find a DetectionOutput layer in the topology") output_dims = output_info.shape if len(output_dims) != 4: log.error("Incorrect output dimensions for SSD model") max_proposal_count, object_size = output_dims[2], output_dims[3] if object_size != 7: log.error("Output item should have 7 as a last dimension") output_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # --------------------------- Performing inference ---------------------------------------------------- log.info("Loading model to the device") exec_net = ie.load_network(network=net, device_name=args.device) log.info("Creating infer request and starting inference") res = exec_net.infer(inputs=data) # ----------------------------------------------------------------------------------------------------- # --------------------------- Read and postprocess output --------------------------------------------- log.info("Processing output blobs") res = res[out_blob] boxes, classes = {}, {} data = res[0][0] for number, proposal in enumerate(data): if proposal[2] > 0: imid = np.int(proposal[0]) ih, iw = images_hw[imid] label = np.int(proposal[1]) confidence = proposal[2] xmin = np.int(iw * proposal[3]) ymin = np.int(ih * proposal[4]) xmax = np.int(iw * proposal[5]) ymax = np.int(ih * proposal[6]) print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}" \ .format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="") if proposal[2] > 0.5: print(" WILL BE PRINTED!") if not imid in boxes.keys(): boxes[imid] = [] boxes[imid].append([xmin, ymin, xmax, ymax]) if not imid in classes.keys(): classes[imid] = [] classes[imid].append(label) else: print() for imid in classes: tmp_image = cv2.imread(args.input) for box in boxes[imid]: cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]), (232, 35, 244), 2) cv2.imwrite("out.bmp", tmp_image) log.info("Image out.bmp created!") # ----------------------------------------------------------------------------------------------------- log.info("Execution successful\n") log.info( "This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool" )
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() log.info("Loading Inference Engine") ie = IECore() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = ie.read_network(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" "*8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format(" "*8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" "*8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers] if len(not_supported_layers) != 0: log.error("Following layers are not supported by the plugin for specified device {}:\n {}". format(args.device, ', '.join(not_supported_layers))) log.error("Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 3. Read and preprocess input -------------------------------------------- input_blob = next(iter(net.inputs)) n, c, h, w = net.inputs[input_blob].shape images = np.ndarray(shape=(n, c, h, w)) images_hw = [] for i in range(n): image = cv2.imread(args.input[i]) ih, iw = image.shape[:-1] images_hw.append((ih, iw)) log.info("File was added: ") log.info(" {}".format(args.input[i])) if (ih, iw) != (h, w): image = cv2.resize(image, (w, h)) log.warning("Image {} is resized from {} to {}".format(args.input[i], image.shape[:-1], (h, w))) image = image.transpose((2, 0, 1)) # Change data layout from HWC to CHW images[i] = image # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 or len(net.inputs.keys()) == 2), "Sample supports topologies only with 1 or 2 inputs" input_blob = next(iter(net.inputs)) out_blob = next(iter(net.outputs)) input_name, input_info_name = "", "" for input_key in net.inputs: if len(net.inputs[input_key].layout) == 4: input_name = input_key log.info("Batch size is {}".format(net.batch_size)) net.inputs[input_key].precision = 'U8' elif len(net.inputs[input_key].layout) == 2: input_info_name = input_key net.inputs[input_key].precision = 'FP32' if net.inputs[input_key].shape[1] != 3 and net.inputs[input_key].shape[1] != 6 or net.inputs[input_key].shape[0] != 1: log.error('Invalid input info. Should be 3 or 6 values length.') # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') output_name, output_info = "", net.outputs[next(iter(net.outputs.keys()))] for output_key in net.outputs: if net.layers[output_key].type == "DetectionOutput": output_name, output_info = output_key, net.outputs[output_key] if output_name == "": log.error("Can't find a DetectionOutput layer in the topology") output_dims = output_info.shape if len(output_dims) != 4: log.error("Incorrect output dimensions for SSD model") max_proposal_count, object_size = output_dims[2], output_dims[3] if object_size != 7: log.error("Output item should have 7 as a last dimension") output_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # --------------------------- Performing inference ---------------------------------------------------- log.info("Loading model to the device") exec_net = ie.load_network(network=net, device_name=args.device) log.info("Creating infer request and starting inference") res = exec_net.infer(inputs={input_blob: images}) # ----------------------------------------------------------------------------------------------------- # --------------------------- Read and postprocess output --------------------------------------------- log.info("Processing output blobs") res = res[out_blob] boxes, classes = {}, {} data = res[0][0] for number, proposal in enumerate(data): if proposal[2] > 0: imid = np.int(proposal[0]) ih, iw = images_hw[imid] label = np.int(proposal[1]) confidence = proposal[2] xmin = np.int(iw * proposal[3]) ymin = np.int(ih * proposal[4]) xmax = np.int(iw * proposal[5]) ymax = np.int(ih * proposal[6]) print("[{},{}] element, prob = {:.6} ({},{})-({},{}) batch id : {}"\ .format(number, label, confidence, xmin, ymin, xmax, ymax, imid), end="") if proposal[2] > 0.5: print(" WILL BE PRINTED!") if not imid in boxes.keys(): boxes[imid] = [] boxes[imid].append([xmin, ymin, xmax, ymax]) if not imid in classes.keys(): classes[imid] = [] classes[imid].append(label) else: print() for imid in classes: tmp_image = cv2.imread(args.input[imid]) for box in boxes[imid]: cv2.rectangle(tmp_image, (box[0], box[1]), (box[2], box[3]), (232, 35, 244), 2) cv2.imshow('output', tmp_image) if (cv2.waitKey(0) == 27): break cv2.destroyAllWindows() cv2.imwrite("out.bmp", tmp_image) log.info("Image out.bmp created!") # ----------------------------------------------------------------------------------------------------- log.info("Execution successful\n") log.info("This sample is an API example, for any performance measurements please use the dedicated benchmark_app tool")
def main(): args = parse_arguments() # --------------------------------- 1. Load Plugin for inference engine --------------------------------- logger.info("Creating Inference Engine") ie = IECore() if 'CPU' in args.target_device: if args.path_to_extension: ie.add_extension(args.path_to_extension, "CPU") if args.number_threads is not None: ie.set_config({'CPU_THREADS_NUM': str(args.number_threads)}, "CPU") elif 'GPU' in args.target_device: if args.path_to_cldnn_config: ie.set_config({'CONFIG_FILE': args.path_to_cldnn_config}, "GPU") logger.info("GPU extensions is loaded {}".format( args.path_to_cldnn_config)) else: raise AttributeError( "Device {} do not support of 3D convolution. " "Please use CPU, GPU or HETERO:*CPU*, HETERO:*GPU*") logger.info("Device is {}".format(args.target_device)) version = ie.get_versions(args.target_device)[args.target_device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) logger.info("Plugin version is {}".format(version_str)) # --------------------- 2. Read IR Generated by ModelOptimizer (.xml and .bin files) --------------------- xml_filename = os.path.abspath(args.path_to_model) bin_filename = os.path.abspath(os.path.splitext(xml_filename)[0] + '.bin') ie_network = IENetwork(xml_filename, bin_filename) input_info = ie_network.inputs if len(input_info) == 0: raise AttributeError('No inputs info is provided') elif len(input_info) != 1: raise AttributeError("only one input layer network is supported") input_name = next(iter(input_info)) out_name = next(iter(ie_network.outputs)) if args.shape: logger.info("Reshape of network from {} to {}".format( input_info[input_name].shape, args.shape)) ie_network.reshape({input_name: args.shape}) input_info = ie_network.inputs # ---------------------------------------- 4. Preparing input data ---------------------------------------- logger.info("Preparing inputs") if len(input_info[input_name].shape) != 5: raise AttributeError( "Incorrect shape {} for 3d convolution network".format(args.shape)) n, c, d, h, w = input_info[input_name].shape ie_network.batch_size = n if not os.path.exists(args.path_to_input_data): raise AttributeError("Path to input data: '{}' does not exist".format( args.path_to_input_data)) is_nifti_data = os.path.isdir(args.path_to_input_data) if is_nifti_data: series_name = find_series_name(args.path_to_input_data) original_data, data_crop, affine, original_size, bbox = \ read_image(args.path_to_input_data, series_name=series_name, sizes=(h, w, d)) else: if not (fnmatch(args.path_to_input_data, '*.tif') or fnmatch(args.path_to_input_data, '*.tiff')): raise AttributeError("Input file extension must have tiff format") data_crop = np.zeros(shape=(n, c, d, h, w), dtype=np.float) im_seq = ImageSequence.Iterator(Image.open(args.path_to_input_data)) for i, page in enumerate(im_seq): im = np.array(page).reshape(h, w, c) for channel in range(c): data_crop[:, channel, i, :, :] = im[:, :, channel] original_data = data_crop original_size = original_data.shape[-3:] test_im = {input_name: data_crop} # ------------------------------------- 4. Loading model to the plugin ------------------------------------- logger.info("Loading model to the plugin") executable_network = ie.load_network(network=ie_network, device_name=args.target_device) del ie_network # ---------------------------------------------- 5. Do inference -------------------------------------------- logger.info("Start inference") start_time = datetime.now() res = executable_network.infer(test_im) infer_time = datetime.now() - start_time logger.info("Finish inference") logger.info("Inference time is {}".format(infer_time)) # ---------------------------- 6. Processing of the received inference results ------------------------------ result = res[out_name] batch, channels, out_d, out_h, out_w = result.shape list_img = list() list_seg_result = list() logger.info("Processing of the received inference results is started") start_time = datetime.now() for batch, data in enumerate(result): seg_result = np.zeros(shape=original_size, dtype=np.uint8) if data.shape[1:] != original_size: x = bbox[1] - bbox[0] y = bbox[3] - bbox[2] z = bbox[5] - bbox[4] seg_result[bbox[0]:bbox[1], bbox[2]:bbox[3], bbox[4]:bbox[5]] = \ np.argmax(resample_np(data, (channels, x, y, z), 1), axis=0) elif channels == 1: reshaped_data = data.reshape(out_d, out_h, out_w) mask = reshaped_data[:, :, :] > 0.5 reshaped_data[mask] = 1 seg_result = reshaped_data.astype(int) else: seg_result = np.argmax(data, axis=0).astype(int) im = np.stack([ original_data[batch, 0, :, :, :], original_data[batch, 0, :, :, :], original_data[batch, 0, :, :, :] ], axis=3) im = 255 * (im - im.min()) / (im.max() - im.min()) color_seg_frame = np.zeros(im.shape, dtype=np.uint8) for idx, c in enumerate(CLASSES_COLOR_MAP): color_seg_frame[seg_result[:, :, :] == idx, :] = np.array( c, dtype=np.uint8) mask = seg_result[:, :, :] > 0 im[mask] = color_seg_frame[mask] for k in range(out_d): if is_nifti_data: list_img.append( Image.fromarray(im[:, :, k, :].astype('uint8'), 'RGB')) else: list_img.append( Image.fromarray(im[k, :, :, :].astype('uint8'), 'RGB')) if args.output_nifti and is_nifti_data: list_seg_result.append(seg_result) result_processing_time = datetime.now() - start_time logger.info("Processing of the received inference results is finished") logger.info("Processing time is {}".format(result_processing_time)) # --------------------------------------------- 7. Save output ----------------------------------------------- tiff_output_name = os.path.join(args.path_to_output, 'output.tiff') Image.new('RGB', (data.shape[3], data.shape[2])).save(tiff_output_name, append_images=list_img, save_all=True) logger.info("Result tiff file was saved to {}".format(tiff_output_name)) if args.output_nifti and is_nifti_data: for seg_res in list_seg_result: nii_filename = os.path.join( args.path_to_output, 'output_{}.nii.gz'.format(list_seg_result.index(seg_res))) nib.save(nib.Nifti1Image(seg_res, affine=affine), nii_filename) logger.info( "Result nifti file was saved to {}".format(nii_filename))
class Benchmark: def __init__(self, device: str, number_infer_requests: int = None, number_iterations: int = None, duration_seconds: int = None, api_type: str = 'async'): self.device = device self.ie = IECore() self.nireq = number_infer_requests self.niter = number_iterations self.duration_seconds = get_duration_seconds(duration_seconds, self.niter, self.device) self.api_type = api_type def __del__(self): del self.ie def add_extension(self, path_to_extension: str = None, path_to_cldnn_config: str = None): if path_to_cldnn_config: self.ie.set_config({'CONFIG_FILE': path_to_cldnn_config}, GPU_DEVICE_NAME) logger.info( 'GPU extensions is loaded {}'.format(path_to_cldnn_config)) if path_to_extension: self.ie.add_extension(extension_path=path_to_extension, device_name=CPU_DEVICE_NAME) logger.info( 'CPU extensions is loaded {}'.format(path_to_extension)) def get_version_info(self) -> str: logger.info('InferenceEngine:\n{: <9}{:.<24} {}'.format( '', 'API version', get_version())) version_string = 'Device info\n' for device, version in self.ie.get_versions(self.device).items(): version_string += '{: <9}{}\n'.format('', device) version_string += '{: <9}{:.<24}{} {}.{}\n'.format( '', version.description, ' version', version.major, version.minor) version_string += '{: <9}{:.<24} {}\n'.format( '', 'Build', version.build_number) return version_string def set_config(self, config={}): for device in config.keys(): self.ie.set_config(config[device], device) def read_network(self, path_to_model: str): model_filename = os.path.abspath(path_to_model) head, ext = os.path.splitext(model_filename) weights_filename = os.path.abspath( head + BIN_EXTENSION) if ext == XML_EXTENSION else "" ie_network = self.ie.read_network(model_filename, weights_filename) return ie_network def load_network(self, ie_network: IENetwork, config={}): exe_network = self.ie.load_network( ie_network, self.device, config=config, num_requests=1 if self.api_type == 'sync' else self.nireq or 0) # Number of requests self.nireq = len(exe_network.requests) return exe_network def import_network(self, path_to_file: str, config={}): exe_network = self.ie.import_network( model_file=path_to_file, device_name=self.device, config=config, num_requests=1 if self.api_type == 'sync' else self.nireq or 0) # Number of requests self.nireq = len(exe_network.requests) return exe_network def first_infer(self, exe_network): infer_request = exe_network.requests[0] # warming up - out of scope if self.api_type == 'sync': infer_request.infer() else: infer_request.async_infer() status = exe_network.wait() if status != StatusCode.OK: raise Exception( "Wait for all requests is failed with status code {}!". format(status)) return infer_request.latency def infer(self, exe_network, batch_size, progress_bar=None): progress_count = 0 infer_requests = exe_network.requests start_time = datetime.utcnow() exec_time = 0 iteration = 0 times = [] in_fly = set() # Start inference & calculate performance # to align number if iterations to guarantee that last infer requests are executed in the same conditions **/ while (self.niter and iteration < self.niter) or \ (self.duration_seconds and exec_time < self.duration_seconds) or \ (self.api_type == 'async' and iteration % self.nireq): if self.api_type == 'sync': infer_requests[0].infer() times.append(infer_requests[0].latency) else: infer_request_id = exe_network.get_idle_request_id() if infer_request_id < 0: status = exe_network.wait(num_requests=1) if status != StatusCode.OK: raise Exception("Wait for idle request failed!") infer_request_id = exe_network.get_idle_request_id() if infer_request_id < 0: raise Exception("Invalid request id!") if infer_request_id in in_fly: times.append(infer_requests[infer_request_id].latency) else: in_fly.add(infer_request_id) infer_requests[infer_request_id].async_infer() iteration += 1 exec_time = (datetime.utcnow() - start_time).total_seconds() if progress_bar: if self.duration_seconds: # calculate how many progress intervals are covered by current iteration. # depends on the current iteration time and time of each progress interval. # Previously covered progress intervals must be skipped. progress_interval_time = self.duration_seconds / progress_bar.total_num new_progress = int(exec_time / progress_interval_time - progress_count) progress_bar.add_progress(new_progress) progress_count += new_progress elif self.niter: progress_bar.add_progress(1) # wait the latest inference executions status = exe_network.wait() if status != StatusCode.OK: raise Exception( "Wait for all requests is failed with status code {}!".format( status)) total_duration_sec = (datetime.utcnow() - start_time).total_seconds() for infer_request_id in in_fly: times.append(infer_requests[infer_request_id].latency) times.sort() latency_ms = median(times) fps = batch_size * 1000 / latency_ms if self.api_type == 'sync' else batch_size * iteration / total_duration_sec if progress_bar: progress_bar.finish() return fps, latency_ms, total_duration_sec, iteration
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() if args.colors: COLOR_RED = "\033[91m" COLOR_RESET = "\033[0m" else: COLOR_RED = "" COLOR_RESET = "" # load vocabulary file for model log.info("Loading vocab file:\t{}".format(args.vocab)) vocab = load_vocab_file(args.vocab) log.info("{} tokens loaded".format(len(vocab))) # get context as a string (as we might need it's length for the sequence reshape) paragraphs = get_paragraphs(args.input) context = '\n'.join(paragraphs) log.info("Size: {} chars".format(len(context))) log.info("Context: " + COLOR_RED + context + COLOR_RESET) # encode context into token ids list c_tokens_id, c_tokens_se = text_to_tokens(context.lower(), vocab) log.info("Initializing Inference Engine") ie = IECore() version = ie.get_versions(args.device)[args.device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) log.info("Plugin version is {}".format(version_str)) # read IR model_xml = args.model model_bin = model_xml.with_suffix(".bin") log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) ie_encoder = ie.read_network(model=model_xml, weights=model_bin) if args.reshape: # reshape the sequence length to the context + maximum question length (in tokens) first_input_layer = next(iter(ie_encoder.input_info)) c = ie_encoder.input_info[first_input_layer].input_data.shape[1] # find the closest multiple of 64, if it is smaller than current network's sequence length, let' use that seq = min(c, int(np.ceil((len(c_tokens_id) + args.max_question_token_num) / 64) * 64)) if seq < c: new_shapes = {} for input_name, input_info in ie_encoder.input_info.items(): n, c = input_info.input_data.shape new_shapes[input_name] = [n, seq] log.info("Reshaped input {} from {} to the {}".format( input_name, input_info.input_data.shape, new_shapes[input_name])) log.info("Attempting to reshape the network to the modified inputs...") try: ie_encoder.reshape(new_shapes) log.info("Successful!") except RuntimeError: log.error("Failed to reshape the network, please retry the demo without '-r' option") sys.exit(-1) else: log.info("Skipping network reshaping," " as (context length + max question length) exceeds the current (input) network sequence length") # check input and output names input_names = [i.strip() for i in args.input_names.split(',')] output_names = [o.strip() for o in args.output_names.split(',')] if ie_encoder.input_info.keys() != set(input_names) or ie_encoder.outputs.keys() != set(output_names): log.error("Input or Output names do not match") log.error(" The demo expects input->output names: {}->{}. " "Please use the --input_names and --output_names to specify the right names " "(see actual values below)".format(input_names, output_names)) log.error(" Actual network input->output names: {}->{}".format(list(ie_encoder.input_info.keys()), list(ie_encoder.outputs.keys()))) raise Exception("Unexpected network input or output names") # load model to the device log.info("Loading model to the {}".format(args.device)) ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device) if args.questions: def questions(): for question in args.questions: log.info("Question: {}".format(question)) yield question else: def questions(): while True: yield input('Type question (empty string to exit):') # loop on user's or prepared questions for question in questions(): if not question.strip(): break q_tokens_id, _ = text_to_tokens(question.lower(), vocab) # maximum number of tokens that can be processed by network at once max_length = ie_encoder.input_info[input_names[0]].input_data.shape[1] # calculate number of tokens for context in each inference request. # reserve 3 positions for special tokens # [CLS] q_tokens [SEP] c_tokens [SEP] c_wnd_len = max_length - (len(q_tokens_id) + 3) # token num between two neighbour context windows # 1/2 means that context windows are overlapped by half c_stride = c_wnd_len // 2 t0 = time.perf_counter() t_count = 0 # array of answers from each window answers = [] # init a window to iterate over context c_s, c_e = 0, min(c_wnd_len, len(c_tokens_id)) # iterate while context window is not empty while c_e > c_s: # form the request tok_cls = vocab['[CLS]'] tok_sep = vocab['[SEP]'] input_ids = [tok_cls] + q_tokens_id + [tok_sep] + c_tokens_id[c_s:c_e] + [tok_sep] token_type_ids = [0] + [0] * len(q_tokens_id) + [0] + [1] * (c_e - c_s) + [0] attention_mask = [1] * len(input_ids) # pad the rest of the request pad_len = max_length - len(input_ids) input_ids += [0] * pad_len token_type_ids += [0] * pad_len attention_mask += [0] * pad_len # create numpy inputs for IE inputs = { input_names[0]: np.array([input_ids], dtype=np.int32), input_names[1]: np.array([attention_mask], dtype=np.int32), input_names[2]: np.array([token_type_ids], dtype=np.int32), } if len(input_names)>3: inputs[input_names[3]] = np.arange(len(input_ids), dtype=np.int32)[None, :] t_start = time.perf_counter() # infer by IE res = ie_encoder_exec.infer(inputs=inputs) t_end = time.perf_counter() t_count += 1 log.info("Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)".format( max_length, 1 / (t_end - t_start), t_end - t_start )) # get start-end scores for context def get_score(name): out = np.exp(res[name].reshape((max_length,))) return out / out.sum(axis=-1) score_s = get_score(output_names[0]) score_e = get_score(output_names[1]) # get 'no-answer' score (not valid if model has been fine-tuned on squad1.x) if args.model_squad_ver.split('.')[0] == '1': score_na = 0 else: score_na = score_s[0] * score_e[0] # find product of all start-end combinations to find the best one c_s_idx = len(q_tokens_id) + 2 # index of first context token in tensor c_e_idx = max_length - (1 + pad_len) # index of last+1 context token in tensor score_mat = np.matmul( score_s[c_s_idx:c_e_idx].reshape((c_e - c_s, 1)), score_e[c_s_idx:c_e_idx].reshape((1, c_e - c_s)) ) # reset candidates with end before start score_mat = np.triu(score_mat) # reset long candidates (>max_answer_token_num) score_mat = np.tril(score_mat, args.max_answer_token_num - 1) # find the best start-end pair max_s, max_e = divmod(score_mat.flatten().argmax(), score_mat.shape[1]) max_score = score_mat[max_s, max_e] * (1 - score_na) # convert to context text start-end index max_s = c_tokens_se[c_s + max_s][0] max_e = c_tokens_se[c_s + max_e][1] # check that answers list does not have duplicates (because of context windows overlapping) same = [i for i, a in enumerate(answers) if a[1] == max_s and a[2] == max_e] if same: assert len(same) == 1 # update existing answer record a = answers[same[0]] answers[same[0]] = (max(max_score, a[0]), max_s, max_e) else: # add new record answers.append((max_score, max_s, max_e)) # check that context window reached the end if c_e == len(c_tokens_id): break # move to next window position c_s = min(c_s + c_stride, len(c_tokens_id)) c_e = min(c_s + c_wnd_len, len(c_tokens_id)) t1 = time.perf_counter() log.info("The performance below is reported only for reference purposes, " "please use the benchmark_app tool (part of the OpenVINO samples) for any actual measurements.") log.info("{} requests of {} length were processed in {:0.2f}sec ({:0.2}sec per request)".format( t_count, max_length, t1 - t0, (t1 - t0) / t_count )) # print top 3 results answers = sorted(answers, key=lambda x: -x[0]) for score, s, e in answers[:3]: log.info("---answer: {:0.2f} {}".format(score, context[s:e])) c_s, c_e = find_sentence_range(context, s, e) log.info(" " + context[c_s:s] + COLOR_RED + context[s:e] + COLOR_RESET + context[e:c_e])
class Model: ''' Class for the Face Detection Model. ''' def __init__(self, model_name, device_name, extensions=None): ''' TODO: Use this to set your instance variables. model_name, device='CPU', extensions=None ''' self.ie = IECore() self.extension = extensions self.model_structure = model_name + '.xml' self.model_weights = model_name + '.bin' _, self.model_name = self.path_leaf(model_name) self.device = device_name self.net = None self.layers_map = None self.exec_net = None try: t_start = time.perf_counter() self.net = self.ie.read_network(model=self.model_structure, weights=self.model_weights) t_end = time.perf_counter() log.info( "model {} is processed with loading {:0.2} sec per request)". format(self.model_name, t_end - t_start)) except Exception as e: raise ValueError( "Could not Initialise the network. Have you enterred the correct model path?", e) self.layers_map = self.ie.query_network(network=self.net, device_name=device_name) self.input_blob = list(self.net.inputs.keys()) self.output_blob = list(self.net.outputs.keys()) self.input_shape = None self.output_shape = None def load_model(self): return self.net def get_input_blob(self): return self.input_blob def get_output_blob(self): return self.output_blob def get_input_shape(self): model_inputs_name = np.array(self.input_blob) array_data_inputs = [] for i in range(len(model_inputs_name)): array_data_inputs.append( self.net.inputs[model_inputs_name[i]].shape) return np.array(array_data_inputs) def get_output_shape(self): model_outputs_name = np.array(self.output_blob) array_data_outputs = [] for i in range(len(model_outputs_name)): array_data_outputs.append( self.net.outputs[model_outputs_name[i]].shape) return np.array(array_data_outputs) def load_network(self): return self.ie.load_network(self.net, self.device, num_requests=0) def get_inference(self, input_data): self.exec_net = self.load_network() return self.exec_net.infer(inputs=input_data) def get_infer_output(self, input_data): self.get_inference(input_data) return self.exec_net.requests[0].outputs def get_supported_layer(self): for pair in self.layers_map.items(): print("supported_layers: ", pair) def get_unsupported_layer(self): not_supported_layers = [ l for l in self.net.layers.keys() if l not in self.layers_map ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(self.device, ', '.join(not_supported_layers))) else: log.info(f"All layers are supported for {self.model_name} model!") def get_model_name(self): return self.model_structure def get_openvino_version(self, device_name): return self.ie.get_versions(device_name) def get_model_name(self): return self.model_name def path_leaf(self, path): tail, head = ntpath.split(path) return tail, ntpath.basename(head)
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # load vocabulary file for model log.info("Loading vocab file:\t{}".format(args.vocab)) vocab = load_vocab_file(args.vocab) log.info("{} tokens loaded".format(len(vocab))) # get context as a string (as we might need it's length for the sequence reshape) paragraphs = get_paragraphs(args.input) context = '\n'.join(paragraphs) log.info("Size: {} chars".format(len(context))) sentences = re.split(sentence_splitter, context) preprocessed_sentences = [text_to_tokens(sentence, vocab) for sentence in sentences] max_sent_length = max([len(tokens) + 2 for tokens, _ in preprocessed_sentences]) log.info("Initializing Inference Engine") ie = IECore() version = ie.get_versions(args.device)[args.device] version_str = "{}.{}.{}".format(version.major, version.minor, version.build_number) log.info("Plugin version is {}".format(version_str)) # read IR model_xml = args.model model_bin = model_xml.with_suffix(".bin") log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) ie_encoder = ie.read_network(model=model_xml, weights=model_bin) # check input and output names input_names = [i.strip() for i in args.input_names.split(',')] if ie_encoder.input_info.keys() != set(input_names): log.error("Input names do not match") log.error(" The demo expects input names: {}. " "Please use the --input_names to specify the right names " "(see actual values below)".format(input_names)) log.error(" Actual network input names: {}".format(list(ie_encoder.input_info.keys()))) raise Exception("Unexpected network input names") if len(ie_encoder.outputs) != 1: log.log.error('Demo expects model with single output, while provided {}'.format(len(ie_encoder.outputs))) raise Exception('Unexpected number of outputs') output_names = list(ie_encoder.outputs) max_length = ie_encoder.input_info[input_names[0]].input_data.shape[1] if max_sent_length > max_length: input_shapes = { input_names[0]: [1, max_sent_length], input_names[1]: [1, max_sent_length], input_names[2]: [1, max_sent_length] } ie_encoder.reshape(input_shapes) max_length = max_sent_length # load model to the device log.info("Loading model to the {}".format(args.device)) ie_encoder_exec = ie.load_network(network=ie_encoder, device_name=args.device) # maximum number of tokens that can be processed by network at once t0 = time.perf_counter() t_count = 0 def get_score(name): out = np.exp(res[name][0]) return out / out.sum(axis=-1, keepdims=True) for sentence, (c_tokens_id, c_token_s_e) in zip(sentences, preprocessed_sentences): # form the request tok_cls = vocab['[CLS]'] tok_sep = vocab['[SEP]'] input_ids = [tok_cls] + c_tokens_id + [tok_sep] token_type_ids = [0] * len(input_ids) attention_mask = [1] * len(input_ids) # pad the rest of the request pad_len = max_length - len(input_ids) input_ids += [0] * pad_len token_type_ids += [0] * pad_len attention_mask += [0] * pad_len # create numpy inputs for IE inputs = { input_names[0]: np.array([input_ids], dtype=np.int32), input_names[1]: np.array([attention_mask], dtype=np.int32), input_names[2]: np.array([token_type_ids], dtype=np.int32), } if len(input_names)>3: inputs[input_names[3]] = np.arange(len(input_ids), dtype=np.int32)[None, :] t_start = time.perf_counter() # infer by IE res = ie_encoder_exec.infer(inputs=inputs) t_end = time.perf_counter() t_count += 1 log.info("Sequence of length {} is processed with {:0.2f} requests/sec ({:0.2} sec per request)".format( max_length, 1 / (t_end - t_start), t_end - t_start )) score = get_score(output_names[0]) labels_idx = score.argmax(-1) filtered_labels_idx = [ (idx, label_idx) for idx, label_idx in enumerate(labels_idx) if label_idx != 0 and 0 < idx < max_length - pad_len ] if not filtered_labels_idx: continue log.info('Sentence: \n\t{}'.format(sentence)) visualized = set() for idx, label_idx in filtered_labels_idx: word_s, word_e = c_token_s_e[idx - 1] if (word_s, word_e) in visualized: continue visualized.add((word_s, word_e)) word = sentence[word_s:word_e] log.info('\n\tWord: {}\n\tConfidence: {}\n\tTag: {}'.format(word, score[idx][label_idx], label_to_tag[label_idx])) t1 = time.perf_counter() log.info("The performance below is reported only for reference purposes, " "please use the benchmark_app tool (part of the OpenVINO samples) for any actual measurements.") log.info("{} requests of {} length were processed in {:0.2f}sec ({:0.2}sec per request)".format( t_count, max_length, t1 - t0, (t1 - t0) / t_count ))
def main(): total_timer = Timer(name='total') total_timer.tic() log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # --------------------------- 1. Read IR Generated by ModelOptimizer (.xml and .bin files) ------------ model_xml = args.model model_bin = os.path.splitext(model_xml)[0] + ".bin" log.info("Loading network files:\n\t{}\n\t{}".format(model_xml, model_bin)) net = IENetwork(model=model_xml, weights=model_bin) # ----------------------------------------------------------------------------------------------------- # ------------- 2. Load Plugin for inference engine and extensions library if specified -------------- log.info("Loading Inference Engine") ie = IECore() log.info("Device info:") versions = ie.get_versions(args.device) print("{}{}".format(" " * 8, args.device)) print("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[args.device].major, versions[args.device].minor)) print("{}Build ........... {}".format(" " * 8, versions[args.device].build_number)) if args.cpu_extension and "CPU" in args.device: ie.add_extension(args.cpu_extension, "CPU") log.info("CPU extension loaded: {}".format(args.cpu_extension)) if "CPU" in args.device: supported_layers = ie.query_network(net, "CPU") not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) # ----------------------------------------------------------------------------------------------------- # --------------------------- 4. Configure input & output --------------------------------------------- # --------------------------- Prepare input blobs ----------------------------------------------------- log.info("Preparing input blobs") assert (len(net.inputs.keys()) == 1 ), "Sample supports topologies only with 1 input" input_name = next(iter(net.inputs.keys())) input_info = net.inputs[input_name] input_info.precision = 'FP32' # --------------------------- Prepare output blobs ---------------------------------------------------- log.info('Preparing output blobs') assert (len(net.outputs.keys()) == 2 ), "Sample supports topologies only with 2 output" loc_out_name = "797" class_out_name = "741" assert (loc_out_name in net.outputs.keys()) and (class_out_name in net.outputs.keys()) loc_out_info = net.outputs[loc_out_name] class_out_info = net.outputs[class_out_name] loc_out_info.precision = "FP32" class_out_info.precision = "FP32" # ----------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------- log.info("Loading model to the device") exec_net = ie.load_network(network=net, device_name=args.device) # --------------------------- 3. Read and preprocess input -------------------------------------------- # ----------------------------------------------------------------------------------------------------- if not os.path.exists(args.result_dir): os.makedirs(args.result_dir) if args.voc_res_file and os.path.exists(args.voc_res_file): os.remove(args.voc_res_file) create_anchor_timer = Timer(name='create_anchor') read_img_timer = Timer(name='read_img') preprocess_timer = Timer(name='preprocess') infer_timer = Timer(name='infer') adapter_timer = Timer(name='adapter') patch_img_nms_timer = Timer(name='patch_img_nms') whole_img_nms_timer = Timer(name='whole_img_nms') add_offset_timer = Timer(name='add_offset') write_result_timer = Timer(name='write_result') create_anchor_timer.tic() adapter = RetinaNetAdapter(input_shape=args.patch_size) create_anchor_timer.toc() image_names = os.listdir(args.image_dir) log.info("image_nums: {}".format(len(image_names))) for image_id, image_name in enumerate(image_names): read_img_timer.tic() image_path = os.path.join(args.image_dir, image_name) img = cv2.imread(image_path).astype('float32') read_img_timer.toc() height, width, _ = img.shape image_shape = (width, height) strides = args.strides patch_size = args.patch_size x_num, y_num = calc_split_num(image_shape, patch_size, strides) log.info("id:{}, name: {}, shape: ({},{}), x_num:{}, y_num:{}".format( image_id, image_name, height, width, x_num, y_num)) preprocess_timer.tic() img = img.transpose((2, 0, 1)) # Change data layout from HWC to CHW preprocess_timer.toc() result_all = [] for i in range(x_num): for j in range(y_num): x = strides[0] * i if i < x_num - 1 else image_shape[ 0] - args.patch_size[0] y = strides[1] * j if j < y_num - 1 else image_shape[ 1] - args.patch_size[1] # print('processing {} , x: {}, y: {}'.format(image_name, x, y)) preprocess_timer.tic() crop_img = img[:, y:y + patch_size[1], x:x + patch_size[0]].copy() crop_img = crop_img[np.newaxis, :, :, :] preprocess_timer.toc() # --------------------------- Performing inference ---------------------------------------------------- infer_timer.tic() res = exec_net.infer(inputs={input_name: crop_img}) loc_out = res[loc_out_name][0] class_out = res[class_out_name][0] infer_timer.toc() adapter_timer.tic() result = adapter.process(loc_out, class_out) adapter_timer.toc() patch_img_nms_timer.tic() result, _ = nms(result, thresh=0.5, keep_top_k=100) patch_img_nms_timer.toc() # import pdb;pdb.set_trace() add_offset_timer.tic() result[:, 0] += x result[:, 1] += y result[:, 2] += x result[:, 3] += y result_all.append(result) add_offset_timer.toc() # import pdb;pdb.set_trace() whole_img_nms_timer.tic() result_all = np.concatenate(result_all, axis=0) nms_result, _ = nms(result_all, thresh=0.5) whole_img_nms_timer.toc() write_result_timer.tic() voc_format = '{} {:.4f} {} {} {} {}' pos_all = [] voc_all = [] for i in range(nms_result.shape[0]): x = int(nms_result[i, 0]) y = int(nms_result[i, 1]) w = max(int(nms_result[i, 2] - nms_result[i, 0]), 1) h = max(int(nms_result[i, 3] - nms_result[i, 1]), 1) p = float(nms_result[i, 4]) pos = {'x': x, 'y': y, 'w': w, 'h': h, 'p': p} pos_all.append(pos) if args.voc_res_file: xmin = x ymin = y xmax = int(nms_result[i, 2]) ymax = int(nms_result[i, 3]) voc_str = voc_format.format( os.path.splitext(image_name)[0], p, xmin, ymin, xmax, ymax) voc_all.append(voc_str) file_name = os.path.splitext(image_name)[0] + '.json' with open(os.path.join(args.result_dir, file_name), 'w') as f: json.dump(pos_all, f) if args.voc_res_file: with open(args.voc_res_file, 'a') as f: for voc_str in voc_all: f.write(voc_str + '\n') write_result_timer.toc() total_timer.toc() # ----------------------------------------------------------------------------------------------------- all_timers = [] all_timers.extend([ create_anchor_timer, read_img_timer, preprocess_timer, infer_timer, adapter_timer, patch_img_nms_timer, whole_img_nms_timer, add_offset_timer, write_result_timer, total_timer ]) for timer in all_timers: log.info('{}: avg: {:.2f} ms, total: {:.2f}s'.format( timer.name, timer.avg * 1000, timer.total)) log.info("Execution successful\n")
class Network: """ Load and configure inference plugins for the specified target devices and performs synchronous and asynchronous modes for the specified infer requests. """ def __init__(self): ### Initialize any class variables desired ### self.plugin = None self.network = None self.input_blob = None self.input_blob_name = None self.output_blob = None self.exec_network = None self.infer_request = None self.output_name = None self.output_info = None self.feed_dict = None def load_model(self, model, device="CPU", cpu_extension=None): ### Load the model ### model_xml = model model_bin = os.path.splitext(model_xml)[0] + ".bin" ### init inference engine plugin self.plugin = IECore() ### Add any necessary extensions ### log.info("attempting to add CPU extension!") log.info(cpu_extension) if cpu_extension and "CPU" in device: plugin_dir = '/opt/intel/openvino/deployment_tools/inference_engine/lib/intel64' self.plugin.add_extension(cpu_extension, device) #plugin = IEPlugin(device=‘CPU', plugin_dirs=plugin_dir) #cpu_extension_path = '/home/temp/intel/openvino/deployment_tools/inference_engine/lib/intel64/libMKLDNNPlugin.so' log.info("Loading network files:\n\t{}\n\t{}".format( model_xml, model_bin)) self.network = IENetwork(model=model_xml, weights=model_bin) ### Check for supported layers ### if "CPU" in device: supported_layers = self.plugin.query_network(self.network, "CPU") not_supported_layers = [ l for l in self.network.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(args.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) versions = self.plugin.get_versions(device) log.info("{}{}".format(" " * 8, device)) log.info("{}MKLDNNPlugin version ......... {}.{}".format( " " * 8, versions[device].major, versions[device].minor)) log.info("{}Build ........... {}".format( " " * 8, versions[device].build_number)) ### Return the loaded inference plugin ### self.network = IENetwork(model=model_xml, weights=model_bin) self.exec_network = self.plugin.load_network(self.network, device) ## grab the input layer ## self.input_blob = self.network.inputs ##next(iter(self.network.inputs)) img_info_input_blob = None self.feed_dict = {} for blob_name in self.network.inputs: log.info("blob_name: " + blob_name) if len(self.network.inputs[blob_name].shape) == 4: self.input_blob_name = blob_name elif len(self.network.inputs[blob_name].shape) == 2: img_info_input_blob = blob_name else: raise RuntimeError( "Unsupported {}D input layer '{}'. Only 2D and 4D input layers are supported" .format(len(net.inputs[blob_name].shape), blob_name)) self.output_blob = next(iter(self.network.outputs)) if img_info_input_blob: log.info("set input info") n, c, h, w = self.get_input_shape() self.feed_dict[img_info_input_blob] = [h, w, 1] self.output_name, self.output_info = "", self.network.outputs[next( iter(self.network.outputs.keys()))] for output_key in self.network.outputs: if self.network.layers[output_key].type == "DetectionOutput": self.output_name, self.output_info = output_key, self.network.outputs[ output_key] if self.output_name == "": log.error("Can't find a DetectionOutput layer in the topology") log.info("load model done") ### Note: You may need to update the function parameters. ### log.info('Preparing output blobs') output_name, output_info = "", self.network.outputs[next( iter(self.network.outputs.keys()))] for output_key in self.network.outputs: log.info("iteration: %s", output_key) if self.network.layers[output_key].type == "DetectionOutput": output_name, output_info = output_key, self.network.outputs[ output_key] if output_name == "": log.error("Can't find a DetectionOutput layer in the topology") output_dims = output_info.shape if len(output_dims) != 4: log.error("Incorrect output dimensions for SSD model") max_proposal_count, object_size = output_dims[2], output_dims[3] log.info("Max Proposal Count is: %s", max_proposal_count) log.info("Output Object Size: %s, %s", output_dims[2], output_dims[3]) if object_size != 7: log.error("Output item should have 7 as a last dimension") output_info.precision = "FP32" return def get_input_blob(self): ##return self.network.inputs[self.input_blob] ##input_blob = None ##return input_blob ##self.network.inputs[input_blob].shape return self.input_blob def get_input_blob_name(self): ##return self.network.inputs[self.input_blob] ##input_blob = None ##return input_blob ##self.network.inputs[input_blob].shape return self.input_blob_name def get_input_shape(self): ### Return the shape of the input layer ### return self.network.inputs[self.input_blob_name].shape def get_output_name(self): return self.output_name def get_output_info(self): return self.output_info def exec_net(self, image): ### Start an asynchronous request ### ### Return any necessary information ### ##log.info("Start Async Inference") self.input_blob[self.input_blob_name] = image self.feed_dict[self.input_blob_name] = image self.exec_network.start_async(request_id=0, inputs=self.feed_dict) return def wait(self): ### Wait for the request to be complete. ### ### Return any necessary information ### ### Note: You may need to update the function parameters. ### status = self.exec_network.requests[0].wait(-1) return status def get_output(self): ### Extract and return the output results ### Note: You may need to update the function parameters. ### res = self.exec_network.requests[0].outputs[self.output_blob] return res