def __init__(self, sources, loop): assert sources self.captures = [] self.transforms = [] self.fps = [] for src in sources: capture = open_images_capture(src, loop) self.captures.append(capture) self.fps.append(capture.fps())
def main(): args = build_argparser().parse_args() if args.labels: with open(args.labels) as f: labels = [line.strip() for line in f] else: labels = None log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() if 'MYRIAD' in args.device: myriad_config = {'VPU_HW_STAGES_OPTIMIZATION': 'YES'} core.set_config(myriad_config, 'MYRIAD') decoder_target_device = 'CPU' if args.device != 'CPU': encoder_target_device = args.device else: encoder_target_device = decoder_target_device models = [IEModel(args.m_encoder, core, encoder_target_device, model_type='Action Recognition Encoder', num_requests=(3 if args.device == 'MYRIAD' else 1))] if args.architecture_type == 'en-de': if args.m_decoder is None: raise RuntimeError('No decoder for encoder-decoder model type (-m_de) provided') models.append(IEModel(args.m_decoder, core, decoder_target_device, model_type='Action Recognition Decoder', num_requests=2)) seq_size = models[1].input_shape[1] elif args.architecture_type == 'en-mean': models.append(DummyDecoder(num_requests=2)) seq_size = args.decoder_seq_size elif args.architecture_type == 'i3d-rgb': seq_size = models[0].input_shape[1] presenter = monitors.Presenter(args.utilization_monitors, 70) result_presenter = ResultRenderer(no_show=args.no_show, presenter=presenter, output=args.output, limit=args.output_limit, labels=labels, label_smoothing_window=args.label_smoothing) cap = open_images_capture(args.input, args.loop) run_pipeline(cap, args.architecture_type, models, result_presenter.render_frame, args.raw_output_message, seq_size=seq_size, fps=cap.fps()) for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() log.info('Initializing Inference Engine...') ie = IECore() plugin_config = get_plugin_configs(args.device, args.num_streams, args.num_threads) log.info('Loading network...') model = get_model(ie, args) detector_pipeline = AsyncPipeline(ie, model, plugin_config, device=args.device, max_num_requests=args.num_infer_requests) ### READ TIME ### read_time_start = time.time() cap = open_images_capture(args.input, args.loop) read_time_end = time.time() next_frame_id = 0 next_frame_id_to_show = 0 image_id = 0 log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) palette = ColorPalette(len(model.labels) if model.labels else 100) metrics = PerformanceMetrics() presenter = None video_writer = cv2.VideoWriter() results_list = [] detection_ids = [1, 3, 4] all_starts = 0 while True: print('NEXT FRAME ID', next_frame_id) id = images[image_id] if next_frame_id == 5000: break if detector_pipeline.callback_exceptions: raise detector_pipeline.callback_exceptions[0] # Process all completed requests #### DETECTION TIME #### detect_time_start = time.time() results = detector_pipeline.get_result(next_frame_id_to_show) detect_time_end = time.time() detect_time_list.append(detect_time_end - detect_time_start) if results: objects, frame_meta = results for detection in objects: x = float(detection.xmin) y = float(detection.ymin) w = float(detection.xmax - detection.xmin) h = float(detection.ymax - detection.ymin) cls = detection.id cls = yolo_to_ssd_classes[cls] id = str(id.lstrip('0').split('.')[0]) conf = detection.score # if cls in detection_ids: results_list.append({ 'image_id': int(id), 'category_id': cls, 'bbox': [x, y, w, h], 'score': float(conf) }) frame = frame_meta['frame'] post_process_start = time.time() start_time = frame_meta['start_time'] all_starts += start_time all_starts += start_time if len(objects) and args.raw_output_message: print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold, images[image_id]) presenter.drawGraphs(frame) frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold, images[image_id]) metrics.update(start_time, frame) post_process_end = time.time() post_process_list.append(post_process_end - post_process_start) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: # cv2.imshow('Detection Results', frame) cv2.imwrite( f"/home/sovit/my_data/Data_Science/Projects/openvino_experiments/model_quantization/data/images/image_{image_id}.jpg", frame) # key = cv2.waitKey(1) ESC_KEY = 27 # Quit. #if key in {ord('q'), ord('Q'), ESC_KEY}: #break #presenter.handleKey(key) next_frame_id_to_show += 1 image_id += 1 continue if detector_pipeline.is_ready(): # Get new image/frame pre_process_start = time.time() start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: presenter = monitors.Presenter( args.utilization_monitors, 55, (round(frame.shape[1] / 4), round(frame.shape[0] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") # Submit for inference detector_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) pre_process_end = time.time() pre_process_list.append(pre_process_end - pre_process_start) next_frame_id += 1 else: # Wait for empty request detector_pipeline.await_any() results_file = 'results.json' with open(results_file, 'w') as f: f.write(json.dumps(results_list, indent=4)) detector_pipeline.await_all() # Process completed requests while detector_pipeline.has_completed_request(): results = detector_pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results frame = frame_meta['frame'] post_process_two_start = time.time() start_time = frame_meta['start_time'] if len(objects) and args.raw_output_message: print() # print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold) presenter.drawGraphs(frame) # frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold) metrics.update(start_time, frame) post_process_two_end = time.time() post_process_list_two.append(post_process_two_end - post_process_two_start) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: # cv2.imshow('Detection Results', frame) cv2.imwrite( f"/home/sovit/my_data/Data_Science/Projects/openvino_experiments/model_quantization/data/images/image_{frame_id}.jpg", frame) # key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) next_frame_id_to_show += 1 else: break metrics.print_total() print("Presentor", presenter.reportMeans())
def main(): """ Main function. """ log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() place_recognition = PlaceRecognition(args.model, args.device, args.gallery_folder, args.cpu_extension, args.gallery_size) cap = open_images_capture(args.input, args.loop) compute_embeddings_times = [] search_in_gallery_times = [] frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 0) video_writer = cv2.VideoWriter() while True: frame = cap.read() if frame is None: if frames_processed == 0: raise ValueError("Can't read an image from the input") break elapsed, probe_embedding = time_elapsed( place_recognition.compute_embedding, frame) compute_embeddings_times.append(elapsed) elapsed, (sorted_indexes, distances) = time_elapsed( place_recognition.search_in_gallery, probe_embedding) search_in_gallery_times.append(elapsed) image, key = visualize( frame, [str(place_recognition.impaths[i]) for i in sorted_indexes], distances[sorted_indexes], place_recognition.input_size, np.mean(compute_embeddings_times), np.mean(search_in_gallery_times), imshow_delay=3, presenter=presenter, no_show=args.no_show) if frames_processed == 0: if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (image.shape[1], image.shape[0])): raise RuntimeError("Can't open video writer") frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(image) if key == 27: break print(presenter.reportMeans())
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) next_frame_id = 1 next_frame_id_to_show = 0 metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() video_writer = cv2.VideoWriter() plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter( create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests, model_parameters={'input_layouts': args.layout}) start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") config = { 'target_size': args.tsize, 'aspect_ratio': frame.shape[1] / frame.shape[0], 'confidence_threshold': args.prob_threshold, 'padding_mode': 'center' if args.architecture_type == 'higherhrnet' else None, # the 'higherhrnet' and 'ae' specific 'delta': 0.5 if args.architecture_type == 'higherhrnet' else None, # the 'higherhrnet' and 'ae' specific } model = ImageModel.create_model(ARCHITECTURES[args.architecture_type], model_adapter, config) model.log_layers_info() hpe_pipeline = AsyncPipeline(model) hpe_pipeline.submit_data(frame, 0, { 'frame': frame, 'start_time': start_time }) output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") while True: if hpe_pipeline.callback_exceptions: raise hpe_pipeline.callback_exceptions[0] # Process all completed requests results = hpe_pipeline.get_result(next_frame_id_to_show) if results: (poses, scores), frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(poses) and args.raw_output_message: print_raw_results(poses, scores, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_poses(frame, poses, args.prob_threshold, output_transform) render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Pose estimation results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) continue if hpe_pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: break # Submit for inference hpe_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request hpe_pipeline.await_any() hpe_pipeline.await_all() if hpe_pipeline.callback_exceptions: raise hpe_pipeline.callback_exceptions[0] # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = hpe_pipeline.get_result(next_frame_id_to_show) (poses, scores), frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(poses) and args.raw_output_message: print_raw_results(poses, scores, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_poses(frame, poses, args.prob_threshold, output_transform) render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Pose estimation results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), hpe_pipeline.preprocess_metrics.get_latency(), hpe_pipeline.inference_metrics.get_latency(), hpe_pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def run_demo(args): ie = IECore() detector_person = Detector(ie, path_to_model_xml=args.model_od, device=args.device, label_class=args.person_label) single_human_pose_estimator = HumanPoseEstimator( ie, path_to_model_xml=args.model_hpe, device=args.device) cap = open_images_capture(args.input, args.loop) frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) video_writer = cv2.VideoWriter() if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 25) while frame is not None: bboxes = detector_person.detect(frame) human_poses = [ single_human_pose_estimator.estimate(frame, bbox) for bbox in bboxes ] presenter.drawGraphs(frame) colors = [(0, 0, 255), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0)] for pose, bbox in zip(human_poses, bboxes): cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0), 2) for id_kpt, kpt in enumerate(pose): cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[id_kpt], -1) cv2.putText( frame, 'summary: {:.1f} FPS (estimation: {:.1f} FPS / detection: {:.1f} FPS)' .format( float(1 / (detector_person.infer_time + single_human_pose_estimator.infer_time * len(human_poses))), float(1 / single_human_pose_estimator.infer_time), float(1 / detector_person.infer_time)), (5, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 200)) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: cv2.imshow('Human Pose Estimation Demo', frame) key = cv2.waitKey(delay) if key == 27: break presenter.handleKey(key) frame = cap.read() print(presenter.reportMeans())
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # Plugin initialization for specified device and load extensions library if specified. log.info('Creating Inference Engine...') ie = IECore() if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, 'CPU') # Read IR log.info('Loading Mask-RCNN network') mask_rcnn_net = ie.read_network( args.mask_rcnn_model, os.path.splitext(args.mask_rcnn_model)[0] + '.bin') log.info('Loading encoder part of text recognition network') text_enc_net = ie.read_network( args.text_enc_model, os.path.splitext(args.text_enc_model)[0] + '.bin') log.info('Loading decoder part of text recognition network') text_dec_net = ie.read_network( args.text_dec_model, os.path.splitext(args.text_dec_model)[0] + '.bin') model_required_inputs = {'image'} if set(mask_rcnn_net.input_info) == model_required_inputs: required_output_keys = {'boxes', 'labels', 'masks', 'text_features.0'} n, c, h, w = mask_rcnn_net.input_info['image'].input_data.shape else: raise RuntimeError( 'Demo supports only topologies with the following input keys: ' f'{model_required_inputs}.') assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \ f'Demo supports only topologies with the following output keys: {required_output_keys}' \ f'Found: {mask_rcnn_net.outputs.keys()}.' assert n == 1, 'Only batch 1 is supported by the demo application' log.info('Loading IR to the plugin...') mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net, device_name=args.device, num_requests=2) text_enc_exec_net = ie.load_network(network=text_enc_net, device_name=args.device) text_dec_exec_net = ie.load_network(network=text_dec_net, device_name=args.device) hidden_shape = text_dec_net.input_info[ args.trd_input_prev_hidden].input_data.shape del mask_rcnn_net del text_enc_net del text_dec_net cap = open_images_capture(args.input, args.loop) frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") if args.no_track: tracker = None else: tracker = StaticIOUTracker() if args.delay: delay = args.delay else: delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) visualizer = Visualizer(['__background__', 'text'], show_boxes=args.show_boxes, show_scores=args.show_scores) render_time = 0 frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 45, (frame.shape[1] // 4, frame.shape[0] // 8)) video_writer = cv2.VideoWriter() if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) while frame is not None: if not args.keep_aspect_ratio: # Resize the image to a target size. scale_x = w / frame.shape[1] scale_y = h / frame.shape[0] input_image = cv2.resize(frame, (w, h)) else: # Resize the image to keep the same aspect ratio and to fit it to a window of a target size. scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1]) input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y) input_image_size = input_image.shape[:2] input_image = np.pad(input_image, ((0, h - input_image_size[0]), (0, w - input_image_size[1]), (0, 0)), mode='constant', constant_values=0) # Change data layout from HWC to CHW. input_image = input_image.transpose((2, 0, 1)) input_image = input_image.reshape((n, c, h, w)).astype(np.float32) # Run the net. inf_start = time.time() outputs = mask_rcnn_exec_net.infer({'image': input_image}) # Parse detection results of the current request boxes = outputs['boxes'][:, :4] scores = outputs['boxes'][:, 4] classes = outputs['labels'].astype(np.uint32) raw_masks = outputs['masks'] text_features = outputs['text_features.0'] # Filter out detections with low confidence. detections_filter = scores > args.prob_threshold scores = scores[detections_filter] classes = classes[detections_filter] boxes = boxes[detections_filter] raw_masks = raw_masks[detections_filter] text_features = text_features[detections_filter] boxes[:, 0::2] /= scale_x boxes[:, 1::2] /= scale_y masks = [] for box, cls, raw_mask in zip(boxes, classes, raw_masks): mask = segm_postprocess(box, raw_mask, frame.shape[0], frame.shape[1]) masks.append(mask) texts = [] for feature in text_features: feature = text_enc_exec_net.infer({'input': feature})['output'] feature = np.reshape(feature, (feature.shape[0], feature.shape[1], -1)) feature = np.transpose(feature, (0, 2, 1)) hidden = np.zeros(hidden_shape) prev_symbol_index = np.ones((1, )) * SOS_INDEX text = '' text_confidence = 1.0 for i in range(MAX_SEQ_LEN): decoder_output = text_dec_exec_net.infer({ args.trd_input_prev_symbol: prev_symbol_index, args.trd_input_prev_hidden: hidden, args.trd_input_encoder_outputs: feature }) symbols_distr = decoder_output[args.trd_output_symbols_distr] symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0] prev_symbol_index = int(np.argmax(symbols_distr, axis=1)) text_confidence *= symbols_distr_softmaxed[prev_symbol_index] if prev_symbol_index == EOS_INDEX: break text += args.alphabet[prev_symbol_index] hidden = decoder_output[args.trd_output_cur_hidden] texts.append(text if text_confidence >= args.tr_threshold else '') inf_end = time.time() inf_time = inf_end - inf_start render_start = time.time() if len(boxes) and args.raw_output_message: log.info('Detected boxes:') log.info( ' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX ' ) for box, cls, score, mask in zip(boxes, classes, scores, masks): log.info( '{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} ' .format(cls, score, *box)) # Get instance track IDs. masks_tracks_ids = None if tracker is not None: masks_tracks_ids = tracker(masks, classes) presenter.drawGraphs(frame) # Visualize masks. frame = visualizer(frame, boxes, classes, scores, masks, texts, masks_tracks_ids) # Draw performance stats. inf_time_message = 'Inference and post-processing time: {:.3f} ms'.format( inf_time * 1000) render_time_message = 'OpenCV rendering time: {:.3f} ms'.format( render_time * 1000) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) # Print performance counters. if args.perf_counts: perf_counts = mask_rcnn_exec_net.requests[0].get_perf_counts() log.info('Performance counters:') print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format( 'name', 'layer_type', 'exet_type', 'status', 'real_time, us')) for layer, stats in perf_counts.items(): print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format( layer, stats['layer_type'], stats['exec_type'], stats['status'], stats['real_time'])) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: # Show resulting image. cv2.imshow('Results', frame) render_end = time.time() render_time = render_end - render_start if not args.no_show: key = cv2.waitKey(delay) esc_code = 27 if key == esc_code: break presenter.handleKey(key) frame = cap.read() print(presenter.reportMeans()) cv2.destroyAllWindows()
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) next_frame_id = 1 next_frame_id_to_show = 0 metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() video_writer = cv2.VideoWriter() if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter( create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") model = Deblurring(model_adapter, preload=False) model.reshape(frame.shape) model.log_layers_info() pipeline = AsyncPipeline(model) pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time}) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(frame.shape[1] / 4), round(frame.shape[0] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (2 * frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: break # Submit for inference pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: result_frame, frame_meta = results input_frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() if input_frame.shape != result_frame.shape: input_frame = cv2.resize( input_frame, (result_frame.shape[1], result_frame.shape[0])) final_image = cv2.hconcat([input_frame, result_frame]) render_metrics.update(rendering_start_time) presenter.drawGraphs(final_image) metrics.update(start_time, final_image) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(final_image) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Deblurring Results', final_image) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break presenter.handleKey(key) pipeline.await_all() # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = pipeline.get_result(next_frame_id_to_show) while results is None: results = pipeline.get_result(next_frame_id_to_show) result_frame, frame_meta = results input_frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() if input_frame.shape != result_frame.shape: input_frame = cv2.resize( input_frame, (result_frame.shape[1], result_frame.shape[0])) final_image = cv2.hconcat([input_frame, result_frame]) render_metrics.update(rendering_start_time) presenter.drawGraphs(final_image) metrics.update(start_time, final_image) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(final_image) if not args.no_show: cv2.imshow('Deblurring Results', final_image) key = cv2.waitKey(1) metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), pipeline.preprocess_metrics.get_latency(), pipeline.inference_metrics.get_latency(), pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def run_demo(args): cap = open_images_capture(args.input, args.loop) log.info('OpenVINO Runtime') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading Object Detection model {}'.format(args.model_od)) detector_person = Detector(core, args.model_od, device=args.device, label_class=args.person_label) log.info('The Object Detection model {} is loaded to {}'.format(args.model_od, args.device)) log.info('Reading Human Pose Estimation model {}'.format(args.model_hpe)) single_human_pose_estimator = HumanPoseEstimator(core, args.model_hpe, device=args.device) log.info('The Human Pose Estimation model {} is loaded to {}'.format(args.model_hpe, args.device)) delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) video_writer = cv2.VideoWriter() frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 25) metrics = PerformanceMetrics() start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") while frame is not None: bboxes = detector_person.detect(frame) human_poses = [single_human_pose_estimator.estimate(frame, bbox) for bbox in bboxes] presenter.drawGraphs(frame) colors = [(0, 0, 255), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0), (255, 0, 0), (0, 255, 0)] for pose, bbox in zip(human_poses, bboxes): cv2.rectangle(frame, (bbox[0], bbox[1]), (bbox[0] + bbox[2], bbox[1] + bbox[3]), (255, 0, 0), 2) for id_kpt, kpt in enumerate(pose): cv2.circle(frame, (int(kpt[0]), int(kpt[1])), 3, colors[id_kpt], -1) metrics.update(start_time, frame) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: cv2.imshow('Human Pose Estimation Demo', frame) key = cv2.waitKey(delay) if key == 27: break presenter.handleKey(key) start_time = perf_counter() frame = cap.read() metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() if args.architecture_type != 'yolov4' and args.anchors: log.warning( 'The "--anchors" option works only for "-at==yolov4". Option will be omitted' ) if args.architecture_type != 'yolov4' and args.masks: log.warning( 'The "--masks" option works only for "-at==yolov4". Option will be omitted' ) if args.architecture_type not in ['nanodet', 'nanodet-plus' ] and args.num_classes: log.warning( 'The "--num_classes" option works only for "-at==nanodet" and "-at==nanodet-plus". Option will be omitted' ) cap = open_images_capture(args.input, args.loop) if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter( create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests, model_parameters={'input_layouts': args.layout}) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) configuration = { 'resize_type': args.resize_type, 'mean_values': args.mean_values, 'scale_values': args.scale_values, 'reverse_input_channels': args.reverse_input_channels, 'path_to_labels': args.labels, 'confidence_threshold': args.prob_threshold, 'input_size': args.input_size, # The CTPN specific 'num_classes': args.num_classes, # The NanoDet and NanoDetPlus specific } model = DetectionModel.create_model(args.architecture_type, model_adapter, configuration) model.log_layers_info() detector_pipeline = AsyncPipeline(model) next_frame_id = 0 next_frame_id_to_show = 0 palette = ColorPalette(len(model.labels) if model.labels else 100) metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() presenter = None output_transform = None video_writer = cv2.VideoWriter() while True: if detector_pipeline.callback_exceptions: raise detector_pipeline.callback_exceptions[0] # Process all completed requests results = detector_pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(objects) and args.raw_output_message: print_raw_results(objects, model.labels, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_detections(frame, objects, palette, model.labels, output_transform) render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Detection Results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) continue if detector_pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference detector_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request detector_pipeline.await_any() detector_pipeline.await_all() if detector_pipeline.callback_exceptions: raise detector_pipeline.callback_exceptions[0] # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = detector_pipeline.get_result(next_frame_id_to_show) objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(objects) and args.raw_output_message: print_raw_results(objects, model.labels, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_detections(frame, objects, palette, model.labels, output_transform) render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Detection Results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), detector_pipeline.preprocess_metrics.get_latency(), detector_pipeline.inference_metrics.get_latency(), detector_pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) model = MonoDepthModel(model_adapter) model.log_layers_info() pipeline = AsyncPipeline(model) next_frame_id = 0 next_frame_id_to_show = 0 metrics = PerformanceMetrics() presenter = None output_transform = None video_writer = cv2.VideoWriter() while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter(args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference pipeline.submit_data(frame, next_frame_id, {'start_time': start_time}) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: depth_map, frame_meta = results depth_map = apply_color_map(depth_map, output_transform) start_time = frame_meta['start_time'] presenter.drawGraphs(depth_map) metrics.update(start_time, depth_map) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): video_writer.write(depth_map) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow(DEMO_NAME, depth_map) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break presenter.handleKey(key) pipeline.await_all() # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = pipeline.get_result(next_frame_id_to_show) while results is None: results = pipeline.get_result(next_frame_id_to_show) depth_map, frame_meta = results depth_map = apply_color_map(depth_map, output_transform) start_time = frame_meta['start_time'] presenter.drawGraphs(depth_map) metrics.update(start_time, depth_map) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): video_writer.write(depth_map) if not args.no_show: cv2.imshow(DEMO_NAME, depth_map) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break presenter.handleKey(key) metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): metrics = PerformanceMetrics() args = build_argparser().parse_args() # Plugin initialization for specified device and load extensions library if specified log.info("Creating Inference Engine") ie = IECore() # Read IR log.info("Loading network files:\n\t{}".format(args.model_pnet)) p_net = ie.read_network(args.model_pnet) assert len(p_net.input_info.keys() ) == 1, "Pnet supports only single input topologies" assert len(p_net.outputs) == 2, "Pnet supports two output topologies" log.info("Loading network files:\n\t{}".format(args.model_rnet)) r_net = ie.read_network(args.model_rnet) assert len(r_net.input_info.keys() ) == 1, "Rnet supports only single input topologies" assert len(r_net.outputs) == 2, "Rnet supports two output topologies" log.info("Loading network files:\n\t{}".format(args.model_onet)) o_net = ie.read_network(args.model_onet) assert len(o_net.input_info.keys() ) == 1, "Onet supports only single input topologies" assert len(o_net.outputs) == 3, "Onet supports three output topologies" log.info("Preparing input blobs") pnet_input_blob = next(iter(p_net.input_info)) rnet_input_blob = next(iter(r_net.input_info)) onet_input_blob = next(iter(o_net.input_info)) log.info("Preparing output blobs") for name, blob in p_net.outputs.items(): if blob.shape[1] == 2: pnet_cls_name = name elif blob.shape[1] == 4: pnet_roi_name = name else: raise RuntimeError("Unsupported output layer for Pnet") for name, blob in r_net.outputs.items(): if blob.shape[1] == 2: rnet_cls_name = name elif blob.shape[1] == 4: rnet_roi_name = name else: raise RuntimeError("Unsupported output layer for Rnet") for name, blob in o_net.outputs.items(): if blob.shape[1] == 2: onet_cls_name = name elif blob.shape[1] == 4: onet_roi_name = name elif blob.shape[1] == 10: onet_pts_name = name else: raise RuntimeError("Unsupported output layer for Onet") cap = open_images_capture(args.input, args.loop) next_frame_id = 0 log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) presenter = None video_writer = cv2.VideoWriter() while True: start_time = perf_counter() origin_image = cap.read() if origin_image is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: presenter = monitors.Presenter(args.utilization_monitors, 55, (round(origin_image.shape[1] / 4), round(origin_image.shape[0] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (origin_image.shape[1], origin_image.shape[0])): raise RuntimeError("Can't open video writer") next_frame_id += 1 rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB) oh, ow, _ = rgb_image.shape scales = utils.calculate_scales(rgb_image) # ************************************* # Pnet stage # ************************************* log.info("Loading Pnet model to the plugin") t0 = cv2.getTickCount() pnet_res = [] for scale in scales: hs = int(oh * scale) ws = int(ow * scale) image = preprocess_image(rgb_image, ws, hs) p_net.reshape({pnet_input_blob: [1, 3, ws, hs]}) # Change weidth and height of input blob exec_pnet = ie.load_network(network=p_net, device_name=args.device) p_res = exec_pnet.infer(inputs={pnet_input_blob: image}) pnet_res.append(p_res) image_num = len(scales) rectangles = [] for i in range(image_num): roi = pnet_res[i][pnet_roi_name] cls = pnet_res[i][pnet_cls_name] _, _, out_h, out_w = cls.shape out_side = max(out_h, out_w) rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side, 1 / scales[i], ow, oh, score_threshold[0], iou_threshold[0]) rectangles.extend(rectangle) rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou') # Rnet stage if len(rectangles) > 0: log.info("Loading Rnet model to the plugin") r_net.reshape({rnet_input_blob: [len(rectangles), 3, 24, 24]}) # Change batch size of input blob exec_rnet = ie.load_network(network=r_net, device_name=args.device) rnet_input = [] for rectangle in rectangles: crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] crop_img = preprocess_image(crop_img, 24, 24) rnet_input.extend(crop_img) rnet_res = exec_rnet.infer(inputs={rnet_input_blob: rnet_input}) roi = rnet_res[rnet_roi_name] cls = rnet_res[rnet_cls_name] rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh, score_threshold[1], iou_threshold[2]) # Onet stage if len(rectangles) > 0: log.info("Loading Onet model to the plugin") o_net.reshape({onet_input_blob: [len(rectangles), 3, 48, 48]}) # Change batch size of input blob exec_onet = ie.load_network(network=o_net, device_name=args.device) onet_input = [] for rectangle in rectangles: crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] crop_img = preprocess_image(crop_img, 48, 48) onet_input.extend(crop_img) onet_res = exec_onet.infer(inputs={onet_input_blob: onet_input}) roi = onet_res[onet_roi_name] cls = onet_res[onet_cls_name] pts = onet_res[onet_pts_name] rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow, oh, score_threshold[2], iou_threshold[3]) # display results for rectangle in rectangles: # Draw detected boxes cv2.putText(origin_image, 'confidence: {:.2f}'.format(rectangle[4]), (int(rectangle[0]), int(rectangle[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0)) cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])), (int(rectangle[2]), int(rectangle[3])), (255, 0, 0), 1) # Draw landmarks for i in range(5, 15, 2): cv2.circle(origin_image, (int(rectangle[i + 0]), int(rectangle[i + 1])), 2, (0, 255, 0)) infer_time = (cv2.getTickCount() - t0) / cv2.getTickFrequency() # Record infer time cv2.putText(origin_image, 'summary: {:.1f} FPS'.format(1.0 / infer_time), (5, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 200)) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id <= args.output_limit - 1): video_writer.write(origin_image) if not args.no_show: cv2.imshow('MTCNN Results', origin_image) key = cv2.waitKey(1) if key in {ord('q'), ord('Q'), 27}: break presenter.handleKey(key) metrics.update(start_time, origin_image) metrics.print_total()
def main(): parser = argparse.ArgumentParser(description='Whiteboard inpainting demo') parser.add_argument('-i', '--input', required=True, help='Required. Path to a video file or a device node of a web-camera.') parser.add_argument('--loop', default=False, action='store_true', help='Optional. Enable reading the input in a loop.') parser.add_argument('-o', '--output', required=False, help='Optional. Name of the output file(s) to save.') parser.add_argument('-limit', '--output_limit', required=False, default=1000, type=int, help='Optional. Number of frames to store in output. ' 'If 0 is set, all frames are stored.') parser.add_argument('-m_i', '--m_instance_segmentation', type=str, required=False, help='Required. Path to the instance segmentation model.') parser.add_argument('-m_s', '--m_semantic_segmentation', type=str, required=False, help='Required. Path to the semantic segmentation model.') parser.add_argument('-t', '--threshold', type=float, default=0.6, help='Optional. Threshold for person instance segmentation model.') parser.add_argument('--no_show', help="Optional. Don't show output.", action='store_true') parser.add_argument('-d', '--device', type=str, default='CPU', help='Optional. Specify a target device to infer on. CPU, GPU, HDDL or MYRIAD is ' 'acceptable. The demo will look for a suitable plugin for the device specified.') parser.add_argument('-l', '--cpu_extension', type=str, default=None, help='MKLDNN (CPU)-targeted custom layers. Absolute \ path to a shared library with the kernels impl.') parser.add_argument('-u', '--utilization_monitors', default='', type=str, help='Optional. List of monitors to show initially.') args = parser.parse_args() cap = open_images_capture(args.input, args.loop) if cap.get_type() not in ('VIDEO', 'CAMERA'): raise RuntimeError("The input should be a video file or a numeric camera ID") if bool(args.m_instance_segmentation) == bool(args.m_semantic_segmentation): raise ValueError('Set up exactly one of segmentation models: ' '--m_instance_segmentation or --m_semantic_segmentation') labels_dir = Path(__file__).resolve().parents[3] / 'data/dataset_classes' mouse = MouseClick() if not args.no_show: cv2.namedWindow(WINNAME) cv2.setMouseCallback(WINNAME, mouse.get_points) log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() model_path = args.m_instance_segmentation if args.m_instance_segmentation else args.m_semantic_segmentation log.info('Reading model {}'.format(model_path)) if args.m_instance_segmentation: labels_file = str(labels_dir / 'coco_80cl_bkgr.txt') segmentation = MaskRCNN(core, args.m_instance_segmentation, labels_file, args.threshold, args.device, args.cpu_extension) elif args.m_semantic_segmentation: labels_file = str(labels_dir / 'cityscapes_19cl_bkgr.txt') segmentation = SemanticSegmentation(core, args.m_semantic_segmentation, labels_file, args.threshold, args.device, args.cpu_extension) log.info('The model {} is loaded to {}'.format(model_path, args.device)) metrics = PerformanceMetrics() video_writer = cv2.VideoWriter() black_board = False frame_number = 0 key = -1 start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") out_frame_size = (frame.shape[1], frame.shape[0] * 2) output_frame = np.full((frame.shape[0], frame.shape[1], 3), 255, dtype='uint8') presenter = monitors.Presenter(args.utilization_monitors, 20, (out_frame_size[0] // 4, out_frame_size[1] // 16)) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), out_frame_size): raise RuntimeError("Can't open video writer") while frame is not None: mask = None detections = segmentation.get_detections([frame]) expand_mask(detections, frame.shape[1] // 27) if len(detections[0]) > 0: mask = detections[0][0][2] for i in range(1, len(detections[0])): mask = cv2.bitwise_or(mask, detections[0][i][2]) if mask is not None: mask = np.stack([mask, mask, mask], axis=-1) else: mask = np.zeros(frame.shape, dtype='uint8') clear_frame = remove_background(frame, invert_colors=not black_board) output_frame = np.where(mask, output_frame, clear_frame) merged_frame = np.vstack([frame, output_frame]) merged_frame = cv2.resize(merged_frame, out_frame_size) metrics.update(start_time, merged_frame) if video_writer.isOpened() and (args.output_limit <= 0 or frame_number <= args.output_limit-1): video_writer.write(merged_frame) presenter.drawGraphs(merged_frame) if not args.no_show: cv2.imshow(WINNAME, merged_frame) key = check_pressed_keys(key) if key == 27: # 'Esc' break if key == ord('i'): # catch pressing of key 'i' black_board = not black_board output_frame = 255 - output_frame else: presenter.handleKey(key) if mouse.crop_available: x0, x1 = min(mouse.points[0][0], mouse.points[1][0]), \ max(mouse.points[0][0], mouse.points[1][0]) y0, y1 = min(mouse.points[0][1], mouse.points[1][1]), \ max(mouse.points[0][1], mouse.points[1][1]) x1, y1 = min(x1, output_frame.shape[1] - 1), min(y1, output_frame.shape[0] - 1) board = output_frame[y0: y1, x0: x1, :] if board.shape[0] > 0 and board.shape[1] > 0: cv2.namedWindow('Board', cv2.WINDOW_KEEPRATIO) cv2.imshow('Board', board) frame_number += 1 start_time = perf_counter() frame = cap.read() metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) place_recognition = PlaceRecognition(args.model, args.device, args.gallery_folder, args.gallery_size) compute_embeddings_times = [] search_in_gallery_times = [] frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 0) video_writer = cv2.VideoWriter() metrics = PerformanceMetrics() while True: start_time = perf_counter() frame = cap.read() if frame is None: if frames_processed == 0: raise ValueError("Can't read an image from the input") break elapsed, probe_embedding = time_elapsed( place_recognition.compute_embedding, frame) compute_embeddings_times.append(elapsed) elapsed, (sorted_indexes, distances) = time_elapsed( place_recognition.search_in_gallery, probe_embedding) search_in_gallery_times.append(elapsed) image, key = visualize( frame, [str(place_recognition.impaths[i]) for i in sorted_indexes], distances[sorted_indexes], place_recognition.input_size, np.mean(compute_embeddings_times), np.mean(search_in_gallery_times), imshow_delay=3, presenter=presenter, no_show=args.no_show) metrics.update(start_time) if frames_processed == 0: if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (image.shape[1], image.shape[0])): raise RuntimeError("Can't open video writer") frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(image) if key == 27: break metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() if args.labels: with open(args.labels) as f: labels = [line.strip() for line in f] else: labels = None ie = IECore() if 'MYRIAD' in args.device: myriad_config = {'VPU_HW_STAGES_OPTIMIZATION': 'YES'} ie.set_config(myriad_config, 'MYRIAD') if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, 'CPU') decoder_target_device = 'CPU' if args.device != 'CPU': encoder_target_device = args.device else: encoder_target_device = decoder_target_device encoder_xml = args.m_encoder encoder_bin = args.m_encoder.replace('.xml', '.bin') models = [ IEModel(encoder_xml, encoder_bin, ie, encoder_target_device, num_requests=(3 if args.device == 'MYRIAD' else 1)) ] if args.architecture_type == 'en-de': if args.m_decoder is None: raise RuntimeError( 'No decoder for encoder-decoder model type (-m_de) provided') decoder_xml = args.m_decoder decoder_bin = args.m_decoder.replace('.xml', '.bin') models.append( IEModel(decoder_xml, decoder_bin, ie, decoder_target_device, num_requests=2)) seq_size = models[1].input_size[1] elif args.architecture_type == 'en-mean': models.append(DummyDecoder(num_requests=2)) seq_size = args.decoder_seq_size elif args.architecture_type == 'i3d-rgb': seq_size = models[0].input_size[2] presenter = monitors.Presenter(args.utilization_monitors, 70) result_presenter = ResultRenderer( no_show=args.no_show, presenter=presenter, output=args.output, limit=args.output_limit, labels=labels, label_smoothing_window=args.label_smoothing) cap = open_images_capture(args.input, args.loop) run_pipeline(cap, args.architecture_type, models, result_presenter.render_frame, seq_size=seq_size, fps=cap.fps()) print(presenter.reportMeans())
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) delay = int(cap.get_type() in {'VIDEO', 'CAMERA'}) if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter( create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) config = { 'mean_values': args.mean_values, 'scale_values': args.scale_values, 'reverse_input_channels': args.reverse_input_channels, 'topk': args.topk, 'path_to_labels': args.labels } model = Classification(model_adapter, config) model.log_layers_info() async_pipeline = AsyncPipeline(model) next_frame_id = 0 next_frame_id_to_show = 0 metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() presenter = None output_transform = None video_writer = cv2.VideoWriter() ESC_KEY = 27 key = -1 while True: if async_pipeline.callback_exceptions: raise async_pipeline.callback_exceptions[0] # Process all completed requests results = async_pipeline.get_result(next_frame_id_to_show) if results: classifications, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if args.raw_output_message: print_raw_results(classifications, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_labels(frame, classifications, output_transform) if delay or args.no_show: render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Classification Results', frame) key = cv2.waitKey(delay) # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) continue if async_pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference async_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request async_pipeline.await_any() async_pipeline.await_all() if key not in {ord('q'), ord('Q'), ESC_KEY}: # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = async_pipeline.get_result(next_frame_id_to_show) while results is None: results = async_pipeline.get_result(next_frame_id_to_show) classifications, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if args.raw_output_message: print_raw_results(classifications, next_frame_id_to_show) presenter.drawGraphs(frame) rendering_start_time = perf_counter() frame = draw_labels(frame, classifications, output_transform) if delay or args.no_show: render_metrics.update(rendering_start_time) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Classification Results', frame) key = cv2.waitKey(delay) # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) if delay or args.no_show: metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), async_pipeline.preprocess_metrics.get_latency(), async_pipeline.inference_metrics.get_latency(), async_pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) # Plugin initialization for specified device and load extensions library if specified. log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) ie = IECore() if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, 'CPU') # Read IR log.info('Reading Mask-RCNN model {}'.format(args.mask_rcnn_model)) mask_rcnn_net = ie.read_network(args.mask_rcnn_model) model_required_inputs = {'image'} if set(mask_rcnn_net.input_info) == model_required_inputs: required_output_keys = {'boxes', 'labels', 'masks', 'text_features.0'} n, c, h, w = mask_rcnn_net.input_info['image'].input_data.shape assert n == 1, 'Only batch 1 is supported by the demo application' else: raise RuntimeError( 'Demo supports only topologies with the following input keys: ' f'{model_required_inputs}.') assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \ f'Demo supports only topologies with the following output keys: {required_output_keys}' \ f'Found: {mask_rcnn_net.outputs.keys()}.' log.info('Reading Text Recognition Encoder model {}'.format( args.text_enc_model)) text_enc_net = ie.read_network(args.text_enc_model) log.info('Reading Text Recognition Decoder model {}'.format( args.text_dec_model)) text_dec_net = ie.read_network(args.text_dec_model) mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net, device_name=args.device, num_requests=2) log.info('The Mask-RCNN model {} is loaded to {}'.format( args.mask_rcnn_model, args.device)) text_enc_exec_net = ie.load_network(network=text_enc_net, device_name=args.device) log.info('The Text Recognition Encoder model {} is loaded to {}'.format( args.text_enc_model, args.device)) text_dec_exec_net = ie.load_network(network=text_dec_net, device_name=args.device) log.info('The Text Recognition Decoder model {} is loaded to {}'.format( args.text_dec_model, args.device)) hidden_shape = text_dec_net.input_info[ args.trd_input_prev_hidden].input_data.shape del mask_rcnn_net del text_enc_net del text_dec_net if args.no_track: tracker = None else: tracker = StaticIOUTracker() if args.delay: delay = args.delay else: delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) visualizer = Visualizer(['__background__', 'text'], show_boxes=args.show_boxes, show_scores=args.show_scores) frames_processed = 0 metrics = PerformanceMetrics() video_writer = cv2.VideoWriter() start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") presenter = monitors.Presenter(args.utilization_monitors, 45, (frame.shape[1] // 4, frame.shape[0] // 8)) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") while frame is not None: if not args.keep_aspect_ratio: # Resize the image to a target size. scale_x = w / frame.shape[1] scale_y = h / frame.shape[0] input_image = cv2.resize(frame, (w, h)) else: # Resize the image to keep the same aspect ratio and to fit it to a window of a target size. scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1]) input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y) input_image_size = input_image.shape[:2] input_image = np.pad(input_image, ((0, h - input_image_size[0]), (0, w - input_image_size[1]), (0, 0)), mode='constant', constant_values=0) # Change data layout from HWC to CHW. input_image = input_image.transpose((2, 0, 1)) input_image = input_image.reshape((n, c, h, w)).astype(np.float32) # Run the net. outputs = mask_rcnn_exec_net.infer({'image': input_image}) # Parse detection results of the current request boxes = outputs['boxes'][:, :4] scores = outputs['boxes'][:, 4] classes = outputs['labels'].astype(np.uint32) raw_masks = outputs['masks'] text_features = outputs['text_features.0'] # Filter out detections with low confidence. detections_filter = scores > args.prob_threshold scores = scores[detections_filter] classes = classes[detections_filter] boxes = boxes[detections_filter] raw_masks = raw_masks[detections_filter] text_features = text_features[detections_filter] boxes[:, 0::2] /= scale_x boxes[:, 1::2] /= scale_y masks = [] for box, cls, raw_mask in zip(boxes, classes, raw_masks): mask = segm_postprocess(box, raw_mask, frame.shape[0], frame.shape[1]) masks.append(mask) texts = [] for feature in text_features: feature = text_enc_exec_net.infer({'input': feature})['output'] feature = np.reshape(feature, (feature.shape[0], feature.shape[1], -1)) feature = np.transpose(feature, (0, 2, 1)) hidden = np.zeros(hidden_shape) prev_symbol_index = np.ones((1, )) * SOS_INDEX text = '' text_confidence = 1.0 for i in range(MAX_SEQ_LEN): decoder_output = text_dec_exec_net.infer({ args.trd_input_prev_symbol: prev_symbol_index, args.trd_input_prev_hidden: hidden, args.trd_input_encoder_outputs: feature }) symbols_distr = decoder_output[args.trd_output_symbols_distr] symbols_distr_softmaxed = softmax(symbols_distr, axis=1)[0] prev_symbol_index = int(np.argmax(symbols_distr, axis=1)) text_confidence *= symbols_distr_softmaxed[prev_symbol_index] if prev_symbol_index == EOS_INDEX: break text += args.alphabet[prev_symbol_index] hidden = decoder_output[args.trd_output_cur_hidden] texts.append(text if text_confidence >= args.tr_threshold else '') if len(boxes) and args.raw_output_message: log.debug( ' -------------------------- Frame # {} -------------------------- ' .format(frames_processed)) log.debug( ' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX ' ) for box, cls, score, mask in zip(boxes, classes, scores, masks): log.debug( '{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} ' .format(cls, score, *box)) # Get instance track IDs. masks_tracks_ids = None if tracker is not None: masks_tracks_ids = tracker(masks, classes) presenter.drawGraphs(frame) # Visualize masks. frame = visualizer(frame, boxes, classes, scores, masks, texts, masks_tracks_ids) metrics.update(start_time, frame) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: # Show resulting image. cv2.imshow('Results', frame) if not args.no_show: key = cv2.waitKey(delay) esc_code = 27 if key == esc_code: break presenter.handleKey(key) start_time = perf_counter() frame = cap.read() metrics.log_total() for rep in presenter.reportMeans(): log.info(rep) cv2.destroyAllWindows()
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) target_bgr = open_images_capture(args.target_bgr, loop=True) if args.target_bgr else None if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter(create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) labels = ['__background__', 'person'] if args.labels is None else load_labels(args.labels) assert len(labels), 'The file with class labels is empty' configuration = { 'confidence_threshold': args.prob_threshold, 'resize_type': args.resize_type } model, need_bgr_input = get_model(model_adapter, configuration, args) input_bgr = open_images_capture(args.background, False).read() if need_bgr_input else None person_id = -1 for i, label in enumerate(labels): if label == 'person': person_id = i break assert person_id >= 0, 'Person class did not find in labels list.' model.log_layers_info() pipeline = AsyncPipeline(model) next_frame_id = 0 next_frame_id_to_show = 0 metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() presenter = None output_transform = None video_writer = cv2.VideoWriter() while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() bgr = target_bgr.read() if target_bgr is not None else None if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter(args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), tuple(output_resolution)): raise RuntimeError("Can't open video writer") # Submit for inference data = {'src': frame, 'bgr': input_bgr} if input_bgr is not None else frame pipeline.submit_data(data, next_frame_id, {'frame': frame, 'start_time': start_time}) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results if args.raw_output_message: print_raw_results(objects, next_frame_id_to_show) frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() frame = render_results(frame, objects, output_resolution, bgr, person_id, args.blur_bgr, args.show_with_original_frame) render_metrics.update(rendering_start_time) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Background subtraction results', frame) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break presenter.handleKey(key) pipeline.await_all() # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = pipeline.get_result(next_frame_id_to_show) while results is None: results = pipeline.get_result(next_frame_id_to_show) objects, frame_meta = results if args.raw_output_message: print_raw_results(objects, next_frame_id_to_show, model.labels) frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() frame = render_results(frame, objects, output_resolution, bgr, person_id, args.blur_bgr, args.show_with_original_frame) render_metrics.update(rendering_start_time) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit-1): video_writer.write(frame) if not args.no_show: cv2.imshow('Background subtraction results', frame) cv2.waitKey(1) metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), pipeline.preprocess_metrics.get_latency(), pipeline.inference_metrics.get_latency(), pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() # Plugin initialization for specified device and load extensions library if specified. log.info('Creating Inference Engine...') ie = IECore() if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, 'CPU') # Read IR log.info('Loading network') net = ie.read_network(args.model, args.model.with_suffix('.bin')) image_input, image_info_input, ( n, c, h, w), model_type, postprocessor = check_model(net) args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio log.info('Loading IR to the plugin...') exec_net = ie.load_network(network=net, device_name=args.device, num_requests=2) cap = open_images_capture(args.input, args.loop) frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") if args.no_track: tracker = None else: tracker = StaticIOUTracker() with open(args.labels, 'rt') as labels_file: class_labels = labels_file.read().splitlines() if args.delay: delay = args.delay else: delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) frames_processed = 0 out_frame_size = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 45, (round(out_frame_size[0] / 4), round(out_frame_size[1] / 8))) visualizer = Visualizer(class_labels, show_boxes=args.show_boxes, show_scores=args.show_scores) video_writer = cv2.VideoWriter() if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), out_frame_size): raise RuntimeError("Can't open video writer") render_time = 0 log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) while frame is not None: if args.no_keep_aspect_ratio: # Resize the image to a target size. scale_x = w / frame.shape[1] scale_y = h / frame.shape[0] input_image = cv2.resize(frame, (w, h)) else: # Resize the image to keep the same aspect ratio and to fit it to a window of a target size. scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1]) input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y) input_image_size = input_image.shape[:2] input_image = np.pad(input_image, ((0, h - input_image_size[0]), (0, w - input_image_size[1]), (0, 0)), mode='constant', constant_values=0) # Change data layout from HWC to CHW. input_image = input_image.transpose((2, 0, 1)) input_image = input_image.reshape((n, c, h, w)).astype(np.float32) input_image_info = np.asarray( [[input_image_size[0], input_image_size[1], 1]], dtype=np.float32) # Run the net. inf_start = time.time() feed_dict = {image_input: input_image} if image_info_input: feed_dict[image_info_input] = input_image_info outputs = exec_net.infer(feed_dict) inf_end = time.time() det_time = inf_end - inf_start # Parse detection results of the current request scores, classes, boxes, masks = postprocessor(outputs, scale_x, scale_y, *frame.shape[:2], h, w, args.prob_threshold) render_start = time.time() if len(boxes) and args.raw_output_message: log.info('Detected boxes:') log.info( ' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX ' ) for box, cls, score, mask in zip(boxes, classes, scores, masks): log.info( '{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} ' .format(cls, score, *box)) # Get instance track IDs. masks_tracks_ids = None if tracker is not None: masks_tracks_ids = tracker(masks, classes) # Visualize masks. frame = visualizer(frame, boxes, classes, scores, presenter, masks, masks_tracks_ids) # Draw performance stats. inf_time_message = 'Inference time: {:.3f} ms'.format(det_time * 1000) render_time_message = 'OpenCV rendering time: {:.3f} ms'.format( render_time * 1000) cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1) cv2.putText(frame, render_time_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1) # Print performance counters. if args.perf_counts: perf_counts = exec_net.requests[0].get_perf_counts() log.info('Performance counters:') print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format( 'name', 'layer_type', 'exet_type', 'status', 'real_time, us')) for layer, stats in perf_counts.items(): print('{:<70} {:<15} {:<15} {:<15} {:<10}'.format( layer, stats['layer_type'], stats['exec_type'], stats['status'], stats['real_time'])) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: # Show resulting image. cv2.imshow('Results', frame) render_end = time.time() render_time = render_end - render_start if not args.no_show: key = cv2.waitKey(delay) esc_code = 27 if key == esc_code: break presenter.handleKey(key) frame = cap.read() print(presenter.reportMeans()) cv2.destroyAllWindows()
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) with open(args.labels, 'rt') as labels_file: class_labels = labels_file.read().splitlines() assert len(class_labels), 'The file with class labels is empty' # Plugin initialization for specified device and load extensions library if specified. log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) ie = IECore() if args.cpu_extension and 'CPU' in args.device: ie.add_extension(args.cpu_extension, 'CPU') # Read IR log.info('Reading model {}'.format(args.model)) net = ie.read_network(args.model, args.model.with_suffix('.bin')) image_input, image_info_input, ( n, c, h, w), model_type, postprocessor = check_model(net) args.no_keep_aspect_ratio = model_type == 'yolact' or args.no_keep_aspect_ratio exec_net = ie.load_network(network=net, device_name=args.device, num_requests=2) log.info('The model {} is loaded to {}'.format(args.model, args.device)) if args.no_track: tracker = None else: tracker = StaticIOUTracker() if args.delay: delay = args.delay else: delay = int(cap.get_type() in ('VIDEO', 'CAMERA')) frames_processed = 0 metrics = PerformanceMetrics() visualizer = Visualizer(class_labels, show_boxes=args.show_boxes, show_scores=args.show_scores) video_writer = cv2.VideoWriter() start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") out_frame_size = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 45, (round(out_frame_size[0] / 4), round(out_frame_size[1] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), out_frame_size): raise RuntimeError("Can't open video writer") while frame is not None: if args.no_keep_aspect_ratio: # Resize the image to a target size. scale_x = w / frame.shape[1] scale_y = h / frame.shape[0] input_image = cv2.resize(frame, (w, h)) else: # Resize the image to keep the same aspect ratio and to fit it to a window of a target size. scale_x = scale_y = min(h / frame.shape[0], w / frame.shape[1]) input_image = cv2.resize(frame, None, fx=scale_x, fy=scale_y) input_image_size = input_image.shape[:2] input_image = np.pad(input_image, ((0, h - input_image_size[0]), (0, w - input_image_size[1]), (0, 0)), mode='constant', constant_values=0) # Change data layout from HWC to CHW. input_image = input_image.transpose((2, 0, 1)) input_image = input_image.reshape((n, c, h, w)).astype(np.float32) input_image_info = np.asarray( [[input_image_size[0], input_image_size[1], 1]], dtype=np.float32) # Run the net. feed_dict = {image_input: input_image} if image_info_input: feed_dict[image_info_input] = input_image_info outputs = exec_net.infer(feed_dict) # Parse detection results of the current request scores, classes, boxes, masks = postprocessor(outputs, scale_x, scale_y, *frame.shape[:2], h, w, args.prob_threshold) if len(boxes) and args.raw_output_message: log.debug( ' -------------------------- Frame # {} -------------------------- ' .format(frames_processed)) log.debug( ' Class ID | Confidence | XMIN | YMIN | XMAX | YMAX ' ) for box, cls, score, mask in zip(boxes, classes, scores, masks): log.debug( '{:>10} | {:>10f} | {:>8.2f} | {:>8.2f} | {:>8.2f} | {:>8.2f} ' .format(cls, score, *box)) # Get instance track IDs. masks_tracks_ids = None if tracker is not None: masks_tracks_ids = tracker(masks, classes) # Visualize masks. frame = visualizer(frame, boxes, classes, scores, presenter, masks, masks_tracks_ids) metrics.update(start_time, frame) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(frame) if not args.no_show: # Show resulting image. cv2.imshow('Results', frame) if not args.no_show: key = cv2.waitKey(delay) esc_code = 27 if key == esc_code: break presenter.handleKey(key) start_time = perf_counter() frame = cap.read() metrics.log_total() for rep in presenter.reportMeans(): log.info(rep) cv2.destroyAllWindows()
def main(): args = build_argparser().parse_args() log.info('Initializing Inference Engine...') ie = IECore() plugin_config = get_plugin_configs(args.device, args.num_streams, args.num_threads) log.info('Loading network...') model = get_model(ie, args) detector_pipeline = AsyncPipeline(ie, model, plugin_config, device=args.device, max_num_requests=args.num_infer_requests) cap = open_images_capture(args.input, args.loop) next_frame_id = 0 next_frame_id_to_show = 0 log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) palette = ColorPalette(len(model.labels) if model.labels else 100) metrics = PerformanceMetrics() presenter = None video_writer = cv2.VideoWriter() while True: if detector_pipeline.callback_exceptions: raise detector_pipeline.callback_exceptions[0] # Process all completed requests results = detector_pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(objects) and args.raw_output_message: print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold) presenter.drawGraphs(frame) frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Detection Results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) next_frame_id_to_show += 1 continue if detector_pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: presenter = monitors.Presenter( args.utilization_monitors, 55, (round(frame.shape[1] / 4), round(frame.shape[0] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") # Submit for inference detector_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request detector_pipeline.await_any() detector_pipeline.await_all() # Process completed requests while detector_pipeline.has_completed_request(): results = detector_pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(objects) and args.raw_output_message: print_raw_results(frame.shape[:2], objects, model.labels, args.prob_threshold) presenter.drawGraphs(frame) frame = draw_detections(frame, objects, palette, model.labels, args.prob_threshold) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Detection Results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) next_frame_id_to_show += 1 else: break metrics.print_total() print(presenter.reportMeans())
default=None) args.add_argument('--fx', type=np.float32, default=-1, help='Optional. Camera focal length.') args.add_argument('--no_show', help='Optional. Do not display output.', action='store_true') args.add_argument("-u", "--utilization_monitors", default='', type=str, help="Optional. List of monitors to show initially.") args = parser.parse_args() cap = open_images_capture(args.input, args.loop) stride = 8 inference_engine = InferenceEngine(args.model, args.device, stride) canvas_3d = np.zeros((720, 1280, 3), dtype=np.uint8) plotter = Plotter3d(canvas_3d.shape[:2]) canvas_3d_window_name = 'Canvas 3D' if not args.no_show: cv2.namedWindow(canvas_3d_window_name) cv2.setMouseCallback(canvas_3d_window_name, Plotter3d.mouse_callback) file_path = args.extrinsics_path if file_path is None: file_path = Path(__file__).parent / 'data/extrinsics.json' with open(file_path, 'r') as f: extrinsics = json.load(f)
def main(): args = build_argparser().parse_args() log.info('Initializing Inference Engine...') ie = IECore() plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) cap = open_images_capture(args.input, args.loop) start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") log.info('Loading network...') model = Deblurring(ie, args.model, frame.shape) pipeline = AsyncPipeline(ie, model, plugin_config, device=args.device, max_num_requests=args.num_infer_requests) log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) pipeline.submit_data(frame, 0, {'frame': frame, 'start_time': start_time}) next_frame_id = 1 next_frame_id_to_show = 0 metrics = PerformanceMetrics() presenter = monitors.Presenter( args.utilization_monitors, 55, (round(frame.shape[1] / 4), round(frame.shape[0] / 8))) video_writer = cv2.VideoWriter() if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (2 * frame.shape[1], frame.shape[0])): raise RuntimeError("Can't open video writer") while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: break # Submit for inference pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: result_frame, frame_meta = results input_frame = frame_meta['frame'] start_time = frame_meta['start_time'] if input_frame.shape != result_frame.shape: input_frame = cv2.resize( input_frame, (result_frame.shape[1], result_frame.shape[0])) final_image = cv2.hconcat([input_frame, result_frame]) presenter.drawGraphs(final_image) metrics.update(start_time, final_image) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(final_image) if not args.no_show: cv2.imshow('Deblurring Results', final_image) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break presenter.handleKey(key) next_frame_id_to_show += 1 pipeline.await_all() # Process completed requests while pipeline.has_completed_request(): results = pipeline.get_result(next_frame_id_to_show) if results: result_frame, frame_meta = results input_frame = frame_meta['frame'] start_time = frame_meta['start_time'] if input_frame.shape != result_frame.shape: input_frame = cv2.resize( input_frame, (result_frame.shape[1], result_frame.shape[0])) final_image = cv2.hconcat([input_frame, result_frame]) presenter.drawGraphs(final_image) metrics.update(start_time, final_image) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(final_image) if not args.no_show: cv2.imshow('Deblurring Results', final_image) key = cv2.waitKey(1) next_frame_id_to_show += 1 else: break metrics.print_total() print(presenter.reportMeans())
def main(args): cap = open_images_capture(args.input, args.loop) log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() log.info('Reading model {}'.format(args.model)) model = core.read_model(args.model) input_tensor_name = 'data_l' input_shape = model.input(input_tensor_name).shape assert input_shape[1] == 1, "Expected model input shape with 1 channel" inputs = {} for input in model.inputs: inputs[input.get_any_name()] = np.zeros(input.shape) assert len(model.outputs) == 1, "Expected number of outputs is equal 1" compiled_model = core.compile_model(model, device_name=args.device) output_tensor = compiled_model.outputs[0] infer_request = compiled_model.create_infer_request() log.info('The model {} is loaded to {}'.format(args.model, args.device)) _, _, h_in, w_in = input_shape frames_processed = 0 imshow_size = (640, 480) graph_size = (imshow_size[0] // 2, imshow_size[1] // 4) presenter = monitors.Presenter(args.utilization_monitors, imshow_size[1] * 2 - graph_size[1], graph_size) metrics = PerformanceMetrics() video_writer = cv.VideoWriter() if args.output and not video_writer.open( args.output, cv.VideoWriter_fourcc(*'MJPG'), cap.fps(), (imshow_size[0] * 2, imshow_size[1] * 2)): raise RuntimeError("Can't open video writer") start_time = perf_counter() original_frame = cap.read() if original_frame is None: raise RuntimeError("Can't read an image from the input") while original_frame is not None: (h_orig, w_orig) = original_frame.shape[:2] if original_frame.shape[2] > 1: frame = cv.cvtColor(cv.cvtColor(original_frame, cv.COLOR_BGR2GRAY), cv.COLOR_GRAY2RGB) else: frame = cv.cvtColor(original_frame, cv.COLOR_GRAY2RGB) img_rgb = frame.astype(np.float32) / 255 img_lab = cv.cvtColor(img_rgb, cv.COLOR_RGB2Lab) img_l_rs = cv.resize(img_lab.copy(), (w_in, h_in))[:, :, 0] inputs[input_tensor_name] = np.expand_dims(img_l_rs, axis=[0, 1]) res = infer_request.infer(inputs)[output_tensor] update_res = np.squeeze(res) out = update_res.transpose((1, 2, 0)) out = cv.resize(out, (w_orig, h_orig)) img_lab_out = np.concatenate((img_lab[:, :, 0][:, :, np.newaxis], out), axis=2) img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1) original_image = cv.resize(original_frame, imshow_size) grayscale_image = cv.resize(frame, imshow_size) colorize_image = (cv.resize(img_bgr_out, imshow_size) * 255).astype( np.uint8) lab_image = cv.resize(img_lab_out, imshow_size).astype(np.uint8) original_image = cv.putText(original_image, 'Original', (25, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA) grayscale_image = cv.putText(grayscale_image, 'Grayscale', (25, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA) colorize_image = cv.putText(colorize_image, 'Colorize', (25, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA) lab_image = cv.putText(lab_image, 'LAB interpretation', (25, 50), cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA) ir_image = [ cv.hconcat([original_image, grayscale_image]), cv.hconcat([lab_image, colorize_image]) ] final_image = cv.vconcat(ir_image) metrics.update(start_time, final_image) frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(final_image) presenter.drawGraphs(final_image) if not args.no_show: cv.imshow('Colorization Demo', final_image) key = cv.waitKey(1) if key in {ord("q"), ord("Q"), 27}: break presenter.handleKey(key) start_time = perf_counter() original_frame = cap.read() metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) if args.adapter == 'openvino': plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) model_adapter = OpenvinoAdapter( create_core(), args.model, device=args.device, plugin_config=plugin_config, max_num_requests=args.num_infer_requests, model_parameters={'input_layouts': args.layout}) elif args.adapter == 'ovms': model_adapter = OVMSAdapter(args.model) model = SegmentationModel.create_model(args.architecture_type, model_adapter, {'path_to_labels': args.labels}) if args.architecture_type == 'segmentation': visualizer = SegmentationVisualizer(args.colors) if args.architecture_type == 'salient_object_detection': visualizer = SaliencyMapVisualizer() model.log_layers_info() pipeline = AsyncPipeline(model) next_frame_id = 0 next_frame_id_to_show = 0 metrics = PerformanceMetrics() render_metrics = PerformanceMetrics() presenter = None output_transform = None video_writer = cv2.VideoWriter() only_masks = args.only_masks while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results if args.raw_output_message: print_raw_results(objects, next_frame_id_to_show, model.labels) frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() frame = render_segmentation(frame, objects, visualizer, output_transform, only_masks) render_metrics.update(rendering_start_time) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Segmentation Results', frame) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break if key == 9: only_masks = not only_masks presenter.handleKey(key) pipeline.await_all() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = pipeline.get_result(next_frame_id_to_show) objects, frame_meta = results if args.raw_output_message: print_raw_results(objects, next_frame_id_to_show, model.labels) frame = frame_meta['frame'] start_time = frame_meta['start_time'] rendering_start_time = perf_counter() frame = render_segmentation(frame, objects, visualizer, output_transform, only_masks) render_metrics.update(rendering_start_time) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Segmentation Results', frame) key = cv2.waitKey(1) metrics.log_total() log_latency_per_stage(cap.reader_metrics.get_latency(), pipeline.preprocess_metrics.get_latency(), pipeline.inference_metrics.get_latency(), pipeline.postprocess_metrics.get_latency(), render_metrics.get_latency()) for rep in presenter.reportMeans(): log.info(rep)
def main(): args = build_argparser().parse_args() metrics = PerformanceMetrics() log.info('Initializing Inference Engine...') ie = IECore() plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) cap = open_images_capture(args.input, args.loop) start_time = perf_counter() frame = cap.read() if frame is None: raise RuntimeError("Can't read an image from the input") log.info('Loading network...') model = get_model(ie, args, frame.shape[1] / frame.shape[0]) hpe_pipeline = AsyncPipeline(ie, model, plugin_config, device=args.device, max_num_requests=args.num_infer_requests) log.info('Starting inference...') hpe_pipeline.submit_data(frame, 0, { 'frame': frame, 'start_time': start_time }) next_frame_id = 1 next_frame_id_to_show = 0 output_transform = models.OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) video_writer = cv2.VideoWriter() if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) while True: if hpe_pipeline.callback_exceptions: raise hpe_pipeline.callback_exceptions[0] # Process all completed requests results = hpe_pipeline.get_result(next_frame_id_to_show) if results: (poses, scores), frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(poses) and args.raw_output_message: print_raw_results(poses, scores) presenter.drawGraphs(frame) frame = draw_poses(frame, poses, args.prob_threshold, output_transform) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Pose estimation results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) continue if hpe_pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: break # Submit for inference hpe_pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request hpe_pipeline.await_any() hpe_pipeline.await_all() # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = hpe_pipeline.get_result(next_frame_id_to_show) while results is None: results = hpe_pipeline.get_result(next_frame_id_to_show) (poses, scores), frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] if len(poses) and args.raw_output_message: print_raw_results(poses, scores) presenter.drawGraphs(frame) frame = draw_poses(frame, poses, args.prob_threshold, output_transform) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Pose estimation results', frame) key = cv2.waitKey(1) ESC_KEY = 27 # Quit. if key in {ord('q'), ord('Q'), ESC_KEY}: break presenter.handleKey(key) metrics.print_total() print(presenter.reportMeans())
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) frame_processor = FrameProcessor(args) frame_num = 0 metrics = PerformanceMetrics() presenter = None output_transform = None input_crop = None if args.crop_size[0] > 0 and args.crop_size[1] > 0: input_crop = np.array(args.crop_size) elif not (args.crop_size[0] == 0 and args.crop_size[1] == 0): raise ValueError('Both crop height and width should be positive') video_writer = cv2.VideoWriter() while True: start_time = perf_counter() frame = cap.read() if frame is None: if frame_num == 0: raise ValueError("Can't read an image from the input") break if input_crop: frame = center_crop(frame, input_crop) if frame_num == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter(args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") detections = frame_processor.process(frame) presenter.drawGraphs(frame) frame = draw_detections(frame, frame_processor, detections, output_transform) metrics.update(start_time, frame) frame_num += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frame_num <= args.output_limit): video_writer.write(frame) if not args.no_show: cv2.imshow('Face recognition demo', frame) key = cv2.waitKey(1) # Quit if key in {ord('q'), ord('Q'), 27}: break presenter.handleKey(key) metrics.log_total() for rep in presenter.reportMeans(): log.info(rep)
def main(): metrics = PerformanceMetrics() args = build_argparser().parse_args() log.info('Initializing Inference Engine...') ie = IECore() plugin_config = get_user_config(args.device, args.num_streams, args.num_threads) log.info('Loading network...') model, visualizer = get_model(ie, args) pipeline = AsyncPipeline(ie, model, plugin_config, device=args.device, max_num_requests=args.num_infer_requests) cap = open_images_capture(args.input, args.loop) next_frame_id = 0 next_frame_id_to_show = 0 log.info('Starting inference...') print( "To close the application, press 'CTRL+C' here or switch to the output window and press ESC key" ) presenter = None output_transform = None video_writer = cv2.VideoWriter() only_masks = args.only_masks while True: if pipeline.is_ready(): # Get new image/frame start_time = perf_counter() frame = cap.read() if frame is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: output_transform = OutputTransform(frame.shape[:2], args.output_resolution) if args.output_resolution: output_resolution = output_transform.new_resolution else: output_resolution = (frame.shape[1], frame.shape[0]) presenter = monitors.Presenter( args.utilization_monitors, 55, (round(output_resolution[0] / 4), round(output_resolution[1] / 8))) if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), output_resolution): raise RuntimeError("Can't open video writer") # Submit for inference pipeline.submit_data(frame, next_frame_id, { 'frame': frame, 'start_time': start_time }) next_frame_id += 1 else: # Wait for empty request pipeline.await_any() if pipeline.callback_exceptions: raise pipeline.callback_exceptions[0] # Process all completed requests results = pipeline.get_result(next_frame_id_to_show) if results: objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] frame = render_segmentation(frame, objects, visualizer, output_transform, only_masks) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) next_frame_id_to_show += 1 if not args.no_show: cv2.imshow('Segmentation Results', frame) key = cv2.waitKey(1) if key == 27 or key == 'q' or key == 'Q': break if key == 9: only_masks = not only_masks presenter.handleKey(key) pipeline.await_all() # Process completed requests for next_frame_id_to_show in range(next_frame_id_to_show, next_frame_id): results = pipeline.get_result(next_frame_id_to_show) while results is None: results = pipeline.get_result(next_frame_id_to_show) objects, frame_meta = results frame = frame_meta['frame'] start_time = frame_meta['start_time'] frame = render_segmentation(frame, objects, visualizer, output_transform, only_masks) presenter.drawGraphs(frame) metrics.update(start_time, frame) if video_writer.isOpened() and ( args.output_limit <= 0 or next_frame_id_to_show <= args.output_limit - 1): video_writer.write(frame) if not args.no_show: cv2.imshow('Segmentation Results', frame) key = cv2.waitKey(1) metrics.print_total() print(presenter.reportMeans())
def main(): log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() img_retrieval = ImageRetrieval(args.model, args.device, args.gallery, INPUT_SIZE, args.cpu_extension) cap = open_images_capture(args.input, args.loop) if cap.get_type() not in ('VIDEO', 'CAMERA'): raise RuntimeError( "The input should be a video file or a numeric camera ID") frames = RoiDetectorOnVideo(cap) compute_embeddings_times = [] search_in_gallery_times = [] positions = [] frames_processed = 0 presenter = monitors.Presenter(args.utilization_monitors, 0) video_writer = cv2.VideoWriter() for image, view_frame in frames: position = None sorted_indexes = [] if image is not None: image = central_crop(image, divide_by=5, shift=1) elapsed, probe_embedding = time_elapsed( img_retrieval.compute_embedding, image) compute_embeddings_times.append(elapsed) elapsed, (sorted_indexes, distances) = time_elapsed( img_retrieval.search_in_gallery, probe_embedding) search_in_gallery_times.append(elapsed) sorted_classes = [ img_retrieval.gallery_classes[i] for i in sorted_indexes ] if args.ground_truth is not None: position = sorted_classes.index( img_retrieval.text_label_to_class_id[args.ground_truth]) positions.append(position) log.info("ROI detected, found: %d, position of target: %d", sorted_classes[0], position) else: log.info("ROI detected, found: %s", sorted_classes[0]) image, key = visualize( view_frame, position, [img_retrieval.impaths[i] for i in sorted_indexes], distances[sorted_indexes] if position is not None else None, img_retrieval.input_size, np.mean(compute_embeddings_times), np.mean(search_in_gallery_times), imshow_delay=3, presenter=presenter, no_show=args.no_show) if frames_processed == 0: if args.output and not video_writer.open( args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (image.shape[1], image.shape[0])): raise RuntimeError("Can't open video writer") frames_processed += 1 if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit): video_writer.write(image) if key == 27: break print(presenter.reportMeans()) if positions: compute_metrics(positions)
def main(): args = build_argparser().parse_args() cap = open_images_capture(args.input, args.loop) # Plugin initialization for specified device and load extensions library if specified log.info('OpenVINO Inference Engine') log.info('\tbuild: {}'.format(get_version())) core = Core() # Read IR log.info('Reading Proposal model {}'.format(args.model_pnet)) p_net = core.read_model(args.model_pnet) if len(p_net.inputs) != 1: raise RuntimeError("Pnet supports only single input topologies") if len(p_net.outputs) != 2: raise RuntimeError("Pnet supports two output topologies") log.info('Reading Refine model {}'.format(args.model_rnet)) r_net = core.read_model(args.model_rnet) if len(r_net.inputs) != 1: raise RuntimeError("Rnet supports only single input topologies") if len(r_net.outputs) != 2: raise RuntimeError("Rnet supports two output topologies") log.info('Reading Output model {}'.format(args.model_onet)) o_net = core.read_model(args.model_onet) if len(o_net.inputs) != 1: raise RuntimeError("Onet supports only single input topologies") if len(o_net.outputs) != 3: raise RuntimeError("Onet supports three output topologies") pnet_input_tensor_name = p_net.inputs[0].get_any_name() rnet_input_tensor_name = r_net.inputs[0].get_any_name() onet_input_tensor_name = o_net.inputs[0].get_any_name() for node in p_net.outputs: if node.shape[1] == 2: pnet_cls_name = node.get_any_name() elif node.shape[1] == 4: pnet_roi_name = node.get_any_name() else: raise RuntimeError("Unsupported output layer for Pnet") for node in r_net.outputs: if node.shape[1] == 2: rnet_cls_name = node.get_any_name() elif node.shape[1] == 4: rnet_roi_name = node.get_any_name() else: raise RuntimeError("Unsupported output layer for Rnet") for node in o_net.outputs: if node.shape[1] == 2: onet_cls_name = node.get_any_name() elif node.shape[1] == 4: onet_roi_name = node.get_any_name() elif node.shape[1] == 10: onet_pts_name = node.get_any_name() else: raise RuntimeError("Unsupported output layer for Onet") next_frame_id = 0 metrics = PerformanceMetrics() presenter = None video_writer = cv2.VideoWriter() is_loaded_before = False while True: start_time = perf_counter() origin_image = cap.read() if origin_image is None: if next_frame_id == 0: raise ValueError("Can't read an image from the input") break if next_frame_id == 0: presenter = monitors.Presenter(args.utilization_monitors, 55, (round(origin_image.shape[1] / 4), round(origin_image.shape[0] / 8))) if args.output and not video_writer.open(args.output, cv2.VideoWriter_fourcc(*'MJPG'), cap.fps(), (origin_image.shape[1], origin_image.shape[0])): raise RuntimeError("Can't open video writer") next_frame_id += 1 rgb_image = cv2.cvtColor(origin_image, cv2.COLOR_BGR2RGB) oh, ow, _ = rgb_image.shape scales = utils.calculate_scales(rgb_image) # ************************************* # Pnet stage # ************************************* pnet_res = [] for i, scale in enumerate(scales): hs = int(oh*scale) ws = int(ow*scale) image = preprocess_image(rgb_image, ws, hs) p_net.reshape({pnet_input_tensor_name: PartialShape([1, 3, ws, hs])}) # Change weidth and height of input blob compiled_pnet = core.compile_model(p_net, args.device) infer_request_pnet = compiled_pnet.create_infer_request() if i == 0 and not is_loaded_before: log.info("The Proposal model {} is loaded to {}".format(args.model_pnet, args.device)) infer_request_pnet.infer(inputs={pnet_input_tensor_name: image}) p_res = {name: infer_request_pnet.get_tensor(name).data[:] for name in {pnet_roi_name, pnet_cls_name}} pnet_res.append(p_res) image_num = len(scales) rectangles = [] for i in range(image_num): roi = pnet_res[i][pnet_roi_name] cls = pnet_res[i][pnet_cls_name] _, _, out_h, out_w = cls.shape out_side = max(out_h, out_w) rectangle = utils.detect_face_12net(cls[0][1], roi[0], out_side, 1/scales[i], ow, oh, score_threshold[0], iou_threshold[0]) rectangles.extend(rectangle) rectangles = utils.NMS(rectangles, iou_threshold[1], 'iou') # Rnet stage if len(rectangles) > 0: r_net.reshape({rnet_input_tensor_name: PartialShape([len(rectangles), 3, 24, 24])}) # Change batch size of input blob compiled_rnet = core.compile_model(r_net, args.device) infer_request_rnet = compiled_rnet.create_infer_request() if not is_loaded_before: log.info("The Refine model {} is loaded to {}".format(args.model_rnet, args.device)) rnet_input = [] for rectangle in rectangles: crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] crop_img = preprocess_image(crop_img, 24, 24) rnet_input.extend(crop_img) infer_request_rnet.infer(inputs={rnet_input_tensor_name: rnet_input}) rnet_res = {name: infer_request_rnet.get_tensor(name).data[:] for name in {rnet_roi_name, rnet_cls_name}} roi = rnet_res[rnet_roi_name] cls = rnet_res[rnet_cls_name] rectangles = utils.filter_face_24net(cls, roi, rectangles, ow, oh, score_threshold[1], iou_threshold[2]) # Onet stage if len(rectangles) > 0: o_net.reshape({onet_input_tensor_name: PartialShape([len(rectangles), 3, 48, 48])}) # Change batch size of input blob compiled_onet = core.compile_model(o_net, args.device) infer_request_onet = compiled_onet.create_infer_request() if not is_loaded_before: log.info("The Output model {} is loaded to {}".format(args.model_onet, args.device)) is_loaded_before = True onet_input = [] for rectangle in rectangles: crop_img = rgb_image[int(rectangle[1]):int(rectangle[3]), int(rectangle[0]):int(rectangle[2])] crop_img = preprocess_image(crop_img, 48, 48) onet_input.extend(crop_img) infer_request_onet.infer(inputs={onet_input_tensor_name: onet_input}) onet_res = {name: infer_request_onet.get_tensor(name).data[:] for name in {onet_roi_name, onet_cls_name, onet_pts_name}} roi = onet_res[onet_roi_name] cls = onet_res[onet_cls_name] pts = onet_res[onet_pts_name] rectangles = utils.filter_face_48net(cls, roi, pts, rectangles, ow, oh, score_threshold[2], iou_threshold[3]) # display results for rectangle in rectangles: # Draw detected boxes cv2.putText(origin_image, 'confidence: {:.2f}'.format(rectangle[4]), (int(rectangle[0]), int(rectangle[1])), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 0)) cv2.rectangle(origin_image, (int(rectangle[0]), int(rectangle[1])), (int(rectangle[2]), int(rectangle[3])), (255, 0, 0), 1) # Draw landmarks for i in range(5, 15, 2): cv2.circle(origin_image, (int(rectangle[i+0]), int(rectangle[i+1])), 2, (0, 255, 0)) metrics.update(start_time, origin_image) if video_writer.isOpened() and (args.output_limit <= 0 or next_frame_id <= args.output_limit): video_writer.write(origin_image) if not args.no_show: cv2.imshow('MTCNN Results', origin_image) key = cv2.waitKey(1) if key in {ord('q'), ord('Q'), 27}: break presenter.handleKey(key) metrics.log_total()