def draw_instance_predictions(self, frame, predictions): """ Draw instance-level prediction results on an image. Args: frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. predictions (Instances): the output of an instance detection/segmentation model. Following fields will be used to draw: "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). Returns: output (VisImage): image object with visualizations. """ frame_visualizer = Visualizer(frame, self.metadata) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has("pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None classes = predictions.pred_classes.numpy() if predictions.has("pred_classes") else None keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None if predictions.has("pred_masks"): masks = predictions.pred_masks # mask IOU is not yet enabled # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) # assert len(masks_rles) == num_instances else: masks = None detected = [ _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( (masks.any(dim=0) > 0).numpy() if masks is not None else None ) alpha = 0.3 else: alpha = 0.5 frame_visualizer.overlay_instances( # boxes=None if masks is not None else boxes, # boxes are a bit distracting boxes=boxes, masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output
def draw_instance_predictions(self, frame, predictions, effect_type): frame_visualizer = Visualizer(frame, self.metadata) cnt = predictions["current_frame"] num_instances = predictions["num_instances"] if num_instances == 0: return frame_visualizer.output boxes = predictions["boxes"] scores = predictions["scores"] classes = predictions["classes"] keypoints = predictions["keypoints"] masks = predictions["masks"] detected = [ _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( (masks.any(dim=0) > 0).numpy() if masks is not None else None) alpha = 0.3 else: alpha = 0.5 if effect_type == 0: frame_visualizer.overlay_instances_scanning( boxes=None if masks is not None else boxes, # boxes are a bit distracting cnt=cnt, masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) else: frame_visualizer.overlay_instances_stop_motion( boxes=None if masks is not None else boxes, # boxes are a bit distracting cnt=cnt, masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output
def draw_panoptic_seg_predictions( self, frame, panoptic_seg, segments_info, area_threshold=None, alpha=0.5 ): frame_visualizer = Visualizer(frame, self.metadata) pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) if self._instance_mode == ColorMode.IMAGE_BW: frame_visualizer.output.img = frame_visualizer._create_grayscale_image( pred.non_empty_mask() ) # draw mask for all semantic segments first i.e. "stuff" for mask, sinfo in pred.semantic_masks(): category_idx = sinfo["category_id"] try: mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] except AttributeError: mask_color = None frame_visualizer.draw_binary_mask( mask, color=mask_color, text=self.metadata.stuff_classes[category_idx], alpha=alpha, area_threshold=area_threshold, ) all_instances = list(pred.instance_masks()) if len(all_instances) == 0: return frame_visualizer.output # draw mask for all instances second masks, sinfo = list(zip(*all_instances)) num_instances = len(masks) masks_rles = mask_util.encode( np.asarray(np.asarray(masks).transpose(1, 2, 0), dtype=np.uint8, order="F") ) assert len(masks_rles) == num_instances category_ids = [x["category_id"] for x in sinfo] detected = [ _DetectedInstance(category_ids[i], bbox=None, mask_rle=masks_rles[i], color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = [self.metadata.thing_classes[k] for k in category_ids] frame_visualizer.overlay_instances( boxes=None, masks=masks, labels=labels, keypoints=None, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output
def draw_instance_predictions(self, frame, predictions): """ Draw instance-level prediction results on an image. Args: frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. predictions (Instances): the output of an instance detection model. Following fields will be used to draw: "pred_boxes", "pred_classes", "scores". Returns: output (VisImage): image object with visualizations. """ frame_visualizer = Visualizer(frame, self.metadata) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has( "pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None classes = predictions.pred_classes.numpy() if predictions.has( "pred_classes") else None detected = [ _DetectedInstance(classes[i], boxes[i], color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( ) alpha = 0.3 else: alpha = 0.5 frame_visualizer.overlay_instances( boxes=boxes, # boxes are a bit distracting labels=labels, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output
def draw_instance_predictions(self, frame, predictions): """ Draw instance-level prediction results on an image. Args: frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. predictions (Instances): the output of an instance detection/segmentation model. Following fields will be used to draw: "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). Returns: output (VisImage): image object with visualizations. """ frame_visualizer = Visualizer(frame, self.metadata) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has( "pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None classes = predictions.pred_classes.numpy() if predictions.has( "pred_classes") else None keypoints = predictions.pred_keypoints if predictions.has( "pred_keypoints") else None colors = predictions.COLOR if predictions.has( "COLOR") else [None] * len(predictions) durations = predictions.ID_duration if predictions.has( "ID_duration") else None duration_threshold = self.metadata.get("duration_threshold", 0) visibilities = None if durations is None else [ x > duration_threshold for x in durations ] if predictions.has("pred_masks"): masks = predictions.pred_masks # mask IOU is not yet enabled # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) # assert len(masks_rles) == num_instances else: masks = None detected = [ _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=colors[i], ttl=8) for i in range(num_instances) ] if not predictions.has("COLOR"): colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.reset_image( frame_visualizer._create_grayscale_image((masks.any( dim=0) > 0).numpy() if masks is not None else None)) alpha = 0.3 else: alpha = 0.5 labels = ( None if labels is None else [y[0] for y in filter(lambda x: x[1], zip(labels, visibilities))] ) # noqa assigned_colors = ( None if colors is None else [y[0] for y in filter(lambda x: x[1], zip(colors, visibilities))] ) # noqa frame_visualizer.overlay_instances( boxes=None if masks is not None else boxes[visibilities], # boxes are a bit distracting masks=None if masks is None else masks[visibilities], labels=labels, keypoints=None if keypoints is None else keypoints[visibilities], assigned_colors=assigned_colors, alpha=alpha, ) return frame_visualizer.output
def draw_instance_predictions_custom(self, frame, predictions, incl_boxes=True, incl_labels=True, incl_scores=True, target_alpha=None): frame_visualizer = Visualizer(frame, self.metadata) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has( "pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None scores = scores if incl_scores else None classes = predictions.pred_classes.numpy() if predictions.has( "pred_classes") else None keypoints = predictions.pred_keypoints if predictions.has( "pred_keypoints") else None if predictions.has("pred_masks"): masks = predictions.pred_masks # mask IOU is not yet enabled # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) # assert len(masks_rles) == num_instances else: masks = None detected = [ _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( (masks.any(dim=0) > 0).numpy() if masks is not None else None) alpha = 0.3 else: alpha = 0.5 boxes = boxes if incl_boxes else None labels = labels if incl_labels else None alpha = alpha if target_alpha is None else target_alpha frame_visualizer.overlay_instances( boxes=None if masks is not None else boxes, # boxes are a bit distracting masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output
def draw_instance_predictions(visualizer, frame, predictions, tracker): """ Draw instance-level prediction results on an image. Args: frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. predictions (Instances): the output of an instance detection/segmentation model. Following fields will be used to draw: "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). Returns: output (VisImage): image object with visualizations. """ frame_visualizer = Visualizer(frame, visualizer.metadata) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has( "pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None classes = predictions.pred_classes.numpy() if predictions.has( "pred_classes") else None keypoints = predictions.pred_keypoints if predictions.has( "pred_keypoints") else None if predictions.has("pred_masks"): masks = predictions.pred_masks # mask IOU is not yet enabled # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) # assert len(masks_rles) == num_instances else: masks = None detected = [ detectron2.utils.video_visualizer._DetectedInstance(classes[i], boxes[i], mask_rle=None, color=None, ttl=8) for i in range(num_instances) ] colors = visualizer._assign_colors(detected) labels = detectron2.utils.video_visualizer._create_text_labels( classes, scores, visualizer.metadata.get("thing_classes", None)) if visualizer._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( (masks.any(dim=0) > 0).numpy() if masks is not None else None) alpha = 0.3 else: alpha = 0.5 frame_visualizer.overlay_instances( boxes=None if masks is not None else boxes, # boxes are a bit distracting masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) for bag_id in tracker.prev_frame_ids['bags']: bag_center = tracker.all_centers['bags'][bag_id] if bag_id in tracker.bag_person_association: person_id = tracker.bag_person_association[bag_id] if person_id is not None and person_id in tracker.prev_frame_ids[ 'persons']: person_center = tracker.all_centers['persons'][person_id] if tracker.is_unattended(bag_id): frame_visualizer.draw_line( [bag_center[0], person_center[0]], [bag_center[1], person_center[1]], 'r') else: frame_visualizer.draw_line( [bag_center[0], person_center[0]], [bag_center[1], person_center[1]], 'g') if tracker.is_unattended(bag_id): frame_visualizer.draw_text('abandoned', tuple(bag_center[0:2]), color='r') return frame_visualizer.output
def draw_instance_predictions(self, frame, predictions): """ Draw instance-level prediction results on an image. Args: frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. predictions (Instances): the output of an instance detection/segmentation model. Following fields will be used to draw: "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). Returns: output (VisImage): image object with visualizations. """ frame_visualizer = Visualizer(frame, self.metadata) # MOONLITE: zero out frame frame_visualizer.output.img = np.zeros( frame_visualizer.output.img.shape) num_instances = len(predictions) if num_instances == 0: return frame_visualizer.output boxes = predictions.pred_boxes.tensor.numpy() if predictions.has( "pred_boxes") else None scores = predictions.scores if predictions.has("scores") else None classes = predictions.pred_classes.numpy() if predictions.has( "pred_classes") else None keypoints = predictions.pred_keypoints if predictions.has( "pred_keypoints") else None if predictions.has("pred_masks"): masks = predictions.pred_masks # mask IOU is not yet enabled # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) # assert len(masks_rles) == num_instances else: masks = None detected = [ _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=None, ttl=8) for i in range(num_instances) ] colors = self._assign_colors(detected) labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) if self._instance_mode == ColorMode.IMAGE_BW: # any() returns uint8 tensor frame_visualizer.output.img = frame_visualizer._create_grayscale_image( (masks.any(dim=0) > 0).numpy() if masks is not None else None) alpha = 0.3 else: alpha = 0.5 # only keep instance if class_names = self.metadata.get("thing_classes", None) # get indices of all focused_class = "person" num_class_instances = [ i for i in range(num_instances) if (class_names[classes[i]] == focused_class) ] # strip instances down to only instances of our focused class boxes = [boxes[i] for i in num_class_instances] masks = [masks[i] for i in num_class_instances] labels = ["" for i in num_class_instances] keypoints = [keypoints[i] for i in num_class_instances] colors = [(scores[i], scores[i], scores[i]) for i in num_class_instances] #for i in range(num_instances): # if class_names[classes[i]] == "person": # colors[i] = (scores[i],scores[i],scores[i]) # else: # colors[i] = (0.0,0.0,0.0) # # labels[i] = "" alpha = 1.0 frame_visualizer.overlay_instances( boxes=None if masks is not None else boxes, # boxes are a bit distracting masks=masks, labels=labels, keypoints=keypoints, assigned_colors=colors, alpha=alpha, ) return frame_visualizer.output