def compute_iou(self, img_id, cat_id): gt, dt = self._get_gt_dt(img_id, cat_id) if len(gt) == 0 and len(dt) == 0: return [] # Sort detections in decreasing order of score. idx = np.argsort([-d["score"] for d in dt], kind="mergesort") dt = [dt[i] for i in idx] # iscrowd = [int(False)] * len(gt) iscrowd = [int('iscrowd' in g and g['iscrowd'] > 0) for g in gt] if self.params.iou_type == "segm": ann_type = "segmentation" elif self.params.iou_type == "bbox": ann_type = "bbox" else: raise ValueError("Unknown iou_type for iou computation.") gt = [g[ann_type] for g in gt] dt = [d[ann_type] for d in dt] # compute iou between each dt and gt region # will return array of shape len(dt), len(gt) ious = mask_utils.iou(dt, gt, iscrowd) return ious
def computeIoU(gt, dt, iouType="bbox", maxDets=100): if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > maxDets: dt = dt[0:maxDets] if iouType == 'segm': g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] elif iouType == 'bbox': g = [g['bbox'] for g in gt] d = [d['bbox'] for d in dt] else: raise Exception('unknown iouType for iou computation') # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] ious = maskUtils.iou(d, g, iscrowd) # #### # print("id:%s - cat:%d" % (imgId, catId)) # print("d:", d) # print("g:", g) # print("ious", ious) # #### return ious
def _compute_j(gt_data, tracker_data, num_gt_ids, num_tracker_ids, num_timesteps): """ Computation of J value for all ground truth IDs and all tracker IDs in the given sequence. Adapted from https://github.com/davisvideochallenge/davis2017-evaluation :param gt_data: the ground truth masks :param tracker_data: the tracker masks :param num_gt_ids: the number of ground truth IDs :param num_tracker_ids: the number of tracker IDs :param num_timesteps: the number of timesteps :return: the J values """ # Only loaded when run to reduce minimum requirements from pycocotools import mask as mask_utils j = np.zeros((num_tracker_ids, num_gt_ids, num_timesteps)) for t, (time_gt, time_data) in enumerate(zip(gt_data, tracker_data)): # run length encoded masks with pycocotools area_gt = mask_utils.area(time_gt) area_tr = mask_utils.area(time_data) area_tr = np.repeat(area_tr[:, np.newaxis], len(area_gt), axis=1) area_gt = np.repeat(area_gt[np.newaxis, :], len(area_tr), axis=0) # mask iou computation with pycocotools ious = mask_utils.iou(time_data, time_gt, np.zeros([len(time_data)])) # set iou to 1 if both masks are close to 0 (no ground truth and no predicted mask in timestep) ious[np.isclose(area_tr, 0) & np.isclose(area_gt, 0)] = 1 assert (ious >= 0).all() assert (ious <= 1).all() j[..., t] = ious return j
def get_negative_ff_props(proposals, templates): if use_ff_negative_props: curr_propsoals = copy(proposals) curr_set = copy(templates) negatives = [] curr_propsoals.sort(key=lambda x: x["score"], reverse=True) for prop in curr_propsoals: is_ol = False for s in curr_set: ol = iou([ s['segmentation'], ], [ prop['segmentation'], ], np.array([0], np.uint8)) if ol > 0: is_ol = True break if is_ol: continue curr_set += copy([ prop, ]) negatives += copy([ prop, ]) else: negatives = [] return negatives
def computeIoU(self, imgId, catId): p = self.params if p.useCats: gt = self._gts[imgId, catId] dt = self._dts[imgId, catId] else: gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: dt = dt[0:p.maxDets[-1]] if p.iouType == 'bbox': g_ = [[g['bbox'].xmin, g['bbox'].ymin, g['width'], g['height']] for g in gt] d_ = [d['bbox'] for d in dt] else: raise Exception('unknown iouType for iou computation') # compute iou between each dt and gt region # iscrowd = [int(o['iscrowd']) for o in gt] iscrowd = [o['details'].occluded for o in gt] ious = maskUtils.iou(d_, g_, iscrowd) return ious
def bo_proposal(proposals, image_id, pointList): if pointList is None or len(pointList) == 0: return [] props_segs = [prop["segmentation"] for prop in proposals] point_segs = [p["segmentation"] for p in au.pointList2annList(pointList)] ious = maskUtils.iou(point_segs, props_segs, np.zeros(len(props_segs))) if 1: annList = [] for i, point in enumerate(pointList): propList = np.array(proposals)[ious[i] != 0] scoreList = np.array([pr["score"] for pr in propList]) if len(propList) == 0: continue prop = propList[scoreList.argmax(0)] mask = au.ann2mask(prop)["mask"] ann = au.mask2ann(mask, category_id=point["cls"], image_id=image_id, maskVoid=None, score=prop["score"], point=None) annList += [ann] return annList
def _checkIgnore(dt, iregion): if iregion is None: return True bb = np.array(dt['bbox']).astype(np.int) x1,y1,x2,y2 = bb[0],bb[1],bb[0]+bb[2],bb[1]+bb[3] x2 = min([x2,iregion.shape[1]]) y2 = min([y2,iregion.shape[0]]) if bb[2]* bb[3] == 0: return False crop_iregion = iregion[y1:y2, x1:x2] if crop_iregion.sum() == 0: return True if not 'uv' in dt.keys(): # filtering boxes return crop_iregion.sum()/bb[2]/bb[3] < self.ignoreThrBB # filtering UVs ignoremask = np.require(crop_iregion, requirements=['F']) uvmask = np.require(np.asarray(dt['uv'][0]>0), dtype = np.uint8, requirements=['F']) uvmask_ = maskUtils.encode(uvmask) ignoremask_ = maskUtils.encode(ignoremask) uviou = maskUtils.iou([uvmask_], [ignoremask_], [1])[0] return uviou < self.ignoreThrUV
def remove_overlap_props(props, props_ids, threshold=0.2): final_props = [] isOverlap = False if len(props_ids) > 0: for i in props_ids: if len(final_props) == 0: final_props.append(i) continue ratios = [ iou([props['seg'][i]], [props['seg'][merged]], arr([0], np.uint8))[:, 0][0] for merged in final_props ] if ratios: if np.max(ratios) >= threshold: isOverlap = True if props['score'][i] > props['score'][final_props[ np.argmax(ratios)]]: final_props.pop(np.argmax(ratios)) final_props.append(i) else: final_props.append(i) if isOverlap: final_props = remove_overlap_props(props, final_props, threshold) return final_props
def keypoint_in_body_mask(frame_number, keypoint_name, animal_name=None): if animal_name is None: animal_name = subject_animal_name _df_k_b = df[df.frame_number == frame_number] try: body_seg = _df_k_b[_df_k_b.instance_name == animal_name]['segmentation'].values[0] body_seg = ast.literal_eval(body_seg) except IndexError: return False try: keypoint_seg = _df_k_b[_df_k_b.instance_name == keypoint_name]['segmentation'].values[0] keypoint_seg = ast.literal_eval(keypoint_seg) except IndexError: return False if keypoint_seg and body_seg: overlap = mask_util.iou([body_seg], [keypoint_seg], [False, False]).flatten()[0] return overlap > 0 else: return False
def computeIoU(self, imgId, catId): p = self.params if p.useCats: gt = self._gts[imgId, catId] dt = self._dts[imgId, catId] else: gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: dt = dt[0:p.maxDets[-1]] if p.iouType == 'segm': g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] elif p.iouType == 'bbox': g = [g['bbox'] for g in gt] d = [d['bbox'] for d in dt] else: raise Exception('unknown iouType for iou computation') # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] ious = maskUtils.iou(d, g, iscrowd) return ious
def compute_iou(self, img_id, cat_id): gt, dt = self._get_gt_dt(img_id, cat_id) if len(gt) == 0 and len(dt) == 0: return [] # Sort detections in decreasing order of score. idx = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in idx] iscrowd = [int(False)] * len(gt) if self.params.iou_type == 'segm': ann_type = 'segmentation' elif self.params.iou_type == 'bbox': ann_type = 'bbox' else: raise ValueError('Unknown iou_type for iou computation.') gt = [g[ann_type] for g in gt] dt = [d[ann_type] for d in dt] # compute iou between each dt and gt region # will return array of shape len(dt), len(gt) ious = mask_utils.iou(dt, gt, iscrowd) return ious
def annotate(self, cocoDt): for imgId in cocoDt.imgs: for catId in cocoDt.cats: annIdsDt = cocoDt.getAnnIds(imgIds=[imgId], catIds=[catId]) annsDt = cocoDt.loadAnns(annIdsDt) # cocoGt ids do not match cocoDt ids cocoGt = self.cocoGt imgIdGt = self.filenameToImg[cocoDt.imgs[imgId] ["file_name"]]["id"] catIdGt = self.nameToCat[cocoDt.cats[catId]["name"]]["id"] annIdsGt = cocoGt.getAnnIds(imgIds=[imgIdGt], catIds=[catIdGt]) annsGt = cocoGt.loadAnns(annIdsGt) gts = [ann["segmentation"] for ann in annsGt] dts = [ann["segmentation"] for ann in annsDt] iscrowds = [0 for _ in gts] if len(gts) == 0 or len(dts) == 0: for annDt in annsDt: annDt["completed_task_sim"] = {} annDt["completed_task_sim"]["type"] = "yesno" annDt["completed_task_sim"]["accepted"] = False annDt["completed_task_sim"]["iou"] = 0 continue ious = COCOmask.iou(dts, gts, iscrowds) for iousDt, annDt in zip(ious, annsDt): iou = np.max(iousDt) # annGt = annsGt[np.argmax(iousDt)] annDt["completed_task_sim"] = {} annDt["completed_task_sim"]["type"] = "yesno" annDt["completed_task_sim"]["accepted"] = float( iou) >= self.thresholdIOU annDt["completed_task_sim"]["iou"] = iou
def computeIoU(self, image_id, attr_id): """ If there are <n_g> GT annotations and <n_d> detections, this produces a IoU matrix of size <n_d x n_g> :param image_id: :param attr_id: :return: """ p = self.params gt = self._gts[image_id, attr_id] # List of annotations for this image-category dt = self._pds[image_id, attr_id] # List of predictions for this image-category if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: dt = dt[0:p.maxDets[-1]] g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] ious = mask_utils.iou(d, g, iscrowd) return ious
def calculate_old_merge_scores(proposals, templates, next_props, reid_scores, other_reid_scores): mask_scores = np.repeat(np.array([prop['score'] for prop in proposals])[np.newaxis, :], len(templates), axis=0) segs = [prop["segmentation"] for prop in proposals] next_segs = [prop["segmentation"] for prop in next_props] warp_scores = arr([ iou(segs, [ next_seg, ], arr([0], np.uint8))[:, 0] for next_seg in next_segs ]) other_warp_scores = np.ones_like(warp_scores) if len(templates) > 1: ids = np.arange(len(templates)) for id in ids: new_vec = 1 - np.max(np.atleast_2d(warp_scores[ids != id, :]), axis=0) other_warp_scores[id, :] = new_vec all_scores = arr([ mask_scores, reid_scores, other_reid_scores, warp_scores, other_warp_scores ]) return all_scores
def _checkIgnore(dt, iregion): if iregion is None: return True bb = np.array(dt["bbox"]).astype(np.int) x1, y1, x2, y2 = bb[0], bb[1], bb[0] + bb[2], bb[1] + bb[3] x2 = min([x2, iregion.shape[1]]) y2 = min([y2, iregion.shape[0]]) if bb[2] * bb[3] == 0: return False crop_iregion = iregion[y1:y2, x1:x2] if crop_iregion.sum() == 0: return True if "densepose" not in dt.keys(): # filtering boxes return crop_iregion.sum() / bb[2] / bb[3] < self.ignoreThrBB # filtering UVs ignoremask = np.require(crop_iregion, requirements=["F"]) mask = self._extract_mask(dt) uvmask = np.require(np.asarray(mask > 0), dtype=np.uint8, requirements=["F"]) uvmask_ = maskUtils.encode(uvmask) ignoremask_ = maskUtils.encode(ignoremask) uviou = maskUtils.iou([uvmask_], [ignoremask_], [1])[0] return uviou < self.ignoreThrUV
def rle_mask_voting(top_masks, all_masks, all_dets, iou_thresh, binarize_thresh, method='AVG'): """Returns new masks (in correspondence with `top_masks`) by combining multiple overlapping masks coming from the pool of `all_masks`. Two methods for combining masks are supported: 'AVG' uses a weighted average of overlapping mask pixels; 'UNION' takes the union of all mask pixels. """ if len(top_masks) == 0: return all_not_crowd = [False] * len(all_masks) top_to_all_overlaps = mask_util.iou(top_masks, all_masks, all_not_crowd) decoded_all_masks = [ np.array(mask_util.decode(rle), dtype=np.float32) for rle in all_masks ] decoded_top_masks = [ np.array(mask_util.decode(rle), dtype=np.float32) for rle in top_masks ] all_boxes = all_dets[:, :4].astype(np.int32) all_scores = all_dets[:, 4] # Fill box support with weights mask_shape = decoded_all_masks[0].shape mask_weights = np.zeros((len(all_masks), mask_shape[0], mask_shape[1])) for k in range(len(all_masks)): ref_box = all_boxes[k] x_0 = max(ref_box[0], 0) x_1 = min(ref_box[2] + 1, mask_shape[1]) y_0 = max(ref_box[1], 0) y_1 = min(ref_box[3] + 1, mask_shape[0]) mask_weights[k, y_0:y_1, x_0:x_1] = all_scores[k] mask_weights = np.maximum(mask_weights, 1e-5) top_segms_out = [] for k in range(len(top_masks)): # Corner case of empty mask if decoded_top_masks[k].sum() == 0: top_segms_out.append(top_masks[k]) continue inds_to_vote = np.where(top_to_all_overlaps[k] >= iou_thresh)[0] # Only matches itself if len(inds_to_vote) == 1: top_segms_out.append(top_masks[k]) continue masks_to_vote = [decoded_all_masks[i] for i in inds_to_vote] if method == 'AVG': ws = mask_weights[inds_to_vote] soft_mask = np.average(masks_to_vote, axis=0, weights=ws) mask = np.array(soft_mask > binarize_thresh, dtype=np.uint8) elif method == 'UNION': # Any pixel that's on joins the mask soft_mask = np.sum(masks_to_vote, axis=0) mask = np.array(soft_mask > 1e-5, dtype=np.uint8) else: raise NotImplementedError('Method {} is unknown'.format(method)) rle = mask_util.encode(np.array(mask[:, :, np.newaxis], order='F'))[0] top_segms_out.append(rle) return top_segms_out
def _walk_dts_wdgt_logic(self, seq_inx): print("INFO: mark widget boundary") thr = self.thr dt_id = self.dtIds[seq_inx] confidence = self.sorted_dts_scores[seq_inx] matched_gt_id = self.dtm[thr][seq_inx] is_ignored = self.dtIgs[thr][seq_inx] curr_prec = self.precision[thr][seq_inx] curr_rcll = self.recall[thr][seq_inx] if self.recall is not None else 0 max_rcll = self.recall[thr][-1] if self.recall is not None else 0 print("walking at {}th dt out of {} dts".format( seq_inx, len(self.dtIds))) print("in total {} valid gts, {} recalled".format( self.num_valid_gts, int(max_rcll * self.num_valid_gts))) print("") print(("dt id: {}, gt id: {}, dt forgiven: {}\n" "prec so far: {:.3f}, rcll so far: {:.3f}, max rcll: {:.3f}\n" "confidence: {:.3f}").format(dt_id, matched_gt_id, is_ignored, curr_prec, curr_rcll, max_rcll, confidence)) dt = self.coco_eval.cocoDt.anns[dt_id] gt = self.coco_eval.cocoGt.anns.get(matched_gt_id, None) if gt is not None: assert dt['image_id'] == gt['image_id'] iou = maskUtils.iou([dt['bbox']], [gt['bbox']], [False]) print("box iou: {:.3f}".format(iou[0][0])) else: print("CURRENT DT UNMATCHED!") image_id = dt['image_id'] print("img id: {}".format(image_id)) im = self.read_img(self.coco_eval.cocoDt.imgs[image_id]['file_name']) plot_image_with_anns(im, [dt], [gt]) return image_id
def computePredsLabelsForImage(self, imgId): p = self.params gts = np.array(self._gts_img[imgId]) dts = np.array(self._dts_img[imgId]) matched_gts = np.zeros(len(gts), dtype=np.int) matched_dts = np.zeros(len(dts), dtype=np.int) preds = [] labels = [] for i, gt in enumerate(gts): g = gt['segmentation'] for j, dt in enumerate(dts): d = dt['segmentation'] iou = maskUtils.iou([d], [g], [int(gt['iscrowd'])])[0][0] if iou > .5: matched_gts[i] = 1 matched_dts[j] = 1 preds.append(dt['category_id']) labels.append(gt['category_id']) for gt in gts[matched_gts == 0]: preds.append(max(p.catIds) + 1) labels.append(gt['category_id']) for dt in dts[matched_dts == 0]: preds.append(dt['category_id']) labels.append(max(p.catIds) + 1) return labels, preds
def matrix_nms(dets, kernel='gaussian', sigma=0.5): N = len(dets) if N == 0: return dets, np.where(dets[:, 0] == 0)[0] dets = np.array(dets) rles = dets[:, 0] scores = dets[:, 1] order = scores.argsort()[::-1] # sort in decending order sorted_scores = scores[order] sorted_rles = rles[order] ious = mutils.iou(rles.tolist(), rles.tolist(), [False]) ious = np.triu(ious, k=1) ious_cmax = ious.max(0) ious_cmax = np.tile(ious_cmax, reps=(N, 1)).T if kernel == 'gaussian': decay = np.exp(-(ious ** 2 - ious_cmax ** 2) / sigma) else: # linear decay = (1 - ious) / (1 - ious_cmax) # decay factor: N decay = decay.min(axis=0) sorted_scores *= decay dets = np.concatenate([sorted_rles.reshape(-1, 1), sorted_scores.reshape(-1, 1)], axis=-1) valid_ind = np.where(sorted_scores >= 0.05)[0] return dets[valid_ind], order[valid_ind]
def rle_to_polygon(rle): mask = COCOmask.decode(rle) mask = mask[:, :, np.newaxis] mask = np.array(mask, np.uint8) mask_new, contours, hierarchy = cv2.findContours(mask.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours = [c for c, h in zip(contours, hierarchy[0]) if h[3] < 0] # Only outermost polygon segmentation = [] for contour in contours: polygon = contour.flatten().tolist() if len(polygon) >= 6: segmentation.append(polygon) if len(segmentation) == 0: return None poly = COCOmask.frPyObjects(segmentation, mask.shape[0], mask.shape[1])[0] iou = COCOmask.iou([rle], [poly], [0])[0, 0] # BUGS ARE PRESENT!! # if iou < 0.9: # poly = COCOmask.decode(poly) # rle = COCOmask.decode(rle) # fname = "/tmp/blah/" + str(uuid.uuid4()) + ".png" # cv2.imwrite(fname, np.array(poly != rle, dtype=np.uint8)*255) # print(iou, fname) return segmentation
def segment_iou(a, b): """ Generic IoU computation with masks, polygons, and boxes. Returns -1 if no intersection, [0; 1] otherwise """ from pycocotools import mask as mask_utils a_bbox = a.get_bbox() b_bbox = b.get_bbox() is_bbox = AnnotationType.bbox in [a.type, b.type] if is_bbox: a = [a_bbox] b = [b_bbox] else: w = max(a_bbox[0] + a_bbox[2], b_bbox[0] + b_bbox[2]) h = max(a_bbox[1] + a_bbox[3], b_bbox[1] + b_bbox[3]) def _to_rle(ann): if ann.type == AnnotationType.polygon: return mask_utils.frPyObjects([ann.points], h, w) elif isinstance(ann, RleMask): return [ann._rle] elif ann.type == AnnotationType.mask: return mask_utils.frPyObjects([mask_to_rle(ann.image)], h, w) else: raise TypeError("Unexpected arguments: %s, %s" % (a, b)) a = _to_rle(a) b = _to_rle(b) return float(mask_utils.iou(a, b, [not is_bbox]))
def test_bbox_dataset_to_prediction_roundtrip(self): """Simulate the process of reading a ground-truth box from a dataset, make predictions from proposals, convert the predictions back to the dataset format, and then use the COCO API to compute IoU overlap between the gt box and the predictions. These should have IoU of 1. """ weights = (5, 5, 10, 10) # 1/ "read" a box from a dataset in the default (x1, y1, w, h) format gt_xywh_box = [10, 20, 100, 150] # 2/ convert it to our internal (x1, y1, x2, y2) format gt_xyxy_box = box_utils.xywh_to_xyxy(gt_xywh_box) # 3/ consider nearby proposal boxes prop_xyxy_boxes = random_boxes(gt_xyxy_box, 10, 10) # 4/ compute proposal-to-gt transformation deltas deltas = box_utils.bbox_transform_inv( prop_xyxy_boxes, np.array([gt_xyxy_box]), weights=weights ) # 5/ use deltas to transform proposals to xyxy predicted box pred_xyxy_boxes = box_utils.bbox_transform( prop_xyxy_boxes, deltas, weights=weights ) # 6/ convert xyxy predicted box to xywh predicted box pred_xywh_boxes = box_utils.xyxy_to_xywh(pred_xyxy_boxes) # 7/ use COCO API to compute IoU not_crowd = [int(False)] * pred_xywh_boxes.shape[0] ious = COCOmask.iou(pred_xywh_boxes, np.array([gt_xywh_box]), not_crowd) np.testing.assert_array_almost_equal(ious, np.ones(ious.shape))
def _checkIgnore(dt, iregion): if iregion is None or (isinstance(iregion, list) and len(iregion) == 0): return True bb = np.array(dt['bbox']).astype(np.int) x1, y1, x2, y2 = bb[0], bb[1], bb[0] + bb[2], bb[1] + bb[3] x2 = min([x2, iregion.shape[1]]) y2 = min([y2, iregion.shape[0]]) if bb[2] * bb[3] == 0: return False crop_iregion = iregion[y1:y2, x1:x2] if crop_iregion.sum() == 0: return True if not 'uv' in dt.keys(): # filtering boxes return crop_iregion.sum() / bb[2] / bb[3] < self.ignoreThrBB # filtering UVs ignoremask = np.require(crop_iregion, requirements=['F']) uvmask = np.require(np.asarray(dt['uv'][0] > 0), dtype=np.uint8, requirements=['F']) uvmask_ = maskUtils.encode(uvmask) ignoremask_ = maskUtils.encode(ignoremask) uviou = maskUtils.iou([uvmask_], [ignoremask_], [1])[0] return uviou < self.ignoreThrUV
def compute_iou(self, det: np.ndarray, gt: np.ndarray) -> np.ndarray: """Compute intersection over union. We leverage `maskUtils.iou`. Args: det: Detection array. gt: Ground truth array. Returns: Intersection of union array. """ num_dt = len(det) num_gt = len(gt) if num_gt == 0 and num_dt == 0: return [] boxes_a = np.zeros(shape=(0, 4), dtype=float) boxes_b = np.zeros(shape=(0, 4), dtype=float) inds = np.argsort([-d['score'] for d in det], kind='mergesort') det = [det[i] for i in inds] if len(det) > self.max_detection: det = det[0:self.max_detection] boxes_a = [[dt_elem['x1'], dt_elem['y1'], dt_elem['w'], dt_elem['h']] for dt_elem in det] boxes_b = [[gt_elem['x1'], gt_elem['y1'], gt_elem['w'], gt_elem['h']] for gt_elem in gt] iscrowd = [0] * num_gt # to leverage maskUtils.iou iou_dt_gt = maskUtils.iou(boxes_a, boxes_b, iscrowd) return iou_dt_gt
def computeIoU(self, imgId, catId): p = self.params if p.useCats: gt = self._gts[imgId,catId] dt = self._dts[imgId,catId] else: gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] if len(gt) == 0 and len(dt) ==0: return [] inds = np.argsort([-d['score'] for d in dt], kind='mergesort') dt = [dt[i] for i in inds] if len(dt) > p.maxDets[-1]: dt=dt[0:p.maxDets[-1]] if p.iouType == 'segm': g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] elif p.iouType == 'bbox': g = [g['bbox'] for g in gt] d = [d['bbox'] for d in dt] else: raise Exception('unknown iouType for iou computation') # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] ious = maskUtils.iou(d, g, iscrowd) return ious
def computeIoU(gt, dt, iouType="bbox", maxDets=100): if len(gt) == 0 and len(dt) == 0: return [] inds = np.argsort([-d["score"] for d in dt], kind="mergesort") dt = [dt[i] for i in inds] if len(dt) > maxDets: dt = dt[0:maxDets] if iouType == "segm": g = [g["segmentation"] for g in gt] d = [d["segmentation"] for d in dt] elif iouType == "bbox": g = [g["bbox"] for g in gt] d = [d["bbox"] for d in dt] else: raise Exception("unknown iouType for iou computation") # compute iou between each dt and gt region iscrowd = [int(o["iscrowd"]) for o in gt] ious = mask_util.iou(d, g, iscrowd) # #### # print("id:%s - cat:%d" % (imgId, catId)) # print("d:", d) # print("g:", g) # print("ious", ious) # #### return ious
def computeOgps(self, imgId, catId): p = self.params # dimention here should be Nxm g = self._gts[imgId, catId] d = self._dts[imgId, catId] inds = np.argsort([-d_['score'] for d_ in d], kind='mergesort') d = [d[i] for i in inds] if len(d) > p.maxDets[-1]: d = d[0:p.maxDets[-1]] # if len(gts) == 0 and len(dts) == 0: if len(g) == 0 or len(d) == 0: return [] ious = np.zeros((len(d), len(g))) # compute opgs between each detection and ground truth object sigma = self.sigma #0.255 # dist = 0.3m corresponds to ogps = 0.5 # 1 # dist = 0.3m corresponds to ogps = 0.96 # 1.45 # dist = 1.7m (person height) corresponds to ogps = 0.5) for j, gt in enumerate(g): if not gt['ignore']: g_ = gt['bbox'] for i, dt in enumerate(d): # dx = dt['bbox'][3] dy = dt['bbox'][2] dp_x = np.array( gt['dp_x'] )*g_[2]/255. dp_y = np.array( gt['dp_y'] )*g_[3]/255. px = ( dp_y + g_[1] - dt['bbox'][1]).astype(np.int) py = ( dp_x + g_[0] - dt['bbox'][0]).astype(np.int) # pts = np.zeros(len(px)) pts[px>=dx] = -1; pts[py>=dy] = -1 pts[px<0] = -1; pts[py<0] = -1 #print(pts.shape) if len(pts) < 1: ogps = 0. elif np.max(pts) == -1: ogps = 0. else: px[pts==-1] = 0; py[pts==-1] = 0; ipoints = dt['uv'][0, px, py] upoints = dt['uv'][1, px, py]/255. # convert from uint8 by /255. vpoints = dt['uv'][2, px, py]/255. ipoints[pts==-1] = 0 ## Find closest vertices in subsampled mesh. cVerts, cVertsGT = self.findAllClosestVerts(gt, upoints, vpoints, ipoints) ## Get pairwise geodesic distances between gt and estimated mesh points. dist = self.getDistances(cVertsGT, cVerts) ## Compute the Ogps measure. ogps = np.sum(np.exp(-(dist**2)/(2*(sigma**2)))) if len(dist)>0: ogps = ogps / len(dist) ious[i, j] = ogps gbb = [gt['bbox'] for gt in g] dbb = [dt['bbox'] for dt in d] # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in g] ious_bb = maskUtils.iou(dbb, gbb, iscrowd) return ious, ious_bb
def iou_rle(boxes1, boxes2, img_size=2048): ''' Use mask and Run Length Encoding to calculate IOU between rotated bboxes. NOTE: rotated bounding boxes format is [cx, cy, w, h, degree]. Args: boxes1: list[list[float]], shape[M,5], 5=(cx, cy, w, h, degree) boxes2: list[list[float]], shape[N,5], 5=(cx, cy, w, h, degree) img_size: int or list, (height, width) Return: ious: np.array[M,N], ious of all bounding box pairs ''' assert isinstance(boxes1, list) and isinstance(boxes2, list) # convert bounding boxes to torch.tensor boxes1 = np.array(boxes1).reshape(-1, 5) boxes2 = np.array(boxes2).reshape(-1, 5) if boxes1.shape[0] == 0 or boxes2.shape[0] == 0: return np.zeros((boxes1.shape[0], boxes2.shape[0])) # Convert angle from degree to radian boxes1[:, 4] = boxes1[:, 4] * np.pi / 180 boxes2[:, 4] = boxes2[:, 4] * np.pi / 180 b1 = xywha2vertex(boxes1, is_degree=False).tolist() b2 = xywha2vertex(boxes2, is_degree=False).tolist() h, w = (img_size, img_size) if isinstance(img_size, int) else img_size b1 = maskUtils.frPyObjects(b1, h, w) b2 = maskUtils.frPyObjects(b2, h, w) ious = maskUtils.iou(b1, b2, [0 for _ in b2]) return ious
def test_bbox_dataset_to_prediction_roundtrip(self): """Simulate the process of reading a ground-truth box from a dataset, make predictions from proposals, convert the predictions back to the dataset format, and then use the COCO API to compute IoU overlap between the gt box and the predictions. These should have IoU of 1. """ weights = (5, 5, 10, 10) # 1/ "read" a box from a dataset in the default (x1, y1, w, h) format gt_xywh_box = [10, 20, 100, 150] # 2/ convert it to our internal (x1, y1, x2, y2) format gt_xyxy_box = box_utils.xywh_to_xyxy(gt_xywh_box) # 3/ consider nearby proposal boxes prop_xyxy_boxes = random_boxes(gt_xyxy_box, 10, 10) # 4/ compute proposal-to-gt transformation deltas deltas = box_utils.bbox_transform_inv(prop_xyxy_boxes, np.array([gt_xyxy_box]), weights=weights) # 5/ use deltas to transform proposals to xyxy predicted box pred_xyxy_boxes = box_utils.bbox_transform(prop_xyxy_boxes, deltas, weights=weights) # 6/ convert xyxy predicted box to xywh predicted box pred_xywh_boxes = box_utils.xyxy_to_xywh(pred_xyxy_boxes) # 7/ use COCO API to compute IoU not_crowd = [int(False)] * pred_xywh_boxes.shape[0] ious = COCOmask.iou(pred_xywh_boxes, np.array([gt_xywh_box]), not_crowd) np.testing.assert_array_almost_equal(ious, np.ones(ious.shape))
def computeIoU(self, imgId, catId): p = self.params if p.useCats: gt = self._gts[imgId, catId] dt = self._dts[imgId, catId] else: gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] if len(gt) == 0 and len(dt) == 0: return [] dt = sorted(dt, key=lambda x: -x['score']) if len(dt) > p.maxDets[-1]: dt = dt[0:p.maxDets[-1]] if p.useSegm: g = [g['segmentation'] for g in gt] d = [d['segmentation'] for d in dt] else: g = [g['bbox'] for g in gt] d = [d['bbox'] for d in dt] # compute iou between each dt and gt region iscrowd = [int(o['iscrowd']) for o in gt] ious = mask.iou(d, g, iscrowd) return ious
def _calculate_mask_ious(masks1, masks2, is_encoded=False, do_ioa=False): """ Calculates the IOU (intersection over union) between two arrays of segmentation masks. If is_encoded a run length encoding with pycocotools is assumed as input format, otherwise an input of numpy arrays of the shape (num_masks, height, width) is assumed and the encoding is performed. If do_ioa (intersection over area) , then calculates the intersection over the area of masks1 - this is commonly used to determine if detections are within crowd ignore region. :param masks1: first set of masks (numpy array of shape (num_masks, height, width) if not encoded, else pycocotools rle encoded format) :param masks2: second set of masks (numpy array of shape (num_masks, height, width) if not encoded, else pycocotools rle encoded format) :param is_encoded: whether the input is in pycocotools rle encoded format :param do_ioa: whether to perform IoA computation :return: the IoU/IoA scores """ # Only loaded when run to reduce minimum requirements from pycocotools import mask as mask_utils # use pycocotools for run length encoding of masks if not is_encoded: masks1 = mask_utils.encode( np.array(np.transpose(masks1, (1, 2, 0)), order='F')) masks2 = mask_utils.encode( np.array(np.transpose(masks2, (1, 2, 0)), order='F')) # use pycocotools for iou computation of rle encoded masks ious = np.asarray( mask_utils.iou(masks1, masks2, [do_ioa for _ in range(len(masks1))])) if len(masks1) == 0 or len(masks2) == 0: ious = ious.reshape(len(masks1), len(masks2)) assert (ious >= 0).all() assert (ious <= 1).all() return ious
def eval_video(final_solution, input_images, ground_truth_anns): _, example_prop = final_solution[0] scores = np.zeros(len(example_prop)) for image_fn, selected_props in final_solution[1:-1]: gt_fn = image_fn.replace(input_images, ground_truth_anns).replace('.jpg', '.png') gt_props = read_ann(gt_fn) segs = [prop['segmentation'] for prop in selected_props] gt_segs = [templ['segmentation'] for templ in gt_props] gt_ids = [templ['id'] for templ in gt_props] for temp_id, seg in enumerate(segs): gt_id = temp_id + 1 if gt_id not in gt_ids: if area(seg) == 0: score = 1 else: score = 0 else: gt_seg = [ c_seg for c_seg, c_id in zip(gt_segs, gt_ids) if c_id == gt_id ][0] score = iou([ seg, ], [ gt_seg, ], np.array([0], np.uint8))[0, 0] scores[temp_id] += score final_scores = scores / (len(final_solution) - 2) return final_scores
def generate_tracklet_props(all_props, overlap_threshold=0.2): all_tracklet_props = [ ] # all possible tracklet proposals in video sequence -> list of dictionaries all_wrap_scores = [] # all wrap score matrices in video sequence frames_no_proposals = [] # frames that have no proposals available def props_info(props, key, props_ids): return [props[key][i] for i in props_ids] prev_props = remove_overlap_props(all_props[0], all_props[0]['id'], threshold=overlap_threshold) prev_next_segs = props_info(all_props[0], 'fwd', prev_props) prev_reid_score = props_info(all_props[0], 'reid', prev_props) prev_seg = props_info(all_props[0], 'seg', prev_props) prev_score = props_info(all_props[0], 'score', prev_props) for t, props in enumerate(all_props[1:]): final_props = remove_overlap_props(props, props['id'], threshold=overlap_threshold) if len(final_props) > 0 and len(prev_next_segs) > 0: wrap_scores = arr([ iou([props['seg'][fp] for fp in final_props], [prev_next_seg], arr([0], np.uint8))[:, 0] for prev_next_seg in prev_next_segs ]) else: wrap_scores = None all_wrap_scores.append(wrap_scores) if len(props['seg']) == 0: frames_no_proposals.append(t + 1) all_tracklet_props.append([{ 'id': pp, 'reid': prev_reid_score[i], 'score': prev_score[i], 'area': area(prev_seg[i]) } for i, pp in enumerate(prev_props)]) prev_props = final_props prev_next_segs = props_info(props, 'fwd', prev_props) prev_reid_score = props_info(props, 'reid', prev_props) prev_seg = props_info(props, 'seg', prev_props) prev_score = props_info(props, 'score', prev_props) all_tracklet_props.append([{ 'id': pp, 'reid': prev_reid_score[i], 'score': prev_score[i], 'area': area(prev_seg[i]) } for i, pp in enumerate(prev_props) ]) # adding the proposals in the last frame return all_tracklet_props, frames_no_proposals, all_wrap_scores
def iou_rle(boxes1, boxes2, xywha, is_degree=True, **kwargs): r''' use mask method to calculate IOU between boxes1 and boxes2 Arguments: boxes1: tensor or numpy, shape(N,5), 5=(x, y, w, h, angle 0~90) boxes2: tensor or numpy, shape(M,5), 5=(x, y, w, h, angle 0~90) xywha: True if xywha, False if xyxya is_degree: True if degree, False if radian Return: iou_matrix: tensor, shape(N,M), float32, ious of all possible pairs between boxes1 and boxes2 ''' assert xywha == True and is_degree == True if not (torch.is_tensor(boxes1) and torch.is_tensor(boxes2)): print( 'Warning: bounding boxes are np.array. converting to torch.tensor') # convert to tensor, (batch, (x,y,w,h,a)) boxes1 = torch.from_numpy(boxes1).float() boxes2 = torch.from_numpy(boxes2).float() assert boxes1.device == boxes2.device device = boxes1.device boxes1, boxes2 = boxes1.cpu().clone().detach(), boxes2.cpu().clone( ).detach() if boxes1.dim() == 1: boxes1 = boxes1.unsqueeze(0) if boxes2.dim() == 1: boxes2 = boxes2.unsqueeze(0) assert boxes1.shape[1] == boxes2.shape[1] == 5 size = kwargs.get('img_size', 2048) h, w = size if isinstance(size, tuple) else size, size if 'normalized' in kwargs and kwargs['normalized'] == True: # the [x,y,w,h] are between 0~1 # assert (boxes1[:,:4] <= 1).all() and (boxes2[:,:4] <= 1).all() boxes1[:, 0] *= w boxes1[:, 1] *= h boxes1[:, 2] *= w boxes1[:, 3] *= h boxes2[:, 0] *= w boxes2[:, 1] *= h boxes2[:, 2] *= w boxes2[:, 3] *= h if is_degree: # convert to radian boxes1[:, 4] = boxes1[:, 4] * pi / 180 boxes2[:, 4] = boxes2[:, 4] * pi / 180 b1 = xywha2vertex(boxes1, is_degree=False, stack=False).tolist() b2 = xywha2vertex(boxes2, is_degree=False, stack=False).tolist() debug = 1 b1 = maskUtils.frPyObjects(b1, h, w) b2 = maskUtils.frPyObjects(b2, h, w) ious = maskUtils.iou(b1, b2, [0 for _ in b2]) return torch.from_numpy(ious).to(device=device)
def rle_mask_nms(masks, dets, thresh, mode='IOU'): """Performs greedy non-maximum suppression based on an overlap measurement between masks. The type of measurement is determined by `mode` and can be either 'IOU' (standard intersection over union) or 'IOMA' (intersection over mininum area). """ if len(masks) == 0: return [] if len(masks) == 1: return [0] if mode == 'IOU': # Computes ious[m1, m2] = area(intersect(m1, m2)) / area(union(m1, m2)) all_not_crowds = [False] * len(masks) ious = mask_util.iou(masks, masks, all_not_crowds) elif mode == 'IOMA': # Computes ious[m1, m2] = area(intersect(m1, m2)) / min(area(m1), area(m2)) all_crowds = [True] * len(masks) # ious[m1, m2] = area(intersect(m1, m2)) / area(m2) ious = mask_util.iou(masks, masks, all_crowds) # ... = max(area(intersect(m1, m2)) / area(m2), # area(intersect(m2, m1)) / area(m1)) ious = np.maximum(ious, ious.transpose()) elif mode == 'CONTAINMENT': # Computes ious[m1, m2] = area(intersect(m1, m2)) / area(m2) # Which measures how much m2 is contained inside m1 all_crowds = [True] * len(masks) ious = mask_util.iou(masks, masks, all_crowds) else: raise NotImplementedError('Mode {} is unknown'.format(mode)) scores = dets[:, 4] order = np.argsort(-scores) keep = [] while order.size > 0: i = order[0] keep.append(i) ovr = ious[i, order[1:]] inds_to_keep = np.where(ovr <= thresh)[0] order = order[inds_to_keep + 1] return keep
def np_iou(A, B): def to_xywh(box): box = box.copy() box[:, 2] -= box[:, 0] box[:, 3] -= box[:, 1] return box ret = cocomask.iou( to_xywh(A), to_xywh(B), np.zeros((len(B),), dtype=np.bool)) # can accelerate even more, if using float32 return ret.astype('float32')
def _do_test(b1, b2): # Compute IoU overlap with the cython implementation cython_iou = box_utils.bbox_overlaps(b1, b2) # Compute IoU overlap with the COCO API implementation # (requires converting boxes from xyxy to xywh format) xywh_b1 = box_utils.xyxy_to_xywh(b1) xywh_b2 = box_utils.xyxy_to_xywh(b2) not_crowd = [int(False)] * b2.shape[0] coco_ious = COCOmask.iou(xywh_b1, xywh_b2, not_crowd) # IoUs should be similar np.testing.assert_array_almost_equal( cython_iou, coco_ious, decimal=5 )
def compute_ious(gt, predictions): gt_ = get_segmentations(gt) predictions_ = get_segmentations(predictions) if len(gt_) == 0 and len(predictions_) == 0: return np.ones((1, 1)) elif len(gt_) != 0 and len(predictions_) == 0: return np.zeros((1, 1)) else: iscrowd = [0 for _ in predictions_] ious = cocomask.iou(gt_, predictions_, iscrowd) if not np.array(ious).size: ious = np.zeros((1, 1)) return ious
def _filter_crowd_proposals(roidb, crowd_thresh): """Finds proposals that are inside crowd regions and marks them as overlap = -1 with each ground-truth rois, which means they will be excluded from training. """ for entry in roidb: gt_overlaps = entry['gt_overlaps'].toarray() crowd_inds = np.where(entry['is_crowd'] == 1)[0] non_gt_inds = np.where(entry['gt_classes'] == 0)[0] if len(crowd_inds) == 0 or len(non_gt_inds) == 0: continue crowd_boxes = box_utils.xyxy_to_xywh(entry['boxes'][crowd_inds, :]) non_gt_boxes = box_utils.xyxy_to_xywh(entry['boxes'][non_gt_inds, :]) iscrowd_flags = [int(True)] * len(crowd_inds) ious = COCOmask.iou(non_gt_boxes, crowd_boxes, iscrowd_flags) bad_inds = np.where(ious.max(axis=1) > crowd_thresh)[0] gt_overlaps[non_gt_inds[bad_inds], :] = -1 entry['gt_overlaps'] = scipy.sparse.csr_matrix(gt_overlaps)
def _filter_crowd_proposals(roidb, crowd_thresh): """ Finds proposals that are inside crowd regions and marks them with overlap = -1 (for all gt rois), which means they will be excluded from training. """ for ix, entry in enumerate(roidb): overlaps = entry['gt_overlaps'].toarray() crowd_inds = np.where(overlaps.max(axis=1) == -1)[0] non_gt_inds = np.where(entry['gt_classes'] == 0)[0] if len(crowd_inds) == 0 or len(non_gt_inds) == 0: continue iscrowd = [int(True) for _ in xrange(len(crowd_inds))] crowd_boxes = ds_utils.xyxy_to_xywh(entry['boxes'][crowd_inds, :]) non_gt_boxes = ds_utils.xyxy_to_xywh(entry['boxes'][non_gt_inds, :]) ious = COCOmask.iou(non_gt_boxes, crowd_boxes, iscrowd) bad_inds = np.where(ious.max(axis=1) > crowd_thresh)[0] overlaps[non_gt_inds[bad_inds], :] = -1 roidb[ix]['gt_overlaps'] = scipy.sparse.csr_matrix(overlaps) return roidb
def rle_mask_voting( top_masks, all_masks, all_dets, iou_thresh, binarize_thresh, method='AVG' ): """Returns new masks (in correspondence with `top_masks`) by combining multiple overlapping masks coming from the pool of `all_masks`. Two methods for combining masks are supported: 'AVG' uses a weighted average of overlapping mask pixels; 'UNION' takes the union of all mask pixels. """ if len(top_masks) == 0: return all_not_crowd = [False] * len(all_masks) top_to_all_overlaps = mask_util.iou(top_masks, all_masks, all_not_crowd) decoded_all_masks = [ np.array(mask_util.decode(rle), dtype=np.float32) for rle in all_masks ] decoded_top_masks = [ np.array(mask_util.decode(rle), dtype=np.float32) for rle in top_masks ] all_boxes = all_dets[:, :4].astype(np.int32) all_scores = all_dets[:, 4] # Fill box support with weights mask_shape = decoded_all_masks[0].shape mask_weights = np.zeros((len(all_masks), mask_shape[0], mask_shape[1])) for k in range(len(all_masks)): ref_box = all_boxes[k] x_0 = max(ref_box[0], 0) x_1 = min(ref_box[2] + 1, mask_shape[1]) y_0 = max(ref_box[1], 0) y_1 = min(ref_box[3] + 1, mask_shape[0]) mask_weights[k, y_0:y_1, x_0:x_1] = all_scores[k] mask_weights = np.maximum(mask_weights, 1e-5) top_segms_out = [] for k in range(len(top_masks)): # Corner case of empty mask if decoded_top_masks[k].sum() == 0: top_segms_out.append(top_masks[k]) continue inds_to_vote = np.where(top_to_all_overlaps[k] >= iou_thresh)[0] # Only matches itself if len(inds_to_vote) == 1: top_segms_out.append(top_masks[k]) continue masks_to_vote = [decoded_all_masks[i] for i in inds_to_vote] if method == 'AVG': ws = mask_weights[inds_to_vote] soft_mask = np.average(masks_to_vote, axis=0, weights=ws) mask = np.array(soft_mask > binarize_thresh, dtype=np.uint8) elif method == 'UNION': # Any pixel that's on joins the mask soft_mask = np.sum(masks_to_vote, axis=0) mask = np.array(soft_mask > 1e-5, dtype=np.uint8) else: raise NotImplementedError('Method {} is unknown'.format(method)) rle = mask_util.encode(np.array(mask[:, :, np.newaxis], order='F'))[0] top_segms_out.append(rle) return top_segms_out