def eval_recall(dataset, predictions, image_ids, curr_iter, output_folder): total_num = 0 recall_num = 0 recall_num_topN = 0 recall_num_det = 0 total_rel_softmax = 0 recall_rel_softmax = 0 for img_sent_id in image_ids: result = predictions[img_sent_id] if cfg.MODEL.VG.TWO_STAGE: if cfg.MODEL.RELATION_ON and cfg.MODEL.RELATION.USE_RELATION_CONST: gt_boxes, pred_boxes, pred_box_topN, pred_boxes_det, \ pred_sim, pred_sim_topN, pred_rel_sim, pred_rel_gt, batch_topN_boxes, batch_reg_offset_topN, batch_rel_score_mat = result pred_rel_sim = pred_rel_sim.numpy() pred_rel_gt = pred_rel_gt.numpy() if pred_rel_gt.shape[0] != 0: pred_rel_sim_argmax = pred_rel_sim.argmax(1) pred_rel_gt = pred_rel_gt[np.arange(pred_rel_gt.shape[0]), pred_rel_sim_argmax] total_rel_softmax += pred_rel_gt.shape[0] recall_rel_softmax += (pred_rel_gt > 0).astype(np.float32).sum() else: gt_boxes, pred_boxes, pred_box_topN, pred_boxes_det, pred_sim = result pred_box_topN = BoxList(pred_box_topN, gt_boxes.size, mode="xyxy") pred_box_topN.clip_to_image() ious_topN = boxlist_iou(gt_boxes, pred_box_topN) ious_topN = ious_topN.cpu().numpy().diagonal() recall_num_topN += int((ious_topN >= cfg.MODEL.VG.EVAL_THRESH).sum()) else: gt_boxes, pred_boxes, pred_boxes_det, pred_sim = result pred_boxes = BoxList(pred_boxes, gt_boxes.size, mode="xyxy") pred_boxes.clip_to_image() ious = boxlist_iou(gt_boxes, pred_boxes) iou = ious.cpu().numpy().diagonal() total_num += iou.shape[0] recall_num += int((iou>=cfg.MODEL.VG.EVAL_THRESH).sum()) # 0.5 pred_boxes_det = BoxList(pred_boxes_det, gt_boxes.size, mode="xyxy") pred_boxes_det.clip_to_image() ious_det = boxlist_iou(gt_boxes, pred_boxes_det) iou_det = ious_det.cpu().numpy().diagonal() recall_num_det += int((iou_det>=cfg.MODEL.VG.EVAL_THRESH).sum()) # 0.5 acc = recall_num/total_num acc_topN = recall_num_topN/total_num acc_det = recall_num_det/total_num acc_rel_softmax = recall_rel_softmax / (total_rel_softmax+1e-6) return (acc, acc_topN, acc_det, acc_rel_softmax)
def detect_relsample(self, proposals, targets): # corresponding to rel_assignments function in neural-motifs """ The input proposals are already processed by subsample function of box_head, in this function, we should only care about fg box, and sample corresponding fg/bg relations Note: this function keeps a state. Arguments: proposals (list[BoxList]) contain fields: labels, predict_logits targets (list[BoxList]) contain fields: labels """ self.num_pos_per_img = int(self.batch_size_per_image * self.positive_fraction) rel_idx_pairs = [] rel_labels = [] rel_sym_binarys = [] for img_id, (proposal, target) in enumerate(zip(proposals, targets)): device = proposal.bbox.device prp_box = proposal.bbox prp_lab = proposal.get_field("labels").long() tgt_box = target.bbox tgt_lab = target.get_field("labels").long() tgt_rel_matrix = target.get_field("relation") # [tgt, tgt] # IoU matching ious = boxlist_iou(target, proposal) # [tgt, prp] is_match = (tgt_lab[:, None] == prp_lab[None]) & ( ious > self.fg_thres) # [tgt, prp] # Proposal self IoU to filter non-overlap prp_self_iou = boxlist_iou(proposal, proposal) # [prp, prp] if self.require_overlap and (not self.use_gt_box): rel_possibility = (prp_self_iou > 0) & ( prp_self_iou < 1) # not self & intersect else: num_prp = prp_box.shape[0] rel_possibility = torch.ones( (num_prp, num_prp), device=device).long() - torch.eye( num_prp, device=device).long() # only select relations between fg proposals rel_possibility[prp_lab == 0] = 0 rel_possibility[:, prp_lab == 0] = 0 img_rel_triplets, binary_rel = self.motif_rel_fg_bg_sampling( device, tgt_rel_matrix, ious, is_match, rel_possibility) rel_idx_pairs.append( img_rel_triplets[:, :2]) # (num_rel, 2), (sub_idx, obj_idx) rel_labels.append(img_rel_triplets[:, 2]) # (num_rel, ) rel_sym_binarys.append(binary_rel) return proposals, rel_labels, rel_idx_pairs, rel_sym_binarys
def match_targets_to_anchors(self, anchor, target_left, target_right, copied_fields=[]): # with torch.no_grad(): # bbox_new = target.convert("xyxy").bbox.clone().detach() # bbox_new2 = target.convert("xyxy").bbox.clone().detach() # disps = target.get_field("depths").convert("disp").depths # bbox_new[:, 0] -= disps # bbox_new2[:, 0] -= disps # bbox_new2[:, 2] -= disps # # print(bbox_new) # target_union = BoxList(bbox_new, target.size, mode="xyxy") # target_right = BoxList(bbox_new2, target.size, mode="xyxy") # target_union.clip_to_image(remove_empty=True) target_union = boxlist_union(target_left, target_right) match_quality_matrix = boxlist_iou(target_union, anchor) matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all target_left = target_left.copy_with_fields(copied_fields) target_right = target_right.copy_with_fields(copied_fields) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target_left[matched_idxs.clamp(min=0)] matched_targets_right = target_right[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) matched_targets_right.add_field("matched_idxs", matched_idxs) return matched_targets, matched_targets_right
def match_targets_to_proposals(self, proposal, target, object3d): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) matched_object3d = object3d[matched_idxs.clamp(min=0)] return matched_object3d, matched_idxs
def match_targets_to_anchors(self, anchor, target, copied_fields=[]): if cfg.ROTATE: match_quality_matrix = boxlist_riou(anchor, target) matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all if "RETINANET" in cfg.MODEL.BACKBONE.CONV_BODY: if cfg.MODEL.RETINANET_DCN_ON: target = target.copy_with_fields(["xyxyr", "labels"]) else: target = target.copy_with_fields(["xywht", "labels"]) else: target = target.copy_with_fields(["xyxy"]) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets else: match_quality_matrix = boxlist_iou(target, anchor) matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all target = target.copy_with_fields(copied_fields) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets copied_fields = ['labels'] if target.has_field('tightness'): copied_fields.append('tightness') target = target.copy_with_fields(copied_fields) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds if len(target) == 0: dummy_bbox = torch.zeros((len(matched_idxs), 4), dtype=torch.float32, device=matched_idxs.device) from maskrcnn_benchmark.structures.bounding_box import BoxList matched_targets = BoxList(dummy_bbox, target.size, target.mode) matched_targets.add_field('labels', self.create_all_bkg_labels( len(matched_idxs), matched_idxs.device)) matched_targets.add_field('tightness', torch.zeros(len(matched_idxs), device=matched_idxs.device)) else: matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) if self.weight_box_loss: matched_iou = torch.full((len(matched_idxs),), -1., device=matched_idxs.device) matched_iou[matched_idxs >= 0] = match_quality_matrix[matched_idxs[matched_idxs >= 0], matched_idxs >= 0] matched_targets.add_field('matched_iou', matched_iou) return matched_targets
def match_both_targets_to_proposals(self, proposal, target, posv_targets): match_quality_matrix = boxlist_iou(target, proposal) match_quality_matrix_iog = boxlist_iog(posv_targets, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) matched_with_visible_idxs = self.proposal_matcher_with_visible_iog( match_quality_matrix_iog) index_out_bounds = np.where(matched_idxs.cpu().numpy() == -2)[0] matched_both = np.where((matched_idxs.cpu().numpy() > -1) & ( matched_with_visible_idxs.cpu().numpy() > -1))[0] matched_both = torch.tensor(matched_both, dtype=matched_idxs.dtype, device=matched_idxs.device) index_out_bounds = torch.tensor(index_out_bounds, dtype=matched_idxs.dtype, device=matched_idxs.device) matched_new = -1 * torch.ones_like( matched_idxs, dtype=matched_idxs.dtype) # background matched_new[matched_both] = matched_idxs[ matched_both] # TODO: label is important matched_new[index_out_bounds] = matched_idxs[ index_out_bounds] # out of bounds # Fast RCNN only need "labels" field for selecting the targets target = target.copy_with_fields("labels") # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_new.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_new) return matched_targets
def match_targets_to_anchors(self, anchor, target, copied_fields=[]): match_quality_matrix = boxlist_iou(target, anchor) # ################# changed by hui ################################################### if len(target.bbox) == 0: matched_idxs = torch.LongTensor([-1] * match_quality_matrix.shape[1]) else: matched_idxs, _ = self.proposal_matcher( match_quality_matrix) # ,_ add by G # for anchor recall cal # if self.debug: # record_for_recall(matched_idxs, target) ##################################################################################### # RPN doesn't need any fields from target # for creating the labels, so clear them all target = target.copy_with_fields(copied_fields) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) target = target.copy_with_fields(["labels", "masks"]) matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): if cfg.ROTATE: if cfg.R2CNN: match_quality_matrix = boxlist_riou(proposal, target) else: match_quality_matrix = boxlist_riou(proposal, target) matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets target = target.copy_with_fields( ["labels", "xyxy", "xywht", "xywht1"]) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) else: match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets target = target.copy_with_fields("labels") # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target, is_source=True): """ 这个函数主要是计算一下proposal以及gt的IOU,然后筛选一下 相当于是选择出来了每个proposal对应的gt """ # 计算基准边框与预测边框相互之间的IoU match_quality_matrix = boxlist_iou(target, proposal) # 计算各个预测边框对应的基准边框(ground truth box)的索引列表,背景边框为-2,模糊边框为-1 matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets # 获得基准边框(groun truth box)附加的属性labels标签,即边框的具体类别 target = target.copy_with_fields("labels") # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds # 计算各个预测边框对应的基准边框(ground truth box)列表,所有背景边框以及模糊边框都对应成第一个gt matched_targets = target[matched_idxs.clamp(min=0)] # DA start # 如果是目标域的数据的话 if not is_source: matched_targets = target[matched_idxs] # DA end # 将对应的列表索引附加到对应基准边框列表中 matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def ga_shape_target(self, square_anchors, approx_anchors, targets): shape_targets = [] shape_weights = [] for square_anchors_per_image, approx_anchors_per_image, targets_per_image in zip( square_anchors, approx_anchors, targets): match_quality_matrix = boxlist_iou(targets_per_image, approx_anchors_per_image) num_gt = targets_per_image.bbox.shape[0] match_quality_matrix = match_quality_matrix.view( num_gt, -1, self.num_approx_anchors_per_location) match_quality_matrix, _ = match_quality_matrix.max(dim=2) matched_idxs = self.proposal_matcher(match_quality_matrix) target = targets_per_image.copy_with_fields(self.copied_fields) matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) weights = self.generate_labels_func(matched_targets) weights = weights.to(dtype=torch.float32) bg_indices = matched_idxs == Matcher.BELOW_LOW_THRESHOLD weights[bg_indices] = 0 #if "not_visibility" in self.discard_cases: # weights[~square_anchors_per_image.get_field("visibility")] = 0 #if "between_thresholds" in self.discard_cases: # inds_to_discard = matched_idxs == Matcher.BETWEEN_THRESHOLDS # weights[inds_to_discard] = 0 shape_targets_per_image = matched_targets.bbox shape_targets.append(shape_targets_per_image) shape_weights.append(weights) return shape_targets, shape_weights
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target[0], proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) target = copy.deepcopy(target[1]) #target = target[1].copy_with_fields(["labels", "left_box", "right_box"]) matched_targets = target[matched_idxs.clamp(min=0)] return matched_targets, matched_idxs
def track_per_video(video_name): print(video_name) json_path = os.path.join(root, video_name+'.json') with open(json_path, 'r') as f: proposal_dict = json.load(f) gt = None frame_num = len(proposal_dict) boxes = np.zeros((frame_num, 4)) # xywh times = np.zeros(frame_num) i = 0 for img_name, proposals_ in proposal_dict.items(): start_time = time.time() if img_name == '00000001.jpg': gt = proposals_[0][:-1] boxes[0] = gt times[0] = time.time() - start_time gt = torch.Tensor(gt).reshape(1, 4) gt = BoxList(gt, (-1,-1), mode="xywh").convert("xyxy") i += 1 continue proposals = [proposal[:-1] for proposal in proposals_] scores = [proposal[-1] for proposal in proposals_] scores = torch.Tensor(scores) proposals = torch.Tensor(proposals) proposals = BoxList(proposals, (-1,-1), mode="xywh").convert("xyxy") proposals.add_field('objectness', scores) '''对proposals执行nms,保留top_n个样本。''' proposals_nms = boxlist_nms( proposals, 0.1, max_proposals=10, score_field="objectness", ) last_box = torch.Tensor(boxes[i - 1]).reshape(1, 4) last_box = BoxList(last_box, (-1, -1), mode="xywh").convert("xyxy") overlaps = boxlist_iou(proposals_nms, last_box).squeeze(0) selected_id = torch.argmax(overlaps) if overlaps[selected_id] == 0: print('消失') selected_id = torch.argmax(proposals_nms.extra_fields['objectness']) proposals_nms = proposals_nms.convert("xywh") res_box = proposals_nms.bbox[selected_id].cpu().numpy() boxes[i] = res_box # visualization(video_name, img_name, proposals_nms.bbox, res_box, boxes[i - 1]) times[i] = time.time() - start_time i += 1 '''保存该帧跟踪结果''' record_file = os.path.join(cfg.OUTPUT_DIR, 'result', video_name, '%s_%03d.txt' % (video_name, 1)) record_dir = os.path.dirname(record_file) if not os.path.isdir(record_dir): os.makedirs(record_dir) np.savetxt(record_file, boxes, fmt='%.3f', delimiter=',') '''保存时间文件''' time_file = record_file[:record_file.rfind('_')] + '_time.txt' times = times[:, np.newaxis] np.savetxt(time_file, times, fmt='%.8f', delimiter=',')
def compute_on_dataset(model, data_loader, device): model.eval() with_results_dict = {} without_results_dict = {} cpu_device = torch.device("cpu") for i, batch in tqdm(enumerate(data_loader)): images, targets, image_ids = batch images = images.to(device) with_overlaps = [] without_overlaps = [] with torch.no_grad(): with_outputs, without_outputs = model(images) for output, target in zip(with_outputs, targets): if len(target) == 0: continue if len(output) == 0: overlap = torch.zeros(len(target), device=cpu_device, dtype=torch.float) with_overlaps.append(overlap) else: overlap = boxlist_iou(output, target.to(device)).max(dim=0).values with_overlaps.append(overlap.to(cpu_device)) for output, target in zip(without_outputs, targets): if len(target) == 0: continue if len(output) == 0: overlap = torch.zeros(len(target), device=cpu_device, dtype=torch.float) without_overlaps.append(overlap) else: overlap = boxlist_iou(output, target.to(device)).max(dim=0).values without_overlaps.append(overlap.to(cpu_device)) with_results_dict.update({ img_id: result for img_id, result in zip(image_ids, with_overlaps) }) without_results_dict.update({ img_id: result for img_id, result in zip(image_ids, without_overlaps) }) return with_results_dict, without_results_dict
def compute_average_recall(ground_truths, proposals): match_quality_matrix = boxlist_iou(ground_truths, proposals) highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) highest_quality_foreach_gt -= 0.5 AR = 2 * torch.mean( torch.max( highest_quality_foreach_gt, torch.zeros(highest_quality_foreach_gt.size(), device='cuda'))).item() return AR
def match_targets_to_proposals(self, proposal, target): # NOTE(H): Seems the code like to keep xyxy. Then let's keep xyxy all around! match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) target = target.copy_with_fields(["labels", "vps", "vp_vert", "masks"]) matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_anchors_forFF(self, anchor, target, copied_fields=[]): match_quality_matrix = boxlist_iou(target, anchor) # ################# changed by G ################################################### if len(target.bbox) == 0: matches_gt_forFF = torch.LongTensor([0] * match_quality_matrix.shape[1]) else: _, matches_gt_forFF = self.proposal_matcher(match_quality_matrix) ##################################################################################### return matches_gt_forFF
def match_targets_to_anchors(self, anchor, target): match_quality_matrix = boxlist_iou(target, anchor) matched_idxs = self.proposal_matcher(match_quality_matrix) target = target.copy_with_fields(['labels']) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Keypoint RCNN needs "labels" and "keypoints "fields for creating the targets target = target.copy_with_fields(["labels", "bb8keypoints"]) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets target = target.copy_with_fields("labels") # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def bbox_overlaps(boxes1, boxes2): """ Parameters: boxes1 (m, 4) [List or np.array] : bounding boxes of (x1,y1,x2,y2) boxes2 (n, 4) [List or np.array] : bounding boxes of (x1,y1,x2,y2) Return: iou (m, n) [np.array] """ boxes1 = BoxList(boxes1, (0, 0), 'xyxy') boxes2 = BoxList(boxes2, (0, 0), 'xyxy') iou = boxlist_iou(boxes1, boxes2).cpu().numpy() return iou
def get_tracking_result(last_box, proposals): """ last_box: xywh proposals: BoxList, mode=xyxy """ last_box = torch.from_numpy(last_box).reshape(1, 4) last_box = BoxList(last_box, proposals.size, mode="xywh").convert("xyxy") '''计算IoU''' overlaps = boxlist_iou(proposals, last_box) proposals = proposals.convert("xywh") res = proposals.bbox[torch.argmax(overlaps)].cpu().numpy() return res
def get_gt_index(gt_lists, predictions, iou_thresh=0.5): score = defaultdict(list) match = defaultdict(list) assert len(gt_lists) == len( predictions), "Length of gt and pred lists need to be same." gt_index_list = [] for gt_list, pred_list in zip(gt_lists, predictions): pred_bbox = pred_list.get_field("left_box").bbox.numpy() pred_label = pred_list.get_field("left_box").get_field( "labels").numpy() pred_score = pred_list.get_field("left_box").get_field( "scores").numpy() gt_bbox = gt_list.get_field("left_box").bbox.numpy() gt_label = gt_list.get_field("labels").numpy() gt_index_per_image = dict() for l in np.unique(np.concatenate((pred_label, gt_label)).astype(int)): pred_mask_l = pred_label == l pred_bbox_l = pred_bbox[pred_mask_l] pred_score_l = pred_score[pred_mask_l] order = pred_score_l.argsort()[::-1] pred_bbox_l = pred_bbox_l[order] pred_score_l = pred_score_l[order] gt_mask_l = gt_label == l gt_bbox_l = gt_bbox[gt_mask_l] score[l].extend(pred_score_l) if len(pred_bbox_l) == 0 or len(gt_bbox_l) == 0: continue pred_bbox_l = pred_bbox_l.copy() pred_bbox_l[:, 2:] += 1 gt_bbox_l = gt_bbox_l.copy() gt_bbox_l[:, 2:] += 1 iou = boxlist_iou( BoxList(pred_bbox_l, gt_list.get_field("left_box").size), BoxList(gt_bbox_l, gt_list.get_field("left_box").size), ).numpy() gt_index = iou.argmax(axis=1) gt_index[iou.max(axis=1) < iou_thresh] = -1 del iou gt_index_per_image[l] = [order, gt_index] gt_index_list.append(gt_index_per_image) return gt_index_list
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets # Added additional field target = target.copy_with_fields(["labels", "fitz_categories"]) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_anchors(self, anchor, target, copied_fields=[]): match_quality_matrix = boxlist_iou(target, anchor) matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all target = target.copy_with_fields(copied_fields) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # print(match_quality_matrix, matched_idxs) target = target.copy_with_fields(["labels", "depths"], skip_missing=True) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_anchors(self, anchor, target): match_quality_matrix = boxlist_iou(target, anchor) matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all target = target.copy_with_fields([]) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou(target, proposal) matched_idxs = self.proposal_matcher(match_quality_matrix) # Mask RCNN needs "labels" and "masks "fields for creating the targets target = target.copy_with_fields( self.cfg['MODEL']['ROI_CAR_CLS_ROT_HEAD']['REGRESS_TARGET']) # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def match_targets_to_proposals(self, proposal, target): match_quality_matrix = boxlist_iou( target, proposal) # target和proposal都是xywh格式的吗 matched_idxs = self.proposal_matcher(match_quality_matrix) # Fast RCNN only need "labels" field for selecting the targets target = target.copy_with_fields("labels") # get the targets corresponding GT for each proposal # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) # proposal.add_field("matched_idxs", matched_idxs) # 后加的 return matched_targets
def compute_objectpairs_iou(targets, proposals): match_quality_matrix = boxlist_iou(targets, proposals) objects_pairs = targets.get_field('objects_pairs') match_s = match_quality_matrix[objects_pairs[:, 0], :].permute(1, 0) match_o = match_quality_matrix[objects_pairs[:, 1], :].permute(1, 0) match_s = match_s.unsqueeze(1).expand(match_s.size(0), match_s.size(0), match_s.size(1)) match_o = match_o.unsqueeze(0).expand(match_o.size(0), match_o.size(0), match_o.size(1)) match = match_s * match_o match = match.view(match_s.size(0)**2, match_s.size(2)).permute(1, 0) return match
def match_targets_to_anchors(self, anchor, target): # print(len(anchor),len(target),'==============================match=====') match_quality_matrix = boxlist_iou(target, anchor, type=1) # type=1: matching the square GT bbox type=0: normal matching matched_idxs = self.proposal_matcher(match_quality_matrix) # RPN doesn't need any fields from target # for creating the labels, so clear them all target = target.copy_with_fields(['rotations']) # get the targets corresponding GT for each anchor # NB: need to clamp the indices because we can have a single # GT in the image, and matched_idxs can be -2, which goes # out of bounds matched_targets = target[matched_idxs.clamp(min=0)] matched_targets.add_field("matched_idxs", matched_idxs) return matched_targets
def calc_detection_voc_prec_rec(gt_boxlists, pred_boxlists, iou_thresh=0.5): """Calculate precision and recall based on evaluation code of PASCAL VOC. This function calculates precision and recall of predicted bounding boxes obtained from a dataset which has :math:`N` images. The code is based on the evaluation code used in PASCAL VOC Challenge. """ n_pos = defaultdict(int) score = defaultdict(list) match = defaultdict(list) for gt_boxlist, pred_boxlist in zip(gt_boxlists, pred_boxlists): pred_bbox = pred_boxlist.bbox.numpy() pred_label = pred_boxlist.get_field("labels").numpy() pred_score = pred_boxlist.get_field("scores").numpy() gt_bbox = gt_boxlist.bbox.numpy() gt_label = gt_boxlist.get_field("labels").numpy() gt_difficult = gt_boxlist.get_field("difficult").numpy() for l in np.unique(np.concatenate((pred_label, gt_label)).astype(int)): pred_mask_l = pred_label == l pred_bbox_l = pred_bbox[pred_mask_l] pred_score_l = pred_score[pred_mask_l] # sort by score order = pred_score_l.argsort()[::-1] pred_bbox_l = pred_bbox_l[order] pred_score_l = pred_score_l[order] gt_mask_l = gt_label == l gt_bbox_l = gt_bbox[gt_mask_l] gt_difficult_l = gt_difficult[gt_mask_l] n_pos[l] += np.logical_not(gt_difficult_l).sum() score[l].extend(pred_score_l) if len(pred_bbox_l) == 0: continue if len(gt_bbox_l) == 0: match[l].extend((0,) * pred_bbox_l.shape[0]) continue # VOC evaluation follows integer typed bounding boxes. pred_bbox_l = pred_bbox_l.copy() pred_bbox_l[:, 2:] += 1 gt_bbox_l = gt_bbox_l.copy() gt_bbox_l[:, 2:] += 1 iou = boxlist_iou( BoxList(pred_bbox_l, gt_boxlist.size), BoxList(gt_bbox_l, gt_boxlist.size), ).numpy() gt_index = iou.argmax(axis=1) # set -1 if there is no matching ground truth gt_index[iou.max(axis=1) < iou_thresh] = -1 del iou selec = np.zeros(gt_bbox_l.shape[0], dtype=bool) for gt_idx in gt_index: if gt_idx >= 0: if gt_difficult_l[gt_idx]: match[l].append(-1) else: if not selec[gt_idx]: match[l].append(1) else: match[l].append(0) selec[gt_idx] = True else: match[l].append(0) n_fg_class = max(n_pos.keys()) + 1 prec = [None] * n_fg_class rec = [None] * n_fg_class for l in n_pos.keys(): score_l = np.array(score[l]) match_l = np.array(match[l], dtype=np.int8) order = score_l.argsort()[::-1] match_l = match_l[order] tp = np.cumsum(match_l == 1) fp = np.cumsum(match_l == 0) # If an element of fp + tp is 0, # the corresponding element of prec[l] is nan. prec[l] = tp / (fp + tp) # If n_pos[l] is 0, rec[l] is None. if n_pos[l] > 0: rec[l] = tp / n_pos[l] return prec, rec
def evaluate_box_proposals( predictions, dataset, thresholds=None, area="all", limit=None ): """Evaluate detection proposal recall metrics. This function is a much faster alternative to the official COCO API recall evaluation code. However, it produces slightly different results. """ # Record max overlap value for each gt box # Return vector of overlap values areas = { "all": 0, "small": 1, "medium": 2, "large": 3, "96-128": 4, "128-256": 5, "256-512": 6, "512-inf": 7, } area_ranges = [ [0 ** 2, 1e5 ** 2], # all [0 ** 2, 32 ** 2], # small [32 ** 2, 96 ** 2], # medium [96 ** 2, 1e5 ** 2], # large [96 ** 2, 128 ** 2], # 96-128 [128 ** 2, 256 ** 2], # 128-256 [256 ** 2, 512 ** 2], # 256-512 [512 ** 2, 1e5 ** 2], ] # 512-inf assert area in areas, "Unknown area range: {}".format(area) area_range = area_ranges[areas[area]] gt_overlaps = [] num_pos = 0 for image_id, prediction in enumerate(predictions): original_id = dataset.id_to_img_map[image_id] # TODO replace with get_img_info? image_width = dataset.coco.imgs[original_id]["width"] image_height = dataset.coco.imgs[original_id]["height"] prediction = prediction.resize((image_width, image_height)) # sort predictions in descending order # TODO maybe remove this and make it explicit in the documentation inds = prediction.get_field("objectness").sort(descending=True)[1] prediction = prediction[inds] ann_ids = dataset.coco.getAnnIds(imgIds=original_id) anno = dataset.coco.loadAnns(ann_ids) gt_boxes = [obj["bbox"] for obj in anno if obj["iscrowd"] == 0] gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes gt_boxes = BoxList(gt_boxes, (image_width, image_height), mode="xywh").convert( "xyxy" ) gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) if len(gt_boxes) == 0: continue valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) gt_boxes = gt_boxes[valid_gt_inds] num_pos += len(gt_boxes) if len(gt_boxes) == 0: continue if len(prediction) == 0: continue if limit is not None and len(prediction) > limit: prediction = prediction[:limit] overlaps = boxlist_iou(prediction, gt_boxes) _gt_overlaps = torch.zeros(len(gt_boxes)) for j in range(min(len(prediction), len(gt_boxes))): # find which proposal box maximally covers each gt box # and get the iou amount of coverage for each gt box max_overlaps, argmax_overlaps = overlaps.max(dim=0) # find which gt box is 'best' covered (i.e. 'best' = most iou) gt_ovr, gt_ind = max_overlaps.max(dim=0) assert gt_ovr >= 0 # find the proposal box that covers the best covered gt box box_ind = argmax_overlaps[gt_ind] # record the iou coverage of this gt box _gt_overlaps[j] = overlaps[box_ind, gt_ind] assert _gt_overlaps[j] == gt_ovr # mark the proposal box and the gt box as used overlaps[box_ind, :] = -1 overlaps[:, gt_ind] = -1 # append recorded iou coverage level gt_overlaps.append(_gt_overlaps) gt_overlaps = torch.cat(gt_overlaps, dim=0) gt_overlaps, _ = torch.sort(gt_overlaps) if thresholds is None: step = 0.05 thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) recalls = torch.zeros_like(thresholds) # compute recall for each iou threshold for i, t in enumerate(thresholds): recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) # ar = 2 * np.trapz(recalls, thresholds) ar = recalls.mean() return { "ar": ar, "recalls": recalls, "thresholds": thresholds, "gt_overlaps": gt_overlaps, "num_pos": num_pos, }