def eval_recall(dataset, predictions, image_ids, curr_iter, output_folder):
total_num = 0
recall_num = 0
recall_num_topN = 0
recall_num_det = 0
total_rel_softmax = 0
recall_rel_softmax = 0
for img_sent_id in image_ids:
result = predictions[img_sent_id]
if cfg.MODEL.VG.TWO_STAGE:
if cfg.MODEL.RELATION_ON and cfg.MODEL.RELATION.USE_RELATION_CONST:
gt_boxes, pred_boxes, pred_box_topN, pred_boxes_det, \
pred_sim, pred_sim_topN, pred_rel_sim, pred_rel_gt, batch_topN_boxes, batch_reg_offset_topN, batch_rel_score_mat = result
pred_rel_sim = pred_rel_sim.numpy()
pred_rel_gt = pred_rel_gt.numpy()
if pred_rel_gt.shape[0] != 0:
pred_rel_sim_argmax = pred_rel_sim.argmax(1)
pred_rel_gt = pred_rel_gt[np.arange(pred_rel_gt.shape[0]), pred_rel_sim_argmax]
total_rel_softmax += pred_rel_gt.shape[0]
recall_rel_softmax += (pred_rel_gt > 0).astype(np.float32).sum()
else:
gt_boxes, pred_boxes, pred_box_topN, pred_boxes_det, pred_sim = result
pred_box_topN = BoxList(pred_box_topN, gt_boxes.size, mode="xyxy")
pred_box_topN.clip_to_image()
ious_topN = boxlist_iou(gt_boxes, pred_box_topN)
ious_topN = ious_topN.cpu().numpy().diagonal()
recall_num_topN += int((ious_topN >= cfg.MODEL.VG.EVAL_THRESH).sum())
else:
gt_boxes, pred_boxes, pred_boxes_det, pred_sim = result
pred_boxes = BoxList(pred_boxes, gt_boxes.size, mode="xyxy")
pred_boxes.clip_to_image()
ious = boxlist_iou(gt_boxes, pred_boxes)
iou = ious.cpu().numpy().diagonal()
total_num += iou.shape[0]
recall_num += int((iou>=cfg.MODEL.VG.EVAL_THRESH).sum()) # 0.5
pred_boxes_det = BoxList(pred_boxes_det, gt_boxes.size, mode="xyxy")
pred_boxes_det.clip_to_image()
ious_det = boxlist_iou(gt_boxes, pred_boxes_det)
iou_det = ious_det.cpu().numpy().diagonal()
recall_num_det += int((iou_det>=cfg.MODEL.VG.EVAL_THRESH).sum()) # 0.5
acc = recall_num/total_num
acc_topN = recall_num_topN/total_num
acc_det = recall_num_det/total_num
acc_rel_softmax = recall_rel_softmax / (total_rel_softmax+1e-6)
return (acc, acc_topN, acc_det, acc_rel_softmax)
def detect_relsample(self, proposals, targets):
# corresponding to rel_assignments function in neural-motifs
"""
The input proposals are already processed by subsample function of box_head,
in this function, we should only care about fg box, and sample corresponding fg/bg relations
Note: this function keeps a state.
Arguments:
proposals (list[BoxList]) contain fields: labels, predict_logits
targets (list[BoxList]) contain fields: labels
"""
self.num_pos_per_img = int(self.batch_size_per_image *
self.positive_fraction)
rel_idx_pairs = []
rel_labels = []
rel_sym_binarys = []
for img_id, (proposal, target) in enumerate(zip(proposals, targets)):
device = proposal.bbox.device
prp_box = proposal.bbox
prp_lab = proposal.get_field("labels").long()
tgt_box = target.bbox
tgt_lab = target.get_field("labels").long()
tgt_rel_matrix = target.get_field("relation") # [tgt, tgt]
# IoU matching
ious = boxlist_iou(target, proposal) # [tgt, prp]
is_match = (tgt_lab[:, None] == prp_lab[None]) & (
ious > self.fg_thres) # [tgt, prp]
# Proposal self IoU to filter non-overlap
prp_self_iou = boxlist_iou(proposal, proposal) # [prp, prp]
if self.require_overlap and (not self.use_gt_box):
rel_possibility = (prp_self_iou > 0) & (
prp_self_iou < 1) # not self & intersect
else:
num_prp = prp_box.shape[0]
rel_possibility = torch.ones(
(num_prp, num_prp), device=device).long() - torch.eye(
num_prp, device=device).long()
# only select relations between fg proposals
rel_possibility[prp_lab == 0] = 0
rel_possibility[:, prp_lab == 0] = 0
img_rel_triplets, binary_rel = self.motif_rel_fg_bg_sampling(
device, tgt_rel_matrix, ious, is_match, rel_possibility)
rel_idx_pairs.append(
img_rel_triplets[:, :2]) # (num_rel, 2), (sub_idx, obj_idx)
rel_labels.append(img_rel_triplets[:, 2]) # (num_rel, )
rel_sym_binarys.append(binary_rel)
return proposals, rel_labels, rel_idx_pairs, rel_sym_binarys
def match_targets_to_anchors(self,
anchor,
target_left,
target_right,
copied_fields=[]):
# with torch.no_grad():
# bbox_new = target.convert("xyxy").bbox.clone().detach()
# bbox_new2 = target.convert("xyxy").bbox.clone().detach()
# disps = target.get_field("depths").convert("disp").depths
# bbox_new[:, 0] -= disps
# bbox_new2[:, 0] -= disps
# bbox_new2[:, 2] -= disps
# # print(bbox_new)
# target_union = BoxList(bbox_new, target.size, mode="xyxy")
# target_right = BoxList(bbox_new2, target.size, mode="xyxy")
# target_union.clip_to_image(remove_empty=True)
target_union = boxlist_union(target_left, target_right)
match_quality_matrix = boxlist_iou(target_union, anchor)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target_left = target_left.copy_with_fields(copied_fields)
target_right = target_right.copy_with_fields(copied_fields)
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target_left[matched_idxs.clamp(min=0)]
matched_targets_right = target_right[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
matched_targets_right.add_field("matched_idxs", matched_idxs)
return matched_targets, matched_targets_right
def match_targets_to_proposals(self, proposal, target, object3d):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
matched_object3d = object3d[matched_idxs.clamp(min=0)]
return matched_object3d, matched_idxs
def match_targets_to_anchors(self, anchor, target, copied_fields=[]):
if cfg.ROTATE:
match_quality_matrix = boxlist_riou(anchor, target)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
if "RETINANET" in cfg.MODEL.BACKBONE.CONV_BODY:
if cfg.MODEL.RETINANET_DCN_ON:
target = target.copy_with_fields(["xyxyr", "labels"])
else:
target = target.copy_with_fields(["xywht", "labels"])
else:
target = target.copy_with_fields(["xyxy"])
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
else:
match_quality_matrix = boxlist_iou(target, anchor)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target = target.copy_with_fields(copied_fields)
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
copied_fields = ['labels']
if target.has_field('tightness'):
copied_fields.append('tightness')
target = target.copy_with_fields(copied_fields)
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
if len(target) == 0:
dummy_bbox = torch.zeros((len(matched_idxs), 4),
dtype=torch.float32, device=matched_idxs.device)
from maskrcnn_benchmark.structures.bounding_box import BoxList
matched_targets = BoxList(dummy_bbox, target.size, target.mode)
matched_targets.add_field('labels', self.create_all_bkg_labels(
len(matched_idxs), matched_idxs.device))
matched_targets.add_field('tightness', torch.zeros(len(matched_idxs),
device=matched_idxs.device))
else:
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
if self.weight_box_loss:
matched_iou = torch.full((len(matched_idxs),), -1., device=matched_idxs.device)
matched_iou[matched_idxs >= 0] = match_quality_matrix[matched_idxs[matched_idxs >= 0],
matched_idxs >= 0]
matched_targets.add_field('matched_iou', matched_iou)
return matched_targets
def match_both_targets_to_proposals(self, proposal, target, posv_targets):
match_quality_matrix = boxlist_iou(target, proposal)
match_quality_matrix_iog = boxlist_iog(posv_targets, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
matched_with_visible_idxs = self.proposal_matcher_with_visible_iog(
match_quality_matrix_iog)
index_out_bounds = np.where(matched_idxs.cpu().numpy() == -2)[0]
matched_both = np.where((matched_idxs.cpu().numpy() > -1) & (
matched_with_visible_idxs.cpu().numpy() > -1))[0]
matched_both = torch.tensor(matched_both,
dtype=matched_idxs.dtype,
device=matched_idxs.device)
index_out_bounds = torch.tensor(index_out_bounds,
dtype=matched_idxs.dtype,
device=matched_idxs.device)
matched_new = -1 * torch.ones_like(
matched_idxs, dtype=matched_idxs.dtype) # background
matched_new[matched_both] = matched_idxs[
matched_both] # TODO: label is important
matched_new[index_out_bounds] = matched_idxs[
index_out_bounds] # out of bounds
# Fast RCNN only need "labels" field for selecting the targets
target = target.copy_with_fields("labels")
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_new.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_new)
return matched_targets
def match_targets_to_anchors(self, anchor, target, copied_fields=[]):
match_quality_matrix = boxlist_iou(target, anchor)
# ################# changed by hui ###################################################
if len(target.bbox) == 0:
matched_idxs = torch.LongTensor([-1] *
match_quality_matrix.shape[1])
else:
matched_idxs, _ = self.proposal_matcher(
match_quality_matrix) # ,_ add by G
# for anchor recall cal
# if self.debug:
# record_for_recall(matched_idxs, target)
#####################################################################################
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target = target.copy_with_fields(copied_fields)
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
target = target.copy_with_fields(["labels", "masks"])
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
if cfg.ROTATE:
if cfg.R2CNN:
match_quality_matrix = boxlist_riou(proposal, target)
else:
match_quality_matrix = boxlist_riou(proposal, target)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
target = target.copy_with_fields(
["labels", "xyxy", "xywht", "xywht1"])
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
else:
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
target = target.copy_with_fields("labels")
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target, is_source=True):
"""
这个函数主要是计算一下proposal以及gt的IOU，然后筛选一下
相当于是选择出来了每个proposal对应的gt
"""
# 计算基准边框与预测边框相互之间的IoU
match_quality_matrix = boxlist_iou(target, proposal)
# 计算各个预测边框对应的基准边框(ground truth box)的索引列表，背景边框为-2，模糊边框为-1
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
# 获得基准边框(groun truth box)附加的属性labels标签,即边框的具体类别
target = target.copy_with_fields("labels")
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
# 计算各个预测边框对应的基准边框(ground truth box)列表,所有背景边框以及模糊边框都对应成第一个gt
matched_targets = target[matched_idxs.clamp(min=0)]
# DA start
# 如果是目标域的数据的话
if not is_source:
matched_targets = target[matched_idxs]
# DA end
# 将对应的列表索引附加到对应基准边框列表中
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def ga_shape_target(self, square_anchors, approx_anchors, targets):
shape_targets = []
shape_weights = []
for square_anchors_per_image, approx_anchors_per_image, targets_per_image in zip(
square_anchors, approx_anchors, targets):
match_quality_matrix = boxlist_iou(targets_per_image,
approx_anchors_per_image)
num_gt = targets_per_image.bbox.shape[0]
match_quality_matrix = match_quality_matrix.view(
num_gt, -1, self.num_approx_anchors_per_location)
match_quality_matrix, _ = match_quality_matrix.max(dim=2)
matched_idxs = self.proposal_matcher(match_quality_matrix)
target = targets_per_image.copy_with_fields(self.copied_fields)
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
weights = self.generate_labels_func(matched_targets)
weights = weights.to(dtype=torch.float32)
bg_indices = matched_idxs == Matcher.BELOW_LOW_THRESHOLD
weights[bg_indices] = 0
#if "not_visibility" in self.discard_cases:
# weights[~square_anchors_per_image.get_field("visibility")] = 0
#if "between_thresholds" in self.discard_cases:
# inds_to_discard = matched_idxs == Matcher.BETWEEN_THRESHOLDS
# weights[inds_to_discard] = 0
shape_targets_per_image = matched_targets.bbox
shape_targets.append(shape_targets_per_image)
shape_weights.append(weights)
return shape_targets, shape_weights
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target[0], proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
target = copy.deepcopy(target[1])
#target = target[1].copy_with_fields(["labels", "left_box", "right_box"])
matched_targets = target[matched_idxs.clamp(min=0)]
return matched_targets, matched_idxs
def track_per_video(video_name):
print(video_name)
json_path = os.path.join(root, video_name+'.json')
with open(json_path, 'r') as f:
proposal_dict = json.load(f)
gt = None
frame_num = len(proposal_dict)
boxes = np.zeros((frame_num, 4)) # xywh
times = np.zeros(frame_num)
i = 0
for img_name, proposals_ in proposal_dict.items():
start_time = time.time()
if img_name == '00000001.jpg':
gt = proposals_[0][:-1]
boxes[0] = gt
times[0] = time.time() - start_time
gt = torch.Tensor(gt).reshape(1, 4)
gt = BoxList(gt, (-1,-1), mode="xywh").convert("xyxy")
i += 1
continue
proposals = [proposal[:-1] for proposal in proposals_]
scores = [proposal[-1] for proposal in proposals_]
scores = torch.Tensor(scores)
proposals = torch.Tensor(proposals)
proposals = BoxList(proposals, (-1,-1), mode="xywh").convert("xyxy")
proposals.add_field('objectness', scores)
'''对proposals执行nms，保留top_n个样本。'''
proposals_nms = boxlist_nms(
proposals,
0.1,
max_proposals=10,
score_field="objectness",
)
last_box = torch.Tensor(boxes[i - 1]).reshape(1, 4)
last_box = BoxList(last_box, (-1, -1), mode="xywh").convert("xyxy")
overlaps = boxlist_iou(proposals_nms, last_box).squeeze(0)
selected_id = torch.argmax(overlaps)
if overlaps[selected_id] == 0:
print('消失')
selected_id = torch.argmax(proposals_nms.extra_fields['objectness'])
proposals_nms = proposals_nms.convert("xywh")
res_box = proposals_nms.bbox[selected_id].cpu().numpy()
boxes[i] = res_box
# visualization(video_name, img_name, proposals_nms.bbox, res_box, boxes[i - 1])
times[i] = time.time() - start_time
i += 1
'''保存该帧跟踪结果'''
record_file = os.path.join(cfg.OUTPUT_DIR,
'result', video_name,
'%s_%03d.txt' % (video_name, 1))
record_dir = os.path.dirname(record_file)
if not os.path.isdir(record_dir):
os.makedirs(record_dir)
np.savetxt(record_file, boxes, fmt='%.3f', delimiter=',')
'''保存时间文件'''
time_file = record_file[:record_file.rfind('_')] + '_time.txt'
times = times[:, np.newaxis]
np.savetxt(time_file, times, fmt='%.8f', delimiter=',')
def compute_on_dataset(model, data_loader, device):
model.eval()
with_results_dict = {}
without_results_dict = {}
cpu_device = torch.device("cpu")
for i, batch in tqdm(enumerate(data_loader)):
images, targets, image_ids = batch
images = images.to(device)
with_overlaps = []
without_overlaps = []
with torch.no_grad():
with_outputs, without_outputs = model(images)
for output, target in zip(with_outputs, targets):
if len(target) == 0:
continue
if len(output) == 0:
overlap = torch.zeros(len(target),
device=cpu_device,
dtype=torch.float)
with_overlaps.append(overlap)
else:
overlap = boxlist_iou(output,
target.to(device)).max(dim=0).values
with_overlaps.append(overlap.to(cpu_device))
for output, target in zip(without_outputs, targets):
if len(target) == 0:
continue
if len(output) == 0:
overlap = torch.zeros(len(target),
device=cpu_device,
dtype=torch.float)
without_overlaps.append(overlap)
else:
overlap = boxlist_iou(output,
target.to(device)).max(dim=0).values
without_overlaps.append(overlap.to(cpu_device))
with_results_dict.update({
img_id: result
for img_id, result in zip(image_ids, with_overlaps)
})
without_results_dict.update({
img_id: result
for img_id, result in zip(image_ids, without_overlaps)
})
return with_results_dict, without_results_dict
def compute_average_recall(ground_truths, proposals):
match_quality_matrix = boxlist_iou(ground_truths, proposals)
highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1)
highest_quality_foreach_gt -= 0.5
AR = 2 * torch.mean(
torch.max(
highest_quality_foreach_gt,
torch.zeros(highest_quality_foreach_gt.size(),
device='cuda'))).item()
return AR
def match_targets_to_proposals(self, proposal, target):
# NOTE(H): Seems the code like to keep xyxy. Then let's keep xyxy all around!
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
target = target.copy_with_fields(["labels", "vps", "vp_vert", "masks"])
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_anchors_forFF(self, anchor, target, copied_fields=[]):
match_quality_matrix = boxlist_iou(target, anchor)
# ################# changed by G ###################################################
if len(target.bbox) == 0:
matches_gt_forFF = torch.LongTensor([0] *
match_quality_matrix.shape[1])
else:
_, matches_gt_forFF = self.proposal_matcher(match_quality_matrix)
#####################################################################################
return matches_gt_forFF
def match_targets_to_anchors(self, anchor, target):
match_quality_matrix = boxlist_iou(target, anchor)
matched_idxs = self.proposal_matcher(match_quality_matrix)
target = target.copy_with_fields(['labels'])
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Keypoint RCNN needs "labels" and "keypoints "fields for creating the targets
target = target.copy_with_fields(["labels", "bb8keypoints"])
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
target = target.copy_with_fields("labels")
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def bbox_overlaps(boxes1, boxes2):
"""
Parameters:
boxes1 (m, 4) [List or np.array] : bounding boxes of (x1,y1,x2,y2)
boxes2 (n, 4) [List or np.array] : bounding boxes of (x1,y1,x2,y2)
Return:
iou (m, n) [np.array]
"""
boxes1 = BoxList(boxes1, (0, 0), 'xyxy')
boxes2 = BoxList(boxes2, (0, 0), 'xyxy')
iou = boxlist_iou(boxes1, boxes2).cpu().numpy()
return iou
def get_tracking_result(last_box, proposals):
"""
last_box: xywh
proposals: BoxList, mode=xyxy
"""
last_box = torch.from_numpy(last_box).reshape(1, 4)
last_box = BoxList(last_box, proposals.size, mode="xywh").convert("xyxy")
'''计算IoU'''
overlaps = boxlist_iou(proposals, last_box)
proposals = proposals.convert("xywh")
res = proposals.bbox[torch.argmax(overlaps)].cpu().numpy()
return res
def get_gt_index(gt_lists, predictions, iou_thresh=0.5):
score = defaultdict(list)
match = defaultdict(list)
assert len(gt_lists) == len(
predictions), "Length of gt and pred lists need to be same."
gt_index_list = []
for gt_list, pred_list in zip(gt_lists, predictions):
pred_bbox = pred_list.get_field("left_box").bbox.numpy()
pred_label = pred_list.get_field("left_box").get_field(
"labels").numpy()
pred_score = pred_list.get_field("left_box").get_field(
"scores").numpy()
gt_bbox = gt_list.get_field("left_box").bbox.numpy()
gt_label = gt_list.get_field("labels").numpy()
gt_index_per_image = dict()
for l in np.unique(np.concatenate((pred_label, gt_label)).astype(int)):
pred_mask_l = pred_label == l
pred_bbox_l = pred_bbox[pred_mask_l]
pred_score_l = pred_score[pred_mask_l]
order = pred_score_l.argsort()[::-1]
pred_bbox_l = pred_bbox_l[order]
pred_score_l = pred_score_l[order]
gt_mask_l = gt_label == l
gt_bbox_l = gt_bbox[gt_mask_l]
score[l].extend(pred_score_l)
if len(pred_bbox_l) == 0 or len(gt_bbox_l) == 0:
continue
pred_bbox_l = pred_bbox_l.copy()
pred_bbox_l[:, 2:] += 1
gt_bbox_l = gt_bbox_l.copy()
gt_bbox_l[:, 2:] += 1
iou = boxlist_iou(
BoxList(pred_bbox_l,
gt_list.get_field("left_box").size),
BoxList(gt_bbox_l,
gt_list.get_field("left_box").size),
).numpy()
gt_index = iou.argmax(axis=1)
gt_index[iou.max(axis=1) < iou_thresh] = -1
del iou
gt_index_per_image[l] = [order, gt_index]
gt_index_list.append(gt_index_per_image)
return gt_index_list
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
# Added additional field
target = target.copy_with_fields(["labels", "fitz_categories"])
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_anchors(self, anchor, target, copied_fields=[]):
match_quality_matrix = boxlist_iou(target, anchor)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target = target.copy_with_fields(copied_fields)
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# print(match_quality_matrix, matched_idxs)
target = target.copy_with_fields(["labels", "depths"],
skip_missing=True)
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_anchors(self, anchor, target):
match_quality_matrix = boxlist_iou(target, anchor)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target = target.copy_with_fields([])
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(target, proposal)
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Mask RCNN needs "labels" and "masks "fields for creating the targets
target = target.copy_with_fields(
self.cfg['MODEL']['ROI_CAR_CLS_ROT_HEAD']['REGRESS_TARGET'])
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def match_targets_to_proposals(self, proposal, target):
match_quality_matrix = boxlist_iou(
target, proposal) # target和proposal都是xywh格式的吗
matched_idxs = self.proposal_matcher(match_quality_matrix)
# Fast RCNN only need "labels" field for selecting the targets
target = target.copy_with_fields("labels")
# get the targets corresponding GT for each proposal
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
# proposal.add_field("matched_idxs", matched_idxs) # 后加的
return matched_targets
def compute_objectpairs_iou(targets, proposals):
match_quality_matrix = boxlist_iou(targets, proposals)
objects_pairs = targets.get_field('objects_pairs')
match_s = match_quality_matrix[objects_pairs[:, 0], :].permute(1, 0)
match_o = match_quality_matrix[objects_pairs[:, 1], :].permute(1, 0)
match_s = match_s.unsqueeze(1).expand(match_s.size(0), match_s.size(0),
match_s.size(1))
match_o = match_o.unsqueeze(0).expand(match_o.size(0), match_o.size(0),
match_o.size(1))
match = match_s * match_o
match = match.view(match_s.size(0)**2, match_s.size(2)).permute(1, 0)
return match
def match_targets_to_anchors(self, anchor, target):
# print(len(anchor),len(target),'==============================match=====')
match_quality_matrix = boxlist_iou(target, anchor, type=1)
# type=1: matching the square GT bbox type=0: normal matching
matched_idxs = self.proposal_matcher(match_quality_matrix)
# RPN doesn't need any fields from target
# for creating the labels, so clear them all
target = target.copy_with_fields(['rotations'])
# get the targets corresponding GT for each anchor
# NB: need to clamp the indices because we can have a single
# GT in the image, and matched_idxs can be -2, which goes
# out of bounds
matched_targets = target[matched_idxs.clamp(min=0)]
matched_targets.add_field("matched_idxs", matched_idxs)
return matched_targets
def calc_detection_voc_prec_rec(gt_boxlists, pred_boxlists, iou_thresh=0.5):
"""Calculate precision and recall based on evaluation code of PASCAL VOC.
This function calculates precision and recall of
predicted bounding boxes obtained from a dataset which has :math:`N`
images.
The code is based on the evaluation code used in PASCAL VOC Challenge.
"""
n_pos = defaultdict(int)
score = defaultdict(list)
match = defaultdict(list)
for gt_boxlist, pred_boxlist in zip(gt_boxlists, pred_boxlists):
pred_bbox = pred_boxlist.bbox.numpy()
pred_label = pred_boxlist.get_field("labels").numpy()
pred_score = pred_boxlist.get_field("scores").numpy()
gt_bbox = gt_boxlist.bbox.numpy()
gt_label = gt_boxlist.get_field("labels").numpy()
gt_difficult = gt_boxlist.get_field("difficult").numpy()
for l in np.unique(np.concatenate((pred_label, gt_label)).astype(int)):
pred_mask_l = pred_label == l
pred_bbox_l = pred_bbox[pred_mask_l]
pred_score_l = pred_score[pred_mask_l]
# sort by score
order = pred_score_l.argsort()[::-1]
pred_bbox_l = pred_bbox_l[order]
pred_score_l = pred_score_l[order]
gt_mask_l = gt_label == l
gt_bbox_l = gt_bbox[gt_mask_l]
gt_difficult_l = gt_difficult[gt_mask_l]
n_pos[l] += np.logical_not(gt_difficult_l).sum()
score[l].extend(pred_score_l)
if len(pred_bbox_l) == 0:
continue
if len(gt_bbox_l) == 0:
match[l].extend((0,) * pred_bbox_l.shape[0])
continue
# VOC evaluation follows integer typed bounding boxes.
pred_bbox_l = pred_bbox_l.copy()
pred_bbox_l[:, 2:] += 1
gt_bbox_l = gt_bbox_l.copy()
gt_bbox_l[:, 2:] += 1
iou = boxlist_iou(
BoxList(pred_bbox_l, gt_boxlist.size),
BoxList(gt_bbox_l, gt_boxlist.size),
).numpy()
gt_index = iou.argmax(axis=1)
# set -1 if there is no matching ground truth
gt_index[iou.max(axis=1) < iou_thresh] = -1
del iou
selec = np.zeros(gt_bbox_l.shape[0], dtype=bool)
for gt_idx in gt_index:
if gt_idx >= 0:
if gt_difficult_l[gt_idx]:
match[l].append(-1)
else:
if not selec[gt_idx]:
match[l].append(1)
else:
match[l].append(0)
selec[gt_idx] = True
else:
match[l].append(0)
n_fg_class = max(n_pos.keys()) + 1
prec = [None] * n_fg_class
rec = [None] * n_fg_class
for l in n_pos.keys():
score_l = np.array(score[l])
match_l = np.array(match[l], dtype=np.int8)
order = score_l.argsort()[::-1]
match_l = match_l[order]
tp = np.cumsum(match_l == 1)
fp = np.cumsum(match_l == 0)
# If an element of fp + tp is 0,
# the corresponding element of prec[l] is nan.
prec[l] = tp / (fp + tp)
# If n_pos[l] is 0, rec[l] is None.
if n_pos[l] > 0:
rec[l] = tp / n_pos[l]
return prec, rec
def evaluate_box_proposals(
predictions, dataset, thresholds=None, area="all", limit=None
):
"""Evaluate detection proposal recall metrics. This function is a much
faster alternative to the official COCO API recall evaluation code. However,
it produces slightly different results.
"""
# Record max overlap value for each gt box
# Return vector of overlap values
areas = {
"all": 0,
"small": 1,
"medium": 2,
"large": 3,
"96-128": 4,
"128-256": 5,
"256-512": 6,
"512-inf": 7,
}
area_ranges = [
[0 ** 2, 1e5 ** 2], # all
[0 ** 2, 32 ** 2], # small
[32 ** 2, 96 ** 2], # medium
[96 ** 2, 1e5 ** 2], # large
[96 ** 2, 128 ** 2], # 96-128
[128 ** 2, 256 ** 2], # 128-256
[256 ** 2, 512 ** 2], # 256-512
[512 ** 2, 1e5 ** 2],
] # 512-inf
assert area in areas, "Unknown area range: {}".format(area)
area_range = area_ranges[areas[area]]
gt_overlaps = []
num_pos = 0
for image_id, prediction in enumerate(predictions):
original_id = dataset.id_to_img_map[image_id]
# TODO replace with get_img_info?
image_width = dataset.coco.imgs[original_id]["width"]
image_height = dataset.coco.imgs[original_id]["height"]
prediction = prediction.resize((image_width, image_height))
# sort predictions in descending order
# TODO maybe remove this and make it explicit in the documentation
inds = prediction.get_field("objectness").sort(descending=True)[1]
prediction = prediction[inds]
ann_ids = dataset.coco.getAnnIds(imgIds=original_id)
anno = dataset.coco.loadAnns(ann_ids)
gt_boxes = [obj["bbox"] for obj in anno if obj["iscrowd"] == 0]
gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes
gt_boxes = BoxList(gt_boxes, (image_width, image_height), mode="xywh").convert(
"xyxy"
)
gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0])
if len(gt_boxes) == 0:
continue
valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1])
gt_boxes = gt_boxes[valid_gt_inds]
num_pos += len(gt_boxes)
if len(gt_boxes) == 0:
continue
if len(prediction) == 0:
continue
if limit is not None and len(prediction) > limit:
prediction = prediction[:limit]
overlaps = boxlist_iou(prediction, gt_boxes)
_gt_overlaps = torch.zeros(len(gt_boxes))
for j in range(min(len(prediction), len(gt_boxes))):
# find which proposal box maximally covers each gt box
# and get the iou amount of coverage for each gt box
max_overlaps, argmax_overlaps = overlaps.max(dim=0)
# find which gt box is 'best' covered (i.e. 'best' = most iou)
gt_ovr, gt_ind = max_overlaps.max(dim=0)
assert gt_ovr >= 0
# find the proposal box that covers the best covered gt box
box_ind = argmax_overlaps[gt_ind]
# record the iou coverage of this gt box
_gt_overlaps[j] = overlaps[box_ind, gt_ind]
assert _gt_overlaps[j] == gt_ovr
# mark the proposal box and the gt box as used
overlaps[box_ind, :] = -1
overlaps[:, gt_ind] = -1
# append recorded iou coverage level
gt_overlaps.append(_gt_overlaps)
gt_overlaps = torch.cat(gt_overlaps, dim=0)
gt_overlaps, _ = torch.sort(gt_overlaps)
if thresholds is None:
step = 0.05
thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32)
recalls = torch.zeros_like(thresholds)
# compute recall for each iou threshold
for i, t in enumerate(thresholds):
recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos)
# ar = 2 * np.trapz(recalls, thresholds)
ar = recalls.mean()
return {
"ar": ar,
"recalls": recalls,
"thresholds": thresholds,
"gt_overlaps": gt_overlaps,
"num_pos": num_pos,
}