def get_anchor_labels(self, anchors, gt_boxes):
'''
label each anchor as fg/bg/ignore
:param anchors: Ax4
:param gt_boxes: Bx4
:return:
anchor_labels: (A,), int, each element is {-1, 0, 1}
anchor_boxes: Ax4, the target gt_box for each anchor when the anchor is fg
'''
def filter_box_label(labels, value, max_num):
curr_inds = np.where(labels == value)[0]
if len(curr_inds) > max_num:
disable_inds = np.random.choice(curr_inds, size=(len(curr_inds) - max_num), replace=False)
labels[disable_inds] = -1
curr_inds = np.where(labels == value)[0]
return curr_inds
NA, NB = len(anchors), len(gt_boxes)
if NB == 0:
anchor_labels = np.zeros((NA,), dtype=np.int32)
filter_box_label(anchor_labels, 0, max_num=cfg.FRCNN.RPN.BATCH_PER_IM)
return anchor_labels, np.zeros((NA, 4), dtype=np.float32)
box_ious = np_iou(anchors, gt_boxes) # NAxNB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]
anchor_labels = -np.ones((NA, ), dtype=np.int32)
anchor_labels[anchors_with_max_iou_per_gt] = 1
anchor_labels[ious_max_per_anchor >= cfg.FRCNN.RPN.POSITIVE_ANCHOR_THRESH] = 1
anchor_labels[ious_max_per_anchor < cfg.FRCNN.RPN.NEGATIVE_ANCHOR_THRESH] = 0
# label all non-ignore candidate boxes which overlap crowd as ignore
# if crowd_boxes.size > 0:
# cand_inds = np.where(anchor_labels >= 0)[0]
# cand_anchors = anchors[cand_inds]
# ioas = np_ioa(crowd_boxes, cand_anchors)
# overlap_with_crowd = cand_inds[ioas.max(axis=0) > cfg.FRCNN.RPN.CROWD_OVERLAP_THRESH]
# anchor_labels[overlap_with_crowd] = -1
# subsample fg labels: ignore some fg if fg is too many
target_sum_fg = int(cfg.FRCNN.RPN.BATCH_PER_IM * cfg.FRCNN.RPN.FG_RATIO)
fg_inds = filter_box_label(anchor_labels, 1, target_sum_fg)
# subsample bg labels: num_bg is not allowed to be too many
old_num_bg = np.sum(anchor_labels == 0)
if old_num_bg == 0:
raise BaseException
target_num_bg = cfg.FRCNN.RPN.BATCH_PER_IM - len(fg_inds)
filter_box_label(anchor_labels, 0, target_num_bg)
anchor_boxes = np.zeros((NA, 4), dtype=np.float32)
fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
anchor_boxes[fg_inds, :] = fg_boxes
return anchor_labels, anchor_boxes
def draw_proposal_recall(img, proposals, proposal_scores, gt_boxes):
"""
Draw top3 proposals for each gt.
Args:
proposals: NPx4
proposal_scores: NP
gt_boxes: NG
"""
box_ious = np_iou(gt_boxes, proposals) # ng x np
box_ious_argsort = np.argsort(-box_ious, axis=1)
good_proposals_ind = box_ious_argsort[:, :3] # for each gt, find 3 best proposals
good_proposals_ind = np.unique(good_proposals_ind.ravel())
proposals = proposals[good_proposals_ind, :]
tags = list(map(str, proposal_scores[good_proposals_ind]))
img = viz.draw_boxes(img, proposals, tags)
return img, good_proposals_ind
def draw_proposal_recall(img, proposals, proposal_scores, gt_boxes):
"""
Draw top3 proposals for each gt.
Args:
proposals: NPx4
proposal_scores: NP
gt_boxes: NG
"""
box_ious = np_iou(gt_boxes, proposals) # ng x np
box_ious_argsort = np.argsort(-box_ious, axis=1)
good_proposals_ind = box_ious_argsort[:, :3] # for each gt, find 3 best proposals
good_proposals_ind = np.unique(good_proposals_ind.ravel())
proposals = proposals[good_proposals_ind, :]
tags = list(map(str, proposal_scores[good_proposals_ind]))
img = viz.draw_boxes(img, proposals, tags)
return img, good_proposals_ind
def get_retinanet_anchor_labels(self, anchors, gt_boxes, gt_labels):
'''
label each anchor as fg/bg/ignore
:param anchors: Ax4
:param gt_boxes: Bx4
:param gt_labels: Bx1
:return:
anchor_labels: (A,), int, each element is {-1, 0, C} C is in [1, NUM_CATEGORY+1]
anchor_boxes: Ax4, the target gt_box for each anchor when the anchor is fg
'''
NA, NB = len(anchors), len(gt_boxes)
if NB == 0:
anchor_labels = np.zeros((NA,), dtype=np.int32)
return anchor_labels, np.zeros((NA, 4), dtype=np.float32)
box_ious = np_iou(anchors, gt_boxes) # NAxNB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]
anchor_labels = -np.ones((NA, ), dtype=np.int32)
anchor_labels[anchors_with_max_iou_per_gt] = 1
anchor_labels[ious_max_per_anchor >= cfg.RETINANET.POSITIVE_ANCHOR_THRESH] = 1
anchor_labels[ious_max_per_anchor < cfg.RETINANET.NEGATIVE_ANCHOR_THRESH] = 0
# label all non-ignore candidate boxes which overlap crowd as ignore
# if crowd_boxes.size > 0:
# cand_inds = np.where(anchor_labels >= 0)[0]
# cand_anchors = anchors[cand_inds]
# ioas = np_ioa(crowd_boxes, cand_anchors)
# overlap_with_crowd = cand_inds[ioas.max(axis=0) > cfg.FRCNN.RPN.CROWD_OVERLAP_THRESH]
# anchor_labels[overlap_with_crowd] = -1
fg_inds = np.where(anchor_labels == 1)[0]
anchor_boxes = np.zeros((NA, 4), dtype=np.float32)
fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
fg_labels = gt_labels[ious_argmax_per_anchor[fg_inds]].reshape(-1)
anchor_boxes[fg_inds, :] = fg_boxes
anchor_labels[fg_inds] = fg_labels # [0, 80]
return anchor_labels, anchor_boxes
def cal_avg_iou_per_gt(gt_boxes, sizes, ratios, imageHW=768):
def filter_box_inside(im, boxes):
h, w = im.shape[:2]
indices = np.where(
(boxes[:, 0] >= 0) &
(boxes[:, 1] >= 0) &
(boxes[:, 2] <= w) &
(boxes[:, 3] <= h))[0]
return indices
im = np.zeros((imageHW, imageHW))
ALL_ANCHORS = get_all_anchors(sizes=sizes, ratios=ratios)
H, W = im.shape[:2]
featureH, featureW = H // 16, W // 16
featuremap_anchors = ALL_ANCHORS[:featureH, :featureW, :, :]
featuremap_anchors_flatten = featuremap_anchors.reshape((-1, 4))
inside_ind = filter_box_inside(im, featuremap_anchors_flatten)
inside_anchors = featuremap_anchors_flatten[inside_ind, :]
anchors = inside_anchors
box_ious = np_iou(anchors, gt_boxes) # NA x NB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA,
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
return ious_max_per_gt.mean()
def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
"""
Label each anchor as fg/bg/ignore.
Args:
anchors: Ax4 float
gt_boxes: Bx4 float
crowd_boxes: Cx4 float
Returns:
anchor_labels: (A,) int. Each element is {-1, 0, 1}
anchor_boxes: Ax4. Contains the target gt_box for each anchor when the anchor is fg.
"""
# This function will modify labels and return the filtered inds
def filter_box_label(labels, value, max_num):
curr_inds = np.where(labels == value)[0]
if len(curr_inds) > max_num:
disable_inds = np.random.choice(curr_inds,
size=(len(curr_inds) - max_num),
replace=False)
labels[disable_inds] = -1 # ignore them
curr_inds = np.where(labels == value)[0]
return curr_inds
NA, NB = len(anchors), len(gt_boxes)
assert NB > 0 # empty images should have been filtered already
box_ious = np_iou(anchors, gt_boxes) # NA x NB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA,
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
# for each gt, find all those anchors (including ties) that has the max ious with it
anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]
# Setting NA labels: 1--fg 0--bg -1--ignore
anchor_labels = -np.ones((NA, ), dtype='int32') # NA,
# the order of setting neg/pos labels matter
anchor_labels[anchors_with_max_iou_per_gt] = 1
anchor_labels[ious_max_per_anchor >= cfg.RPN.POSITIVE_ANCHOR_THRESH] = 1
anchor_labels[ious_max_per_anchor < cfg.RPN.NEGATIVE_ANCHOR_THRESH] = 0
# We can label all non-ignore candidate boxes which overlap crowd as ignore
# But detectron did not do this.
# if crowd_boxes.size > 0:
# cand_inds = np.where(anchor_labels >= 0)[0]
# cand_anchors = anchors[cand_inds]
# ious = np_iou(cand_anchors, crowd_boxes)
# overlap_with_crowd = cand_inds[ious.max(axis=1) > cfg.RPN.CROWD_OVERLAP_THRES]
# anchor_labels[overlap_with_crowd] = -1
# Subsample fg labels: ignore some fg if fg is too many
target_num_fg = int(cfg.RPN.BATCH_PER_IM * cfg.RPN.FG_RATIO)
fg_inds = filter_box_label(anchor_labels, 1, target_num_fg)
# Keep an image even if there is no foreground anchors
# if len(fg_inds) == 0:
# raise MalformedData("No valid foreground for RPN!")
# Subsample bg labels. num_bg is not allowed to be too many
old_num_bg = np.sum(anchor_labels == 0)
if old_num_bg == 0:
# No valid bg in this image, skip.
raise MalformedData("No valid background for RPN!")
target_num_bg = cfg.RPN.BATCH_PER_IM - len(fg_inds)
filter_box_label(anchor_labels, 0, target_num_bg) # ignore return values
# Set anchor boxes: the best gt_box for each fg anchor
anchor_boxes = np.zeros((NA, 4), dtype='float32')
fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
anchor_boxes[fg_inds, :] = fg_boxes
# assert len(fg_inds) + np.sum(anchor_labels == 0) == cfg.RPN.BATCH_PER_IM
return anchor_labels, anchor_boxes
def get_anchor_labels(anchors, gt_boxes, crowd_boxes):
"""
Label each anchor as fg/bg/ignore.
Args:
anchors: Ax4 float
gt_boxes: Bx4 float
crowd_boxes: Cx4 float
Returns:
anchor_labels: (A,) int. Each element is {-1, 0, 1}
anchor_boxes: Ax4. Contains the target gt_box for each anchor when the anchor is fg.
"""
# This function will modify labels and return the filtered inds
def filter_box_label(labels, value, max_num):
curr_inds = np.where(labels == value)[0]
if len(curr_inds) > max_num:
disable_inds = np.random.choice(
curr_inds, size=(len(curr_inds) - max_num),
replace=False)
labels[disable_inds] = -1 # ignore them
curr_inds = np.where(labels == value)[0]
return curr_inds
NA, NB = len(anchors), len(gt_boxes)
assert NB > 0 # empty images should have been filtered already
box_ious = np_iou(anchors, gt_boxes) # NA x NB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA,
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
# for each gt, find all those anchors (including ties) that has the max ious with it
anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]
# Setting NA labels: 1--fg 0--bg -1--ignore
anchor_labels = -np.ones((NA,), dtype='int32') # NA,
# the order of setting neg/pos labels matter
anchor_labels[anchors_with_max_iou_per_gt] = 1
anchor_labels[ious_max_per_anchor >= config.POSITIVE_ANCHOR_THRES] = 1
anchor_labels[ious_max_per_anchor < config.NEGATIVE_ANCHOR_THRES] = 0
# First label all non-ignore candidate boxes which overlap crowd as ignore
if crowd_boxes.size > 0:
cand_inds = np.where(anchor_labels >= 0)[0]
cand_anchors = anchors[cand_inds]
ious = np_iou(cand_anchors, crowd_boxes)
overlap_with_crowd = cand_inds[ious.max(axis=1) > config.CROWD_OVERLAP_THRES]
anchor_labels[overlap_with_crowd] = -1
# Filter fg labels: ignore some fg if fg is too many
target_num_fg = int(config.RPN_BATCH_PER_IM * config.RPN_FG_RATIO)
fg_inds = filter_box_label(anchor_labels, 1, target_num_fg)
# Note that fg could be fewer than the target ratio
# filter bg labels. num_bg is not allowed to be too many
old_num_bg = np.sum(anchor_labels == 0)
if old_num_bg == 0 or len(fg_inds) == 0:
# No valid bg/fg in this image, skip.
# This can happen if, e.g. the image has large crowd.
raise MalformedData("No valid foreground/background for RPN!")
target_num_bg = config.RPN_BATCH_PER_IM - len(fg_inds)
filter_box_label(anchor_labels, 0, target_num_bg) # ignore return values
# Set anchor boxes: the best gt_box for each fg anchor
anchor_boxes = np.zeros((NA, 4), dtype='float32')
fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
anchor_boxes[fg_inds, :] = fg_boxes
return anchor_labels, anchor_boxes
def get_anchor_labels_ignore(anchors, gt_boxes, crowd_boxes, ignore_boxes):
"""
Label each anchor as fg/bg/ignore.
Args:
anchors: Ax4 float
gt_boxes: Bx4 float
crowd_boxes: Cx4 float
ignore_boxes: Dx4 int
Returns:
anchor_labels: (A,) int. Each element is {-1, 0, 1}
anchor_boxes: Ax4. Contains the target gt_box for each anchor when the anchor is fg.
"""
# This function will modify labels and return the filtered inds
def filter_box_label(labels, value, max_num):
curr_inds = np.where(labels == value)[0]
if len(curr_inds) > max_num:
disable_inds = np.random.choice(curr_inds,
size=(len(curr_inds) - max_num),
replace=False)
labels[disable_inds] = -1 # ignore them
curr_inds = np.where(labels == value)[0]
return curr_inds
NA, NB = len(anchors), len(gt_boxes)
assert NB > 0 # empty images should have been filtered already
box_ious = np_iou(anchors, gt_boxes) # NA x NB
ious_argmax_per_anchor = box_ious.argmax(axis=1) # NA,
ious_max_per_anchor = box_ious.max(axis=1)
ious_max_per_gt = np.amax(box_ious, axis=0, keepdims=True) # 1xNB
# for each gt, find all those anchors (including ties) that has the max ious with it
anchors_with_max_iou_per_gt = np.where(box_ious == ious_max_per_gt)[0]
# Setting NA labels: 1--fg 0--bg -1--ignore
anchor_labels = -np.ones((NA, ), dtype='int32') # NA,
# the order of setting neg/pos labels matter
anchor_labels[anchors_with_max_iou_per_gt] = 1
anchor_labels[ious_max_per_anchor >= cfg.RPN.POSITIVE_ANCHOR_THRESH] = 1
anchor_labels[ious_max_per_anchor < cfg.RPN.NEGATIVE_ANCHOR_THRESH] = 0
# We can label all non-ignore candidate boxes which overlap crowd as ignore
# But detectron did not do this.
# if crowd_boxes.size > 0:
# cand_inds = np.where(anchor_labels >= 0)[0]
# cand_anchors = anchors[cand_inds]
# ious = np_iou(cand_anchors, crowd_boxes)
# overlap_with_crowd = cand_inds[ious.max(axis=1) > cfg.RPN.CROWD_OVERLAP_THRES]
# anchor_labels[overlap_with_crowd] = -1
def to_xyxy(box):
box = np.array(box)
box[:, 2] += box[:, 0]
box[:, 3] += box[:, 1]
return box
if len(ignore_boxes) > 0:
ignores_xyxy = to_xyxy(ignore_boxes)
box_left = anchors[:, 0].reshape(1, len(anchors))
box_top = anchors[:, 1].reshape(1, len(anchors))
box_right = anchors[:, 2].reshape(1, len(anchors))
box_bottom = anchors[:, 3].reshape(1, len(anchors))
box_area = (box_right - box_left) * (box_bottom - box_top)
ignore_left = ignores_xyxy[:, 0].reshape(len(ignores_xyxy), 1)
ignore_top = ignores_xyxy[:, 1].reshape(len(ignores_xyxy), 1)
ignore_right = ignores_xyxy[:, 2].reshape(len(ignores_xyxy), 1)
ignore_bottom = ignores_xyxy[:, 3].reshape(len(ignores_xyxy), 1)
left = np.maximum(box_left, ignore_left)
top = np.maximum(box_top, ignore_top)
right = np.minimum(box_right, ignore_right)
bottom = np.minimum(box_bottom, ignore_bottom)
overlap = np.logical_and(left < right, top < bottom)
overlap_area = (right - left) * (bottom - top)
overlap_ratio = overlap_area / box_area
overlap_pos = np.logical_and(overlap_ratio > 0.5, overlap)
ignored = np.any(overlap_pos, axis=0)
anchor_labels[ignored == True] = -1
# def area(box):
# xmin, ymin, xmax, ymax = box
# return (xmax - xmin) * (yma
# xmin, ymin, xmax, ymax = box
# return (xmax - xmin) * (ymax - ymin)
# for i in range(len(anchors)):
# for ignore in ignores_xyxy:
# top_left = [
# max(anchors[i][0], ignore[0]),
# max(anchors[i][1], ignore[1])
# ]
# bottom_right = [
# min(anchors[i][2], ignore[2]),
# min(anchors[i][3], ignore[3])
# ]
# if np.all(top_left<bottom_right):
# inter_area = area([*top_left, *bottom_right])
# anchor_area = area(anchors[i])
# if inter_area/anchor_area > 0.5:
# anchor_labels[i] = -1
# Subsample fg labels: ignore some fg if fg is too many
target_num_fg = int(cfg.RPN.BATCH_PER_IM * cfg.RPN.FG_RATIO)
fg_inds = filter_box_label(anchor_labels, 1, target_num_fg)
# Keep an image even if there is no foreground anchors
# if len(fg_inds) == 0:
# raise MalformedData("No valid foreground for RPN!")
# Subsample bg labels. num_bg is not allowed to be too many
old_num_bg = np.sum(anchor_labels == 0)
if old_num_bg == 0:
# No valid bg in this image, skip.
raise MalformedData("No valid background for RPN!")
target_num_bg = cfg.RPN.BATCH_PER_IM - len(fg_inds)
filter_box_label(anchor_labels, 0, target_num_bg) # ignore return values
# Set anchor boxes: the best gt_box for each fg anchor
anchor_boxes = np.zeros((NA, 4), dtype='float32')
fg_boxes = gt_boxes[ious_argmax_per_anchor[fg_inds], :]
anchor_boxes[fg_inds, :] = fg_boxes
# assert len(fg_inds) + np.sum(anchor_labels == 0) == cfg.RPN.BATCH_PER_IM
return anchor_labels, anchor_boxes