def proposal_target_layer(rpn_bbox, rpn_cls_prob, gt_boxes, num_classes):
confidence_scores = rpn_cls_prob[:, 1]
# Add ground truth boxes as part of proposals
rpn_bbox = np.vstack([rpn_bbox, gt_boxes[:, 0:-1]])
confidence_scores = np.concatenate(confidence_scores,
np.ones(gt_boxes.shape[0], np.float32))
# Sample objects and backgrounds
fg_cnt = int(BATCH_SIZE * FG_RATIO)
fg_idxs = np.where(confidence_scores >= 0.5)[0]
if len(fg_idxs) > fg_cnt:
pos_inds = np.random.choice(fg_idxs, size=fg_cnt, replace=False)
bg_cnt = BATCH_SIZE - len(pos_inds)
bg_idxs = np.where((confidence_scores >= 0.1)
& (confidence_scores < 0.5))[0]
if len(bg_idxs) > bg_cnt:
neg_inds = np.random.choice(bg_idxs, size=bg_cnt, replace=False)
pos_bbox = rpn_bbox[pos_inds]
overlaps = bbox.bbox_overlaps(pos_bbox, gt_boxes[:, 0:-1])
argmax_overlaps = np.argmax(overlaps, axis=-1)
pos_labels = gt_boxes[:, -1][argmax_overlaps] + 1
neg_labels = np.zeros(len(neg_inds), np.int32)
labels = np.concatenate(pos_labels, neg_labels)
bbox_reg = np.zeros([len(labels), (num_classes + 1) * 4], np.float32)
bbox_reg_ = bbox.bbox_transform(rpn_bbox,
gt_boxes[argmax_overlaps][:, :-1])
for i in range(len(pos_labels)):
bbox_reg[i, pos_labels[i] * 4:(pos_labels[i] + 1) * 4] = bbox_reg_[i]
neg_bbox = rpn_bbox[neg_inds]
rpn_bbox = np.vstack([pos_bbox, neg_bbox])
return labels, bbox_reg, rpn_bbox
def anchor_target_layer(gt_boxes, all_anchors, image_shape, feature_map_shape, k):
"""
:param gt_boxes:
:param all_anchors:
:param image_shape:
:param feature_map_shape:
:param k:
:return:
"""
# If there is no object in the image
if len(gt_boxes) == 0:
labels = np.zeros((len(all_anchors),), dtype=np.int32)
targets = np.zeros(all_anchors.shape, dtype=np.float32)
return labels, targets
num_total_anchors = all_anchors.shape[0]
# Keep anchors that inside the image
valid_idx = np.where((all_anchors[:, 0] >= 0) &
(all_anchors[:, 1] >= 0) &
(all_anchors[:, 2] < image_shape[1]) &
(all_anchors[:, 3] < image_shape[0]))[0]
anchors = all_anchors[valid_idx, :]
labels = np.empty((len(valid_idx),), dtype=np.int32)
labels.fill(-1)
overlaps = bbox.bbox_overlaps(anchors, gt_boxes)
argmax_overlaps = np.argmax(overlaps, axis=1)
max_overlaps = overlaps[np.arange(0, len(valid_idx), 1), argmax_overlaps]
gt_argmax_overlaps = np.argmax(overlaps, axis=0)
gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(0, gt_boxes.shape[0], 1)]
gt_max_overlaps = np.where(overlaps == gt_max_overlaps)[0]
labels[np.where(max_overlaps < BG_LOW_THRES)[0]] = 0
labels[gt_max_overlaps] = 1
labels[np.where(max_overlaps > FG_HIGH_THRES)[0]] = 1
targets = bbox.bbox_transform(anchors, gt_boxes[argmax_overlaps, :])
# Sampling positive and negative anchors
fg_cnt = int(SAMPLE_NUMBER * FG_RATIO)
fg_idxs = np.where(labels == 1)[0]
if len(fg_idxs) > fg_cnt:
disable_inds = np.random.choice(
fg_idxs, size=(len(fg_idxs) - fg_cnt), replace=False)
labels[disable_inds] = -1
bg_cnt = SAMPLE_NUMBER - np.sum(labels == 1)
bg_idxs = np.where(labels == 0)[0]
if len(bg_idxs) > bg_cnt:
disable_inds = np.random.choice(
bg_idxs, size=(len(bg_idxs) - bg_cnt), replace=False)
labels[disable_inds] = -1
labels = _unmap(labels, num_total_anchors, valid_idx, -1)
targets = _unmap(targets, num_total_anchors, valid_idx, 0)
return labels, targets
def worker(self, data):
im_info, cur_crop, im_scale, nids, gtids, gt_boxes, boxes, classes = data[0:8]
has_mask = True if len(data) > 8 else False
anchors = self.all_anchors.copy()
inds_inside = np.where((anchors[:, 0] >= -32) &
(anchors[:, 1] >= -32) &
(anchors[:, 2] < im_info[0] + 32) &
(anchors[:, 3] < im_info[1] + 32))[0]
anchors = anchors[inds_inside, :]
labels = np.empty((len(inds_inside),), dtype=np.float32)
labels.fill(-1)
total_anchors = int(self.K * self.num_anchors)
gt_boxes[:, 0] -= cur_crop[0]
gt_boxes[:, 2] -= cur_crop[0]
gt_boxes[:, 1] -= cur_crop[1]
gt_boxes[:, 3] -= cur_crop[1]
vgt_boxes = boxes[np.intersect1d(gtids, nids)]
vgt_boxes[:, 0] -= cur_crop[0]
vgt_boxes[:, 2] -= cur_crop[0]
vgt_boxes[:, 1] -= cur_crop[1]
vgt_boxes[:, 3] -= cur_crop[1]
gt_boxes = clip_boxes(np.round(gt_boxes * im_scale), im_info[:2])
vgt_boxes = clip_boxes(np.round(vgt_boxes * im_scale), im_info[:2])
ids = filter_boxes(gt_boxes, 10)
if len(ids) == 0:
gt_boxes = np.zeros((0, 4))
classes = np.zeros((0, 1))
if has_mask:
mask_polys = data[8]
# Shift and crop the mask polygons
mask_polys = crop_polys(mask_polys, cur_crop, im_scale)
# Create the padded encoded array
if len(ids) > 0:
polylen = len(mask_polys)
tmask_polys = []
tgt_boxes = []
tclasses = []
for i in ids:
if i < polylen:
tmask_polys.append(mask_polys[i])
tgt_boxes.append(gt_boxes[i])
tclasses.append(classes[i])
if len(gt_boxes) > 0:
gt_boxes = np.array(tgt_boxes)
classes = np.array(tclasses).reshape(len(tclasses), 1)
mask_polys = tmask_polys
else:
gt_boxes = np.zeros((0, 4))
classes = np.zeros((0, 1))
encoded_polys = poly_encoder(mask_polys, classes[:, 0] - 1,
max_poly_len=self.max_poly_len, max_n_gts=self.max_n_gts)
else:
encoded_polys = -np.ones((self.max_n_gts, self.max_poly_len), dtype=np.float32)
else:
if len(ids) > 0:
gt_boxes = gt_boxes[ids]
classes = classes[ids]
agt_boxes = gt_boxes.copy()
ids = filter_boxes(vgt_boxes, 10)
if len(ids) > 0:
vgt_boxes = vgt_boxes[ids]
else:
vgt_boxes = np.zeros((0, 4))
if len(vgt_boxes) > 0:
ov = bbox_overlaps(np.ascontiguousarray(gt_boxes).astype(float),
np.ascontiguousarray(vgt_boxes).astype(float))
mov = np.max(ov, axis=1)
else:
mov = np.zeros((len(gt_boxes)))
invalid_gtids = np.where(mov < 1)[0]
valid_gtids = np.where(mov == 1)[0]
invalid_boxes = gt_boxes[invalid_gtids, :]
gt_boxes = gt_boxes[valid_gtids, :]
def _unmap(data, count, inds, fill=0):
"""" unmap a subset inds of data into original data of size count """
if len(data.shape) == 1:
ret = np.empty((count,), dtype=np.float32)
ret.fill(fill)
ret[inds] = data
else:
ret = np.empty((count,) + data.shape[1:], dtype=np.float32)
ret.fill(fill)
ret[inds, :] = data
return ret
if gt_boxes.size > 0:
# overlap between the anchors and the gt boxes
# overlaps (ex, gt)
overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float))
if invalid_boxes is not None:
if len(invalid_boxes) > 0:
overlapsn = bbox_overlaps(anchors.astype(np.float), invalid_boxes.astype(np.float))
argmax_overlapsn = overlapsn.argmax(axis=1)
max_overlapsn = overlapsn[np.arange(len(inds_inside)), argmax_overlapsn]
argmax_overlaps = overlaps.argmax(axis=1)
max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]
gt_argmax_overlaps = overlaps.argmax(axis=0)
gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])]
gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0]
labels[max_overlaps < self.neg_thresh] = 0
labels[gt_argmax_overlaps] = 1
# fg label: above threshold IoU
labels[max_overlaps >= self.pos_thresh] = 1
if invalid_boxes is not None:
if len(invalid_boxes) > 0:
labels[max_overlapsn > 0.3] = -1
else:
labels[:] = 0
if len(invalid_boxes) > 0:
overlapsn = bbox_overlaps(anchors.astype(np.float), invalid_boxes.astype(np.float))
argmax_overlapsn = overlapsn.argmax(axis=1)
max_overlapsn = overlapsn[np.arange(len(inds_inside)), argmax_overlapsn]
if len(invalid_boxes) > 0:
labels[max_overlapsn > 0.3] = -1
# subsample positive labels if we have too many
fg_inds = np.where(labels == 1)[0]
if len(fg_inds) > self.num_fg:
disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - self.num_fg), replace=False)
labels[disable_inds] = -1
# subsample negative labels if we have too many
num_bg = self.batch_size - np.sum(labels == 1)
bg_inds = np.where(labels == 0)[0]
if len(bg_inds) > num_bg:
disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False)
labels[disable_inds] = -1
bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)
if gt_boxes.size > 0:
bbox_targets[:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :4])
bbox_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
bbox_weights[labels == 1, :] = np.array([1.0, 1.0, 1.0, 1.0])
# map up to original set of anchors
labels = _unmap(labels, total_anchors, inds_inside, fill=-1)
bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)
bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0)
labels = labels.reshape((1, self.feat_height, self.feat_width, self.num_anchors)).transpose(0, 3, 1, 2)
labels = labels.reshape((1, self.num_anchors * self.feat_height * self.feat_width)).astype(np.float16)
bbox_targets = bbox_targets.reshape((self.feat_height, self.feat_width, self.num_anchors * 4)).transpose(2, 0, 1)
bbox_weights = bbox_weights.reshape((self.feat_height, self.feat_width, self.num_anchors * 4)).transpose((2, 0, 1))
pids = np.where(bbox_weights == 1)
bbox_targets = bbox_targets[pids]
fgt_boxes = -np.ones((100, 5))
if len(agt_boxes) > 0:
fgt_boxes[:min(len(agt_boxes), 100), :] = np.hstack((agt_boxes, classes))
rval = [mx.nd.array(labels, dtype='float16'), bbox_targets, mx.nd.array(pids), mx.nd.array(fgt_boxes)]
if has_mask:
rval.append(mx.nd.array(encoded_polys))
return rval
def worker(self, data):
im_info, cur_crop, im_scale, nids, gtids, gt_boxes, boxes, classes = data[0:8]
has_mask = True if len(data) > 8 else False
anchors = self.all_anchors.copy()
inds_inside = np.where((anchors[:, 0] >= -32) &
(anchors[:, 1] >= -32) &
(anchors[:, 2] < im_info[0] + 32) &
(anchors[:, 3] < im_info[1] + 32))[0]
anchors = anchors[inds_inside, :]
labels = np.empty((len(inds_inside),), dtype=np.float32)
labels.fill(-1)
total_anchors = int(self.K * self.num_anchors)
gt_boxes[:, 0] -= cur_crop[0]
gt_boxes[:, 2] -= cur_crop[0]
gt_boxes[:, 1] -= cur_crop[1]
gt_boxes[:, 3] -= cur_crop[1]
vgt_boxes = boxes[np.intersect1d(gtids, nids)]
vgt_boxes[:, 0] -= cur_crop[0]
vgt_boxes[:, 2] -= cur_crop[0]
vgt_boxes[:, 1] -= cur_crop[1]
vgt_boxes[:, 3] -= cur_crop[1]
gt_boxes = clip_boxes(np.round(gt_boxes * im_scale), im_info[:2])
vgt_boxes = clip_boxes(np.round(vgt_boxes * im_scale), im_info[:2])
ids = filter_boxes(gt_boxes, 10)
if len(ids) == 0:
gt_boxes = np.zeros((0, 4))
classes = np.zeros((0, 1))
if has_mask:
mask_polys = data[8]
# Shift and crop the mask polygons
mask_polys = crop_polys(mask_polys, cur_crop, im_scale)
# Create the padded encoded array
if len(ids) > 0:
polylen = len(mask_polys)
tmask_polys = []
tgt_boxes = []
tclasses = []
for i in ids:
if i < polylen:
tmask_polys.append(mask_polys[i])
tgt_boxes.append(gt_boxes[i])
tclasses.append(classes[i])
if len(gt_boxes) > 0:
gt_boxes = np.array(tgt_boxes)
classes = np.array(tclasses).reshape(len(tclasses), 1)
mask_polys = tmask_polys
else:
gt_boxes = np.zeros((0, 4))
classes = np.zeros((0, 1))
encoded_polys = poly_encoder(mask_polys, classes[:, 0] - 1,
max_poly_len=self.max_poly_len, max_n_gts=self.max_n_gts)
else:
encoded_polys = -np.ones((self.max_n_gts, self.max_poly_len), dtype=np.float32)
else:
if len(ids) > 0:
gt_boxes = gt_boxes[ids]
classes = classes[ids]
agt_boxes = gt_boxes.copy()
ids = filter_boxes(vgt_boxes, 10)
if len(ids) > 0:
vgt_boxes = vgt_boxes[ids]
else:
vgt_boxes = np.zeros((0, 4))
if len(vgt_boxes) > 0:
ov = bbox_overlaps(np.ascontiguousarray(gt_boxes).astype(float),
np.ascontiguousarray(vgt_boxes).astype(float))
mov = np.max(ov, axis=1)
else:
mov = np.zeros((len(gt_boxes)))
invalid_gtids = np.where(mov < 1)[0]
valid_gtids = np.where(mov == 1)[0]
invalid_boxes = gt_boxes[invalid_gtids, :]
gt_boxes = gt_boxes[valid_gtids, :]
def _unmap(data, count, inds, fill=0):
"""" unmap a subset inds of data into original data of size count """
if len(data.shape) == 1:
ret = np.empty((count,), dtype=np.float32)
ret.fill(fill)
ret[inds] = data
else:
ret = np.empty((count,) + data.shape[1:], dtype=np.float32)
ret.fill(fill)
ret[inds, :] = data
return ret
if gt_boxes.size > 0:
# overlap between the anchors and the gt boxes
# overlaps (ex, gt)
overlaps = bbox_overlaps(anchors.astype(np.float), gt_boxes.astype(np.float))
if invalid_boxes is not None:
if len(invalid_boxes) > 0:
overlapsn = bbox_overlaps(anchors.astype(np.float), invalid_boxes.astype(np.float))
argmax_overlapsn = overlapsn.argmax(axis=1)
max_overlapsn = overlapsn[np.arange(len(inds_inside)), argmax_overlapsn]
argmax_overlaps = overlaps.argmax(axis=1)
max_overlaps = overlaps[np.arange(len(inds_inside)), argmax_overlaps]
gt_argmax_overlaps = overlaps.argmax(axis=0)
gt_max_overlaps = overlaps[gt_argmax_overlaps, np.arange(overlaps.shape[1])]
gt_argmax_overlaps = np.where(overlaps == gt_max_overlaps)[0]
labels[max_overlaps < self.neg_thresh] = 0
labels[gt_argmax_overlaps] = 1
# fg label: above threshold IoU
labels[max_overlaps >= self.pos_thresh] = 1
if invalid_boxes is not None:
if len(invalid_boxes) > 0:
labels[max_overlapsn > 0.3] = -1
else:
labels[:] = 0
if len(invalid_boxes) > 0:
overlapsn = bbox_overlaps(anchors.astype(np.float), invalid_boxes.astype(np.float))
argmax_overlapsn = overlapsn.argmax(axis=1)
max_overlapsn = overlapsn[np.arange(len(inds_inside)), argmax_overlapsn]
if len(invalid_boxes) > 0:
labels[max_overlapsn > 0.3] = -1
# subsample positive labels if we have too many
fg_inds = np.where(labels == 1)[0]
if len(fg_inds) > self.num_fg:
disable_inds = npr.choice(fg_inds, size=(len(fg_inds) - self.num_fg), replace=False)
labels[disable_inds] = -1
# subsample negative labels if we have too many
num_bg = self.batch_size - np.sum(labels == 1)
bg_inds = np.where(labels == 0)[0]
if len(bg_inds) > num_bg:
disable_inds = npr.choice(bg_inds, size=(len(bg_inds) - num_bg), replace=False)
labels[disable_inds] = -1
bbox_targets = np.zeros((len(inds_inside), 4), dtype=np.float32)
if gt_boxes.size > 0:
bbox_targets[:] = bbox_transform(anchors, gt_boxes[argmax_overlaps, :4])
bbox_weights = np.zeros((len(inds_inside), 4), dtype=np.float32)
bbox_weights[labels == 1, :] = np.array([1.0, 1.0, 1.0, 1.0])
# map up to original set of anchors
labels = _unmap(labels, total_anchors, inds_inside, fill=-1)
bbox_targets = _unmap(bbox_targets, total_anchors, inds_inside, fill=0)
bbox_weights = _unmap(bbox_weights, total_anchors, inds_inside, fill=0)
labels = labels.reshape((1, self.feat_height, self.feat_width, self.num_anchors)).transpose(0, 3, 1, 2)
labels = labels.reshape((1, self.num_anchors * self.feat_height * self.feat_width)).astype(np.float16)
bbox_targets = bbox_targets.reshape((self.feat_height, self.feat_width, self.num_anchors * 4)).transpose(2, 0, 1)
bbox_weights = bbox_weights.reshape((self.feat_height, self.feat_width, self.num_anchors * 4)).transpose((2, 0, 1))
pids = np.where(bbox_weights == 1)
bbox_targets = bbox_targets[pids]
fgt_boxes = -np.ones((100, 5))
if len(agt_boxes) > 0:
fgt_boxes[:min(len(agt_boxes), 100), :] = np.hstack((agt_boxes, classes))
rval = [mx.nd.array(labels, dtype='float16'), bbox_targets, mx.nd.array(pids), mx.nd.array(fgt_boxes)]
if has_mask:
rval.append(mx.nd.array(encoded_polys))
return rval