def sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes):
"""
generate random sample of ROIs comprising foreground and background examples
:param roidb: database of selected rois
:param fg_rois_per_image: foreground roi number
:param rois_per_image: total roi number
:param num_classes: number of classes
:return: (labels, rois, bbox_targets, bbox_inside_weights, overlaps)
"""
# label = class RoI has max overlap with
labels = roidb['max_classes']
overlaps = roidb['max_overlaps']
rois = roidb['boxes']
# foreground RoI with FG_THRESH overlap
fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0]
# guard against the case when an image has fewer than fg_rois_per_image foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size)
# Sample foreground regions without replacement
if fg_indexes.size > 0:
fg_indexes = npr.choice(fg_indexes,
size=fg_rois_per_this_image,
replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI)
& (overlaps >= config.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image,
bg_indexes.size)
# Sample foreground regions without replacement
if bg_indexes.size > 0:
bg_indexes = npr.choice(bg_indexes,
size=bg_rois_per_this_image,
replace=False)
# indexes selected
keep_indexes = np.append(fg_indexes, bg_indexes)
# pad more to ensure a fixed minibatch size
if keep_indexes.shape[0] < rois_per_image:
gap = rois_per_image - keep_indexes.shape[0]
gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False)
keep_indexes = np.append(keep_indexes, gap_indexes)
# select labels
labels = labels[keep_indexes]
# set labels of bg_rois to be 0
labels[fg_rois_per_this_image:] = 0
overlaps = overlaps[keep_indexes]
rois = rois[keep_indexes]
print "num class: ", num_classes
bbox_targets, bbox_inside_weights = \
expand_bbox_regression_targets(roidb['bbox_targets'][keep_indexes, :], num_classes)
return rois, labels, bbox_targets, bbox_inside_weights, overlaps
def sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes):
"""
generate random sample of ROIs comprising foreground and background examples
:param roidb: database of selected rois
:param fg_rois_per_image: foreground roi number
:param rois_per_image: total roi number
:param num_classes: number of classes
:return: (labels, rois, bbox_targets, bbox_inside_weights, overlaps)
"""
# label = class RoI has max overlap with
labels = roidb['max_classes']
overlaps = roidb['max_overlaps']
rois = roidb['boxes']
# foreground RoI with FG_THRESH overlap
fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0]
# guard against the case when an image has fewer than fg_rois_per_image foreground RoIs
fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size)
# Sample foreground regions without replacement
if fg_indexes.size > 0:
fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False)
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0]
# Compute number of background RoIs to take from this image (guarding against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image
bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size)
# Sample foreground regions without replacement
if bg_indexes.size > 0:
bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False)
# indexes selected
keep_indexes = np.append(fg_indexes, bg_indexes)
# pad more to ensure a fixed minibatch size
if keep_indexes.shape[0] < rois_per_image:
gap = rois_per_image - keep_indexes.shape[0]
gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False)
keep_indexes = np.append(keep_indexes, gap_indexes)
# select labels
labels = labels[keep_indexes]
# set labels of bg_rois to be 0
labels[fg_rois_per_this_image:] = 0
overlaps = overlaps[keep_indexes]
rois = rois[keep_indexes]
bbox_targets, bbox_inside_weights = \
expand_bbox_regression_targets(roidb['bbox_targets'][keep_indexes, :], num_classes)
return rois, labels, bbox_targets, bbox_inside_weights, overlaps
def visualize_bbox_regression(imdb, roidb, thresh=0.5):
"""
visualize the target of bounding box regression
:param imdb: the imdb to be visualized
:param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
['image', 'max_classes', 'max_overlaps', 'bbox_targets']
:param thresh: only bbox with overlap > thresh will be visualized
:return: None
"""
import matplotlib.pyplot as plt
import skimage.io
for i in range(len(roidb)):
roi_rec = roidb[i]
im_path = roi_rec['image']
im = skimage.io.imread(im_path)
ss_indexes = np.where(roi_rec['gt_classes'] == 0)[0]
gt_indexes = np.where(roi_rec['gt_classes'] != 0)[0]
for ss_bbox, ss_class, overlap, bb_target in \
zip(roi_rec['boxes'][ss_indexes], roi_rec['max_classes'][ss_indexes],
roi_rec['max_overlaps'][ss_indexes], roi_rec['bbox_targets'][ss_indexes]):
if overlap < thresh:
continue
plt.imshow(im)
for bbox, gt_class in zip(roi_rec['boxes'][gt_indexes], roi_rec['gt_classes'][gt_indexes]):
bbox = bbox.astype(np.float, copy=False)
box = plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1], fill=True, alpha=0.3, edgecolor='g',
facecolor='g', linewidth=3)
plt.gca().add_patch(box)
ss_bbox = ss_bbox.astype(np.float, copy=False)
ss_box = plt.Rectangle((ss_bbox[0], ss_bbox[1]),
ss_bbox[2] - ss_bbox[0],
ss_bbox[3] - ss_bbox[1], fill=False,
edgecolor='b', linewidth=3)
plt.gca().add_patch(ss_box)
plt.gca().text(ss_bbox[0], ss_bbox[1], imdb.classes[ss_class] + ' {}'.format(overlap), color='w')
bbox_target, _ = expand_bbox_regression_targets(bb_target[np.newaxis, :], imdb.num_classes)
print bbox_target[0][ss_class * 4:(ss_class + 1) * 4]
regressed_bbox = bbox_pred(ss_bbox[np.newaxis, :], bbox_target)[0]
regressed_bbox = regressed_bbox[ss_class * 4:(ss_class + 1) * 4]
regressed_box = plt.Rectangle((regressed_bbox[0], regressed_bbox[1]),
regressed_bbox[2] - regressed_bbox[0],
regressed_bbox[3] - regressed_bbox[1], fill=False,
edgecolor='r', linewidth=3)
plt.gca().add_patch(regressed_box)
plt.show()
def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes, key):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
# overlaps: (rois x gt_boxes)
overlaps = bbox_overlaps(
np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float),
np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float))
gt_assignment = overlaps.argmax(axis=1)
max_overlaps = overlaps.max(axis=1)
labels = gt_boxes[gt_assignment, 4]
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= config.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False)
if fg_inds.size < fg_rois_per_image:
fg_inds_ = npr.choice(fg_inds, size=fg_rois_per_image-fg_inds.size, replace=True)
fg_inds = np.hstack((fg_inds_, fg_inds))
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds_ = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI) &
(max_overlaps >= config.TRAIN.BG_THRESH_LO))[0]
if len(bg_inds_) == 0 and key == 'TRAIN':
bg_inds = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI+0.2) &
(max_overlaps >= 0))[0]
else:
bg_inds = bg_inds_
if len(bg_inds) == 0:
logging.log(logging.ERROR, "currently len(bg_inds) is zero")
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - len(fg_inds)
bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size)
# Sample background regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False)
if bg_inds.size < rois_per_image-fg_rois_per_image:
bg_inds_ = npr.choice(bg_inds, size=rois_per_image-fg_rois_per_image-bg_inds.size, replace=True)
bg_inds = np.hstack((bg_inds_, bg_inds))
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Select sampled values from various arrays:
labels = labels[keep_inds]
# Clamp labels for the background RoIs to 0
labels[fg_rois_per_this_image:] = 0
rois = all_rois[keep_inds]
bbox_target_data = _compute_targets(
rois[:, 1:5], gt_boxes[gt_assignment[keep_inds], :4], labels)
bbox_targets, bbox_inside_weights = \
expand_bbox_regression_targets(bbox_target_data, num_classes)
return labels, rois, bbox_targets, bbox_inside_weights
def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image,
num_classes, key):
"""Generate a random sample of RoIs comprising foreground and background
examples.
"""
# overlaps: (rois x gt_boxes)
overlaps = bbox_overlaps(
np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float),
np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float))
gt_assignment = overlaps.argmax(axis=1)
max_overlaps = overlaps.max(axis=1)
labels = gt_boxes[gt_assignment, 4]
# Select foreground RoIs as those with >= FG_THRESH overlap
fg_inds = np.where(max_overlaps >= config.TRAIN.FG_THRESH)[0]
# Guard against the case when an image has fewer than fg_rois_per_image
# foreground RoIs
fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size)
# Sample foreground regions without replacement
if fg_inds.size > 0:
fg_inds = npr.choice(fg_inds,
size=fg_rois_per_this_image,
replace=False)
if fg_inds.size < fg_rois_per_image:
fg_inds_ = npr.choice(fg_inds,
size=fg_rois_per_image - fg_inds.size,
replace=True)
fg_inds = np.hstack((fg_inds_, fg_inds))
# Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
bg_inds_ = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI)
& (max_overlaps >= config.TRAIN.BG_THRESH_LO))[0]
if len(bg_inds_) == 0 and key == 'TRAIN':
bg_inds = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI + 0.2)
& (max_overlaps >= 0))[0]
else:
bg_inds = bg_inds_
if len(bg_inds) == 0:
logging.log(logging.ERROR, "currently len(bg_inds) is zero")
# Compute number of background RoIs to take from this image (guarding
# against there being fewer than desired)
bg_rois_per_this_image = rois_per_image - len(fg_inds)
bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size)
# Sample background regions without replacement
if bg_inds.size > 0:
bg_inds = npr.choice(bg_inds,
size=bg_rois_per_this_image,
replace=False)
if bg_inds.size < rois_per_image - fg_rois_per_image:
bg_inds_ = npr.choice(bg_inds,
size=rois_per_image - fg_rois_per_image -
bg_inds.size,
replace=True)
bg_inds = np.hstack((bg_inds_, bg_inds))
# The indices that we're selecting (both fg and bg)
keep_inds = np.append(fg_inds, bg_inds)
# Select sampled values from various arrays:
labels = labels[keep_inds]
# Clamp labels for the background RoIs to 0
labels[fg_rois_per_this_image:] = 0
rois = all_rois[keep_inds]
bbox_target_data = _compute_targets(rois[:, 1:5],
gt_boxes[gt_assignment[keep_inds], :4],
labels)
bbox_targets, bbox_inside_weights = \
expand_bbox_regression_targets(bbox_target_data, num_classes)
return labels, rois, bbox_targets, bbox_inside_weights
def visualize_bbox_regression(imdb, roidb, thresh=0.5):
"""
visualize the target of bounding box regression
:param imdb: the imdb to be visualized
:param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped']
['image', 'max_classes', 'max_overlaps', 'bbox_targets']
:param thresh: only bbox with overlap > thresh will be visualized
:return: None
"""
import matplotlib.pyplot as plt
import skimage.io
for i in range(len(roidb)):
roi_rec = roidb[i]
im_path = roi_rec['image']
im = skimage.io.imread(im_path)
ss_indexes = np.where(roi_rec['gt_classes'] == 0)[0]
gt_indexes = np.where(roi_rec['gt_classes'] != 0)[0]
for ss_bbox, ss_class, overlap, bb_target in \
zip(roi_rec['boxes'][ss_indexes], roi_rec['max_classes'][ss_indexes],
roi_rec['max_overlaps'][ss_indexes], roi_rec['bbox_targets'][ss_indexes]):
if overlap < thresh:
continue
plt.imshow(im)
for bbox, gt_class in zip(roi_rec['boxes'][gt_indexes],
roi_rec['gt_classes'][gt_indexes]):
bbox = bbox.astype(np.float, copy=False)
box = plt.Rectangle((bbox[0], bbox[1]),
bbox[2] - bbox[0],
bbox[3] - bbox[1],
fill=True,
alpha=0.3,
edgecolor='g',
facecolor='g',
linewidth=3)
plt.gca().add_patch(box)
ss_bbox = ss_bbox.astype(np.float, copy=False)
ss_box = plt.Rectangle((ss_bbox[0], ss_bbox[1]),
ss_bbox[2] - ss_bbox[0],
ss_bbox[3] - ss_bbox[1],
fill=False,
edgecolor='b',
linewidth=3)
plt.gca().add_patch(ss_box)
plt.gca().text(ss_bbox[0],
ss_bbox[1],
imdb.classes[ss_class] + ' {}'.format(overlap),
color='w')
bbox_target, _ = expand_bbox_regression_targets(
bb_target[np.newaxis, :], imdb.num_classes)
print bbox_target[0][ss_class * 4:(ss_class + 1) * 4]
regressed_bbox = bbox_pred(ss_bbox[np.newaxis, :], bbox_target)[0]
regressed_bbox = regressed_bbox[ss_class * 4:(ss_class + 1) * 4]
regressed_box = plt.Rectangle(
(regressed_bbox[0], regressed_bbox[1]),
regressed_bbox[2] - regressed_bbox[0],
regressed_bbox[3] - regressed_bbox[1],
fill=False,
edgecolor='r',
linewidth=3)
plt.gca().add_patch(regressed_box)
plt.show()
