def sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes): """ generate random sample of ROIs comprising foreground and background examples :param roidb: database of selected rois :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :return: (labels, rois, bbox_targets, bbox_inside_weights, overlaps) """ # label = class RoI has max overlap with labels = roidb['max_classes'] overlaps = roidb['max_overlaps'] rois = roidb['boxes'] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if fg_indexes.size > 0: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if bg_indexes.size > 0: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size if keep_indexes.shape[0] < rois_per_image: gap = rois_per_image - keep_indexes.shape[0] gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 overlaps = overlaps[keep_indexes] rois = rois[keep_indexes] print "num class: ", num_classes bbox_targets, bbox_inside_weights = \ expand_bbox_regression_targets(roidb['bbox_targets'][keep_indexes, :], num_classes) return rois, labels, bbox_targets, bbox_inside_weights, overlaps
def sample_rois(roidb, fg_rois_per_image, rois_per_image, num_classes): """ generate random sample of ROIs comprising foreground and background examples :param roidb: database of selected rois :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :return: (labels, rois, bbox_targets, bbox_inside_weights, overlaps) """ # label = class RoI has max overlap with labels = roidb['max_classes'] overlaps = roidb['max_overlaps'] rois = roidb['boxes'] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= config.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if fg_indexes.size > 0: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < config.TRAIN.BG_THRESH_HI) & (overlaps >= config.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if bg_indexes.size > 0: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size if keep_indexes.shape[0] < rois_per_image: gap = rois_per_image - keep_indexes.shape[0] gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 overlaps = overlaps[keep_indexes] rois = rois[keep_indexes] bbox_targets, bbox_inside_weights = \ expand_bbox_regression_targets(roidb['bbox_targets'][keep_indexes, :], num_classes) return rois, labels, bbox_targets, bbox_inside_weights, overlaps
def visualize_bbox_regression(imdb, roidb, thresh=0.5): """ visualize the target of bounding box regression :param imdb: the imdb to be visualized :param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] ['image', 'max_classes', 'max_overlaps', 'bbox_targets'] :param thresh: only bbox with overlap > thresh will be visualized :return: None """ import matplotlib.pyplot as plt import skimage.io for i in range(len(roidb)): roi_rec = roidb[i] im_path = roi_rec['image'] im = skimage.io.imread(im_path) ss_indexes = np.where(roi_rec['gt_classes'] == 0)[0] gt_indexes = np.where(roi_rec['gt_classes'] != 0)[0] for ss_bbox, ss_class, overlap, bb_target in \ zip(roi_rec['boxes'][ss_indexes], roi_rec['max_classes'][ss_indexes], roi_rec['max_overlaps'][ss_indexes], roi_rec['bbox_targets'][ss_indexes]): if overlap < thresh: continue plt.imshow(im) for bbox, gt_class in zip(roi_rec['boxes'][gt_indexes], roi_rec['gt_classes'][gt_indexes]): bbox = bbox.astype(np.float, copy=False) box = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=True, alpha=0.3, edgecolor='g', facecolor='g', linewidth=3) plt.gca().add_patch(box) ss_bbox = ss_bbox.astype(np.float, copy=False) ss_box = plt.Rectangle((ss_bbox[0], ss_bbox[1]), ss_bbox[2] - ss_bbox[0], ss_bbox[3] - ss_bbox[1], fill=False, edgecolor='b', linewidth=3) plt.gca().add_patch(ss_box) plt.gca().text(ss_bbox[0], ss_bbox[1], imdb.classes[ss_class] + ' {}'.format(overlap), color='w') bbox_target, _ = expand_bbox_regression_targets(bb_target[np.newaxis, :], imdb.num_classes) print bbox_target[0][ss_class * 4:(ss_class + 1) * 4] regressed_bbox = bbox_pred(ss_bbox[np.newaxis, :], bbox_target)[0] regressed_bbox = regressed_bbox[ss_class * 4:(ss_class + 1) * 4] regressed_box = plt.Rectangle((regressed_bbox[0], regressed_bbox[1]), regressed_bbox[2] - regressed_bbox[0], regressed_bbox[3] - regressed_bbox[1], fill=False, edgecolor='r', linewidth=3) plt.gca().add_patch(regressed_box) plt.show()
def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes, key): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps = bbox_overlaps( np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float), np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float)) gt_assignment = overlaps.argmax(axis=1) max_overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= config.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) if fg_inds.size < fg_rois_per_image: fg_inds_ = npr.choice(fg_inds, size=fg_rois_per_image-fg_inds.size, replace=True) fg_inds = np.hstack((fg_inds_, fg_inds)) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds_ = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI) & (max_overlaps >= config.TRAIN.BG_THRESH_LO))[0] if len(bg_inds_) == 0 and key == 'TRAIN': bg_inds = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI+0.2) & (max_overlaps >= 0))[0] else: bg_inds = bg_inds_ if len(bg_inds) == 0: logging.log(logging.ERROR, "currently len(bg_inds) is zero") # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - len(fg_inds) bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size) # Sample background regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) if bg_inds.size < rois_per_image-fg_rois_per_image: bg_inds_ = npr.choice(bg_inds, size=rois_per_image-fg_rois_per_image-bg_inds.size, replace=True) bg_inds = np.hstack((bg_inds_, bg_inds)) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Select sampled values from various arrays: labels = labels[keep_inds] # Clamp labels for the background RoIs to 0 labels[fg_rois_per_this_image:] = 0 rois = all_rois[keep_inds] bbox_target_data = _compute_targets( rois[:, 1:5], gt_boxes[gt_assignment[keep_inds], :4], labels) bbox_targets, bbox_inside_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes) return labels, rois, bbox_targets, bbox_inside_weights
def _sample_rois(all_rois, gt_boxes, fg_rois_per_image, rois_per_image, num_classes, key): """Generate a random sample of RoIs comprising foreground and background examples. """ # overlaps: (rois x gt_boxes) overlaps = bbox_overlaps( np.ascontiguousarray(all_rois[:, 1:5], dtype=np.float), np.ascontiguousarray(gt_boxes[:, :4], dtype=np.float)) gt_assignment = overlaps.argmax(axis=1) max_overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # Select foreground RoIs as those with >= FG_THRESH overlap fg_inds = np.where(max_overlaps >= config.TRAIN.FG_THRESH)[0] # Guard against the case when an image has fewer than fg_rois_per_image # foreground RoIs fg_rois_per_this_image = min(fg_rois_per_image, fg_inds.size) # Sample foreground regions without replacement if fg_inds.size > 0: fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False) if fg_inds.size < fg_rois_per_image: fg_inds_ = npr.choice(fg_inds, size=fg_rois_per_image - fg_inds.size, replace=True) fg_inds = np.hstack((fg_inds_, fg_inds)) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_inds_ = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI) & (max_overlaps >= config.TRAIN.BG_THRESH_LO))[0] if len(bg_inds_) == 0 and key == 'TRAIN': bg_inds = np.where((max_overlaps < config.TRAIN.BG_THRESH_HI + 0.2) & (max_overlaps >= 0))[0] else: bg_inds = bg_inds_ if len(bg_inds) == 0: logging.log(logging.ERROR, "currently len(bg_inds) is zero") # Compute number of background RoIs to take from this image (guarding # against there being fewer than desired) bg_rois_per_this_image = rois_per_image - len(fg_inds) bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size) # Sample background regions without replacement if bg_inds.size > 0: bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False) if bg_inds.size < rois_per_image - fg_rois_per_image: bg_inds_ = npr.choice(bg_inds, size=rois_per_image - fg_rois_per_image - bg_inds.size, replace=True) bg_inds = np.hstack((bg_inds_, bg_inds)) # The indices that we're selecting (both fg and bg) keep_inds = np.append(fg_inds, bg_inds) # Select sampled values from various arrays: labels = labels[keep_inds] # Clamp labels for the background RoIs to 0 labels[fg_rois_per_this_image:] = 0 rois = all_rois[keep_inds] bbox_target_data = _compute_targets(rois[:, 1:5], gt_boxes[gt_assignment[keep_inds], :4], labels) bbox_targets, bbox_inside_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes) return labels, rois, bbox_targets, bbox_inside_weights
def visualize_bbox_regression(imdb, roidb, thresh=0.5): """ visualize the target of bounding box regression :param imdb: the imdb to be visualized :param roidb: [image_index]['boxes', 'gt_classes', 'gt_overlaps', 'flipped'] ['image', 'max_classes', 'max_overlaps', 'bbox_targets'] :param thresh: only bbox with overlap > thresh will be visualized :return: None """ import matplotlib.pyplot as plt import skimage.io for i in range(len(roidb)): roi_rec = roidb[i] im_path = roi_rec['image'] im = skimage.io.imread(im_path) ss_indexes = np.where(roi_rec['gt_classes'] == 0)[0] gt_indexes = np.where(roi_rec['gt_classes'] != 0)[0] for ss_bbox, ss_class, overlap, bb_target in \ zip(roi_rec['boxes'][ss_indexes], roi_rec['max_classes'][ss_indexes], roi_rec['max_overlaps'][ss_indexes], roi_rec['bbox_targets'][ss_indexes]): if overlap < thresh: continue plt.imshow(im) for bbox, gt_class in zip(roi_rec['boxes'][gt_indexes], roi_rec['gt_classes'][gt_indexes]): bbox = bbox.astype(np.float, copy=False) box = plt.Rectangle((bbox[0], bbox[1]), bbox[2] - bbox[0], bbox[3] - bbox[1], fill=True, alpha=0.3, edgecolor='g', facecolor='g', linewidth=3) plt.gca().add_patch(box) ss_bbox = ss_bbox.astype(np.float, copy=False) ss_box = plt.Rectangle((ss_bbox[0], ss_bbox[1]), ss_bbox[2] - ss_bbox[0], ss_bbox[3] - ss_bbox[1], fill=False, edgecolor='b', linewidth=3) plt.gca().add_patch(ss_box) plt.gca().text(ss_bbox[0], ss_bbox[1], imdb.classes[ss_class] + ' {}'.format(overlap), color='w') bbox_target, _ = expand_bbox_regression_targets( bb_target[np.newaxis, :], imdb.num_classes) print bbox_target[0][ss_class * 4:(ss_class + 1) * 4] regressed_bbox = bbox_pred(ss_bbox[np.newaxis, :], bbox_target)[0] regressed_bbox = regressed_bbox[ss_class * 4:(ss_class + 1) * 4] regressed_box = plt.Rectangle( (regressed_bbox[0], regressed_bbox[1]), regressed_bbox[2] - regressed_bbox[0], regressed_bbox[3] - regressed_bbox[1], fill=False, edgecolor='r', linewidth=3) plt.gca().add_patch(regressed_box) plt.show()