def sample_rois_v2(rois, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # set labels of bg_rois to be 0 bg_ind = np.where(overlaps < cfg.TRAIN.BG_THRESH_HI)[0] labels[bg_ind] = 0 # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment, :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, bbox_targets, bbox_weights
def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, bbox_targets, bbox_weights
def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] ''' #yangyk print('gt_boxes:',gt_boxes[:,4]) print('gt_assignment:',gt_assignment) print('labels:',labels) print('rois shape:',rois.shape,'overlaps shape:',overlaps.shape,'labels shape',labels.shape) ''' # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) debug = False if debug: #yangyk print('fg_indexes size:', fg_indexes.size, 'fg_rois_per_image:', fg_rois_per_image, 'fg_rois_per_this_image:', fg_rois_per_this_image) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) #print('fg_over_laps:', overlaps[fg_indexes]) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] #yangyk labels_all = labels.copy() # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] #print('labels:',labels) # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) #yangyk overlaps = overlaps[keep_indexes] #print('fg_over_laps:', overlaps[:fg_rois_per_this_image]) neg_low = 0.0 neg_middle = 0.2 neg_high = 0.3 neg_indexes_L1 = np.where((overlaps < neg_middle) & (overlaps >= neg_low))[0] neg_indexes_L2 = np.where((overlaps < neg_high) & (overlaps >= neg_middle))[0] neg_indexes_L3 = np.where(overlaps >= neg_high)[0] neg_labels = np.zeros(labels.shape) #print(neg_indexes_L2) neg_labels[neg_indexes_L2] = labels_all[neg_indexes_L2] if debug: print('neg_indexes_L1:', len(neg_indexes_L1), 'neg_indexes_L2:', len(neg_indexes_L2), 'neg_indexes_L3', len(neg_indexes_L3)) print('labels_all:', labels_all) print('neg_labels:', neg_labels, 'neg_labels_shape:', neg_labels.shape) #print(neg_labels[neg_indexes_L2]) print('<<<fg neg labels>>>>', neg_labels[neg_indexes_L2]) print('fg neg labels sum', np.sum(neg_labels[neg_indexes_L2])) print('neg labels sum', np.sum(neg_labels)) print('over_laps:', overlaps) print('neg_fg_over_laps:', overlaps[neg_indexes_L2]) print('<<<<<<<<<<<<<<<<<<<<<<<<<>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, neg_labels, bbox_targets, bbox_weights
def sample_rois(self, rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None, gt_masks=None): if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] if cfg.TRAIN.IGNORE_GAP: keep_inds = remove_repetition(rois[fg_indexes, 1:]) fg_indexes = fg_indexes[keep_inds] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = np.random.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] if cfg.TRAIN.IGNORE_GAP: keep_inds = remove_repetition(rois[bg_indexes, 1:]) bg_indexes = bg_indexes[keep_inds] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = np.random.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) if cfg.TRAIN.GAP_SELECT_FROM_ALL: gap_indexes = np.random.choice(range(len(rois)), size=gap, replace=False) else: bg_full_indexes = list(set(range(len(rois))) - set(fg_indexes)) gap_indexes = np.random.choice(bg_full_indexes, size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) if cfg.TRAIN.IGNORE_GAP: valid_rois_per_this_image = fg_rois_per_this_image + bg_rois_per_this_image labels[valid_rois_per_this_image:] = -1 bbox_weights[valid_rois_per_this_image:] = 0 # masks # debug_gt_image_buffer = cv2.imread('debug_im_buffer.jpg') mask_reg_targets = -np.ones( (len(keep_indexes), 1, self._mask_size, self._mask_size)) for idx, obj in enumerate(fg_indexes): gt_roi = np.round(gt_boxes[gt_assignment[obj], :-1]).astype(int) ex_roi = np.round(rois[idx, 1:]).astype(int) gt_mask = gt_masks[gt_assignment[obj]] mask_reg_target = intersect_box_mask(ex_roi, gt_roi, gt_mask) mask_reg_target = cv2.resize(mask_reg_target.astype(np.float), (self._mask_size, self._mask_size)) mask_reg_target = mask_reg_target >= self._binary_thresh mask_reg_targets[idx, ...] = mask_reg_target return rois, labels, bbox_targets, bbox_weights, mask_reg_targets
def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None, gt_kps=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :param gt_kps: optional for e2e [n, num_kps*3] (x1, y1, v1, ...) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps = bbox_overlaps(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) overlaps = overlaps.max(axis=1) labels = gt_boxes[gt_assignment, 4] # foreground RoI with FG_THRESH overlap fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) res = {'rois_output': rois, 'label' : labels, 'bbox_target': bbox_targets, 'bbox_weight': bbox_weights, } if gt_kps is not None: keep_kps = gt_kps[gt_assignment[keep_indexes]] n_keep = keep_kps.shape[0] K = cfg.dataset.NUM_KEYPOINTS assert gt_kps.shape[1] == K*3 G = cfg.network.KEYPOINTS_POOLED_SIZE kps_labels = np.empty([n_keep, K], dtype=np.float32) kps_labels.fill(-1) kps_targets = np.zeros([n_keep, K, G, G, 2], dtype=np.float32) kps_weights = kps_targets.copy() num_fg = fg_indexes.size assert num_fg > 0, 'need at least one roi' # assgin kp targets fg_kps_label, fg_kps_target, fg_kps_weight = assign_keypoints(rois[:num_fg, 1:], keep_kps[:num_fg], pooled_size=G) kps_labels[:num_fg] = fg_kps_label kps_targets[:num_fg] = fg_kps_target normalizer = 1.0 / (num_fg + 1e-3) kps_weights[:num_fg] = fg_kps_weight * normalizer res['kps_label'] = kps_labels.reshape([-1]) res['kps_target'] = kps_targets.transpose([0,1,4,2,3]).reshape([n_keep, -1, G, G]) res['kps_weight'] = kps_weights.transpose([0,1,4,2,3]).reshape([n_keep, -1, G, G]) return res
def sample_rois(rois, fg_rois_per_image, rois_per_image, num_classes, cfg, labels=None, overlaps=None, bbox_targets=None, gt_boxes=None): """ generate random sample of ROIs comprising foreground and background examples :param rois: all_rois [n, 4]; e2e: [n, 5] with batch_index :param fg_rois_per_image: foreground roi number :param rois_per_image: total roi number :param num_classes: number of classes :param labels: maybe precomputed :param overlaps: maybe precomputed (max_overlaps) :param bbox_targets: maybe precomputed :param gt_boxes: optional for e2e [n, 5] (x1, y1, x2, y2, cls) :return: (labels, rois, bbox_targets, bbox_weights) """ if labels is None: overlaps,overlaps1,overlaps2,tboxcenter_ins = bbox_overlaps_py1(rois[:, 1:].astype(np.float), gt_boxes[:, :4].astype(np.float)) gt_assignment = overlaps.argmax(axis=1) boxcenter_ins=np.zeros(gt_assignment.shape[0]) for i in range(gt_assignment.shape[0]): boxcenter_ins[i]=tboxcenter_ins[i,gt_assignment[i]] overlaps = overlaps.max(axis=1) overlaps1 = overlaps1.max(axis=1) overlaps2 = overlaps2.max(axis=1) labels = gt_boxes[gt_assignment, 4] #print labels #print gt_boxes #print gt_assignment # foreground RoI with FG_THRESH overlap #print "gt_boxes:"+str(gt_boxes) new_order = np.argsort(overlaps) if DEBUG: print "overlaps:"+str(overlaps[new_order[-100:]]) print "overlaps1:"+str(overlaps1[new_order[-100:]]) print "overlaps2:"+str(overlaps2[new_order[-100:]]) print "boxcenter_ins"+str(boxcenter_ins[new_order[-100:]]) fg_indexes = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0] if DEBUG: print "fg_indexes:"+str(fg_indexes) for i in range(len(overlaps)): if overlaps[i]>0.1: if overlaps1[i]>0.7: if boxcenter_ins[i]==1: if not(i in fg_indexes): fg_indexes = np.append(fg_indexes,i) if DEBUG: print "fg_indexes:"+str(fg_indexes) print "**********proposal-gt:"+str(len(fg_indexes)-gt_boxes.shape[0]) f_chan = open('channels.txt') sf_chan = f_chan.read() channels = sf_chan.split(" ") for ii in range(len(fg_indexes)-gt_boxes.shape[0]): if fg_indexes[ii] <len(channels): print channels[fg_indexes[ii]] print labels[fg_indexes[ii]] # guard against the case when an image has fewer than fg_rois_per_image foreground RoIs fg_rois_per_this_image = np.minimum(fg_rois_per_image, fg_indexes.size) # Sample foreground regions without replacement if len(fg_indexes) > fg_rois_per_this_image: fg_indexes = npr.choice(fg_indexes, size=fg_rois_per_this_image, replace=False) # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI) bg_indexes = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) & (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0] # Compute number of background RoIs to take from this image (guarding against there being fewer than desired) bg_rois_per_this_image = rois_per_image - fg_rois_per_this_image bg_rois_per_this_image = np.minimum(bg_rois_per_this_image, bg_indexes.size) # Sample foreground regions without replacement if len(bg_indexes) > bg_rois_per_this_image: bg_indexes = npr.choice(bg_indexes, size=bg_rois_per_this_image, replace=False) # indexes selected keep_indexes = np.append(fg_indexes, bg_indexes) # pad more to ensure a fixed minibatch size while keep_indexes.shape[0] < rois_per_image: gap = np.minimum(len(rois), rois_per_image - keep_indexes.shape[0]) gap_indexes = npr.choice(range(len(rois)), size=gap, replace=False) keep_indexes = np.append(keep_indexes, gap_indexes) # select labels labels = labels[keep_indexes] # set labels of bg_rois to be 0 labels[fg_rois_per_this_image:] = 0 rois = rois[keep_indexes] # load or compute bbox_target if bbox_targets is not None: bbox_target_data = bbox_targets[keep_indexes, :] else: targets = bbox_transform(rois[:, 1:], gt_boxes[gt_assignment[keep_indexes], :4]) if cfg.TRAIN.BBOX_NORMALIZATION_PRECOMPUTED: targets = ((targets - np.array(cfg.TRAIN.BBOX_MEANS)) / np.array(cfg.TRAIN.BBOX_STDS)) bbox_target_data = np.hstack((labels[:, np.newaxis], targets)) bbox_targets, bbox_weights = \ expand_bbox_regression_targets(bbox_target_data, num_classes, cfg) return rois, labels, bbox_targets, bbox_weights