def distribute(rois, label_blobs): """To understand the output blob order see return value of roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn_utils.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) # Delete roi entries that have negative area # idx_neg = np.where(lvls == -1)[0] # rois = np.delete(rois, idx_neg, axis=0) # lvls = np.delete(lvls, idx_neg, axis=0) output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) outputs = [None] * len(output_blob_names) outputs[0] = rois # Create new roi blobs for each FPN level # (See: utils.fpn.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) rois_idx_restore = np.argsort(rois_idx_order) outputs[-1] = rois_idx_restore.astype(np.int32) return dict(zip(output_blob_names, outputs))
def distribute(rois, label_blobs): """To understand the output blob order see return value of roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn_utils.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max) # Delete roi entries that have negative area # idx_neg = np.where(lvls == -1)[0] # rois = np.delete(rois, idx_neg, axis=0) # lvls = np.delete(lvls, idx_neg, axis=0) output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names( is_training=False) outputs = [None] * len(output_blob_names) outputs[0] = rois # Create new roi blobs for each FPN level # (See: utils.fpn.add_multilevel_roi_blobs which is similar but annoying # to generalize to support this particular case.) rois_idx_order = np.empty((0, )) for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] blob_roi_level = rois[idx_lvl, :] outputs[output_idx + 1] = blob_roi_level rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) rois_idx_restore = np.argsort(rois_idx_order) outputs[-1] = rois_idx_restore.astype(np.int32) return dict(zip(output_blob_names, outputs))
def _distribute_rois_over_fpn_levels(rois_blob_name): """Distribute rois over the different FPN levels.""" # Get target level for each roi # Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take # the box coordinates from columns 1:5 target_lvls = fpn_utils.map_rois_to_fpn_levels( blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max) # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> fpn_utils.add_multilevel_roi_blobs(blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name): """Distribute rois over the different FPN levels.""" # Get target level for each roi # Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take # the box coordinates from columns 1:5 target_lvls = fpn_utils.map_rois_to_fpn_levels( blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max ) # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> fpn_utils.add_multilevel_roi_blobs( blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min, lvl_max )
def _add_multilevel_rois_for_test(blobs, name): """Distributes a set of RoIs across FPN pyramid levels by creating new level specific RoI blobs. Arguments: blobs (dict): dictionary of blobs name (str): a key in 'blobs' identifying the source RoI blob Returns: [by ref] blobs (dict): new keys named by `name + 'fpn' + level` are added to dict each with a value that's an R_level x 5 ndarray of RoIs (see _get_rois_blob for format) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min, lvl_max)
def _add_multilevel_rois_for_test(blobs, name): """Distributes a set of RoIs across FPN pyramid levels by creating new level specific RoI blobs. Arguments: blobs (dict): dictionary of blobs name (str): a key in 'blobs' identifying the source RoI blob Returns: [by ref] blobs (dict): new keys named by `name + 'fpn' + level` are added to dict each with a value that's an R_level x 5 ndarray of RoIs (see _get_rois_blob for format) """ lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( blobs, name, blobs[name], lvls, lvl_min, lvl_max )
def obj_feature_map(self, blob_conv, det_rois, use_relu=True): fpn_ret = {'rois': det_rois} if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL # when use min_rel_area, the same sbj/obj area could be mapped to different feature levels # when they are associated with different relationships # Thus we cannot get det_rois features then gather sbj/obj features # The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois rois_blob_names = ['rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls, lvl_min, lvl_max) return self.Box_Head_sg(blob_conv, fpn_ret, rois_name='rois', use_relu=use_relu)
def visual_rep(self, blob_conv_prd, rois, pair_inds, device_id, use_relu=False): assert pair_inds.shape[1] == 2 rel_rois = box_utils_rel.rois_union(rois[pair_inds[:, 0]], rois[pair_inds[:, 1]]) rel_ret = {'sbj_rois': rois[pair_inds[:, 0]], 'obj_rois': rois[pair_inds[:, 1]], 'rel_rois': rel_rois} union_mask = self.union_mask(rel_ret, device_id) if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL # when use min_rel_area, the same sbj/obj area could be mapped to different feature levels # when they are associated with different relationships # Thus we cannot get det_rois features then gather sbj/obj features # The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois rois_blob_names = ['rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( rel_ret, rois_blob_name, rel_ret[rois_blob_name], target_lvls, lvl_min, lvl_max) return self.Box_Head_prd(blob_conv_prd, rel_ret, union_mask, rois_name='rel_rois', use_relu=use_relu)
def _forward(self, data, im_info, do_vis=False, dataset_name=None, roidb=None, use_gt_labels=False, **rpn_kwargs): im_data = data if self.training: roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) if dataset_name is not None: dataset_name = blob_utils.deserialize(dataset_name) else: dataset_name = cfg.TRAIN.DATASETS[ 0] if self.training else cfg.TEST.DATASETS[ 0] # assuming only one dataset per run device_id = im_data.get_device() return_dict = {} # A dict to collect return variables blob_conv = self.Conv_Body(im_data) if not cfg.MODEL.USE_REL_PYRAMID: blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data) rpn_ret = self.RPN(blob_conv, im_info, roidb) if cfg.FPN.FPN_ON: # Retain only the blobs that will be used for RoI heads. `blob_conv` may include # extra blobs that are used for RPN proposals, but not for RoI heads. blob_conv = blob_conv[-self.num_roi_levels:] if not cfg.MODEL.USE_REL_PYRAMID: blob_conv_prd = blob_conv_prd[-self.num_roi_levels:] else: blob_conv_prd = self.RelPyramid(blob_conv) if cfg.MODEL.SHARE_RES5 and self.training: box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) else: box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) cls_score, bbox_pred = self.Box_Outs(box_feat) # now go through the predicate branch use_relu = False if cfg.MODEL.NO_FC7_RELU else True if self.training: fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0] det_rois = rpn_ret['rois'][fg_inds] det_labels = rpn_ret['labels_int32'][fg_inds] det_scores = F.softmax(cls_score[fg_inds], dim=1) rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) if cfg.MODEL.ADD_SO_SCORES: sbj_feat = self.S_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.O_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) else: sbj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS: if cfg.MODEL.ADD_SO_SCORES: # sbj sbj_feat_sbj_pos = self.S_Head( blob_conv, rel_ret, rois_name='sbj_rois_sbj_pos', use_relu=use_relu) obj_feat_sbj_pos = self.O_Head( blob_conv, rel_ret, rois_name='obj_rois_sbj_pos', use_relu=use_relu) # obj sbj_feat_obj_pos = self.S_Head( blob_conv, rel_ret, rois_name='sbj_rois_obj_pos', use_relu=use_relu) obj_feat_obj_pos = self.O_Head( blob_conv, rel_ret, rois_name='obj_rois_obj_pos', use_relu=use_relu) else: # sbj sbj_feat_sbj_pos = self.Box_Head( blob_conv, rel_ret, rois_name='sbj_rois_sbj_pos', use_relu=use_relu) obj_feat_sbj_pos = self.Box_Head( blob_conv, rel_ret, rois_name='obj_rois_sbj_pos', use_relu=use_relu) # obj sbj_feat_obj_pos = self.Box_Head( blob_conv, rel_ret, rois_name='sbj_rois_obj_pos', use_relu=use_relu) obj_feat_obj_pos = self.Box_Head( blob_conv, rel_ret, rois_name='obj_rois_obj_pos', use_relu=use_relu) else: if roidb is not None: im_scale = im_info.data.numpy()[:, 2][0] im_w = im_info.data.numpy()[:, 1][0] im_h = im_info.data.numpy()[:, 0][0] sbj_boxes = roidb['sbj_gt_boxes'] obj_boxes = roidb['obj_gt_boxes'] sbj_rois = sbj_boxes * im_scale obj_rois = obj_boxes * im_scale repeated_batch_idx = 0 * blob_utils.ones( (sbj_rois.shape[0], 1)) sbj_rois = np.hstack((repeated_batch_idx, sbj_rois)) obj_rois = np.hstack((repeated_batch_idx, obj_rois)) rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois) rel_ret = {} rel_ret['sbj_rois'] = sbj_rois rel_ret['obj_rois'] = obj_rois rel_ret['rel_rois'] = rel_rois if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( rel_ret, rois_blob_name, rel_ret[rois_blob_name], target_lvls, lvl_min, lvl_max) sbj_det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=True) sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat) sbj_cls_scores = sbj_cls_scores.data.cpu().numpy() obj_det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=True) obj_cls_scores, _ = self.Box_Outs(obj_det_feat) obj_cls_scores = obj_cls_scores.data.cpu().numpy() if use_gt_labels: sbj_labels = roidb['sbj_gt_classes'] # start from 0 obj_labels = roidb['obj_gt_classes'] # start from 0 sbj_scores = np.ones_like(sbj_labels, dtype=np.float32) obj_scores = np.ones_like(obj_labels, dtype=np.float32) else: sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1) obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1) sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1) obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1) rel_ret['sbj_scores'] = sbj_scores.astype(np.float32, copy=False) rel_ret['obj_scores'] = obj_scores.astype(np.float32, copy=False) rel_ret['sbj_labels'] = sbj_labels.astype( np.int32, copy=False) + 1 # need to start from 1 rel_ret['obj_labels'] = obj_labels.astype( np.int32, copy=False) + 1 # need to start from 1 rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32, copy=False) rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32, copy=False) if cfg.MODEL.USE_SPATIAL_FEAT: spt_feat = box_utils_rel.get_spt_features( sbj_boxes, obj_boxes, im_w, im_h) rel_ret['spt_feat'] = spt_feat if cfg.MODEL.ADD_SO_SCORES: sbj_feat = self.S_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.O_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) else: sbj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) else: score_thresh = cfg.TEST.SCORE_THRESH while score_thresh >= -1e-06: # a negative value very close to 0.0 det_rois, det_labels, det_scores = \ self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh) rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) valid_len = len(rel_ret['rel_rois']) if valid_len > 0: break logger.info( 'Got {} rel_rois when score_thresh={}, changing to {}'. format(valid_len, score_thresh, score_thresh - 0.01)) score_thresh -= 0.01 if cfg.MODEL.ADD_SO_SCORES: det_s_feat = self.S_Head(blob_conv, rel_ret, rois_name='det_rois', use_relu=use_relu) det_o_feat = self.O_Head(blob_conv, rel_ret, rois_name='det_rois', use_relu=use_relu) sbj_feat = det_s_feat[rel_ret['sbj_inds']] obj_feat = det_o_feat[rel_ret['obj_inds']] else: det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='det_rois', use_relu=use_relu) sbj_feat = det_feat[rel_ret['sbj_inds']] obj_feat = det_feat[rel_ret['obj_inds']] rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd, rel_ret, rois_name='rel_rois', use_relu=use_relu) spo_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1) if cfg.MODEL.USE_SPATIAL_FEAT: spt_feat = rel_ret['spt_feat'] else: spt_feat = None if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE: sbj_labels = rel_ret['all_sbj_labels_int32'] obj_labels = rel_ret['all_obj_labels_int32'] else: sbj_labels = None obj_labels = None # prd_scores is the visual scores. See reldn_heads.py prd_scores, prd_bias_scores, prd_spt_scores, ttl_cls_scores, sbj_cls_scores, obj_cls_scores = \ self.RelDN(spo_feat, spt_feat, sbj_labels, obj_labels, sbj_feat, obj_feat) if self.training: return_dict['losses'] = {} return_dict['metrics'] = {} # rpn loss rpn_kwargs.update( dict((k, rpn_ret[k]) for k in rpn_ret.keys() if (k.startswith('rpn_cls_logits') or k.startswith('rpn_bbox_pred')))) loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses( **rpn_kwargs) if cfg.FPN.FPN_ON: for i, lvl in enumerate( range(cfg.FPN.RPN_MIN_LEVEL, cfg.FPN.RPN_MAX_LEVEL + 1)): return_dict['losses']['loss_rpn_cls_fpn%d' % lvl] = loss_rpn_cls[i] return_dict['losses']['loss_rpn_bbox_fpn%d' % lvl] = loss_rpn_bbox[i] else: return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox # bbox loss loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses( cls_score, bbox_pred, rpn_ret['labels_int32'], rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'], rpn_ret['bbox_outside_weights']) return_dict['losses']['loss_cls'] = loss_cls return_dict['losses']['loss_bbox'] = loss_bbox return_dict['metrics']['accuracy_cls'] = accuracy_cls if cfg.MODEL.USE_FREQ_BIAS and not cfg.MODEL.ADD_SCORES_ALL: loss_cls_bias, accuracy_cls_bias = reldn_heads.reldn_losses( prd_bias_scores, rel_ret['all_prd_labels_int32']) return_dict['losses']['loss_cls_bias'] = loss_cls_bias return_dict['metrics']['accuracy_cls_bias'] = accuracy_cls_bias if cfg.MODEL.USE_SPATIAL_FEAT and not cfg.MODEL.ADD_SCORES_ALL: loss_cls_spt, accuracy_cls_spt = reldn_heads.reldn_losses( prd_spt_scores, rel_ret['all_prd_labels_int32']) return_dict['losses']['loss_cls_spt'] = loss_cls_spt return_dict['metrics']['accuracy_cls_spt'] = accuracy_cls_spt if cfg.MODEL.ADD_SCORES_ALL: loss_cls_ttl, accuracy_cls_ttl = reldn_heads.reldn_losses( ttl_cls_scores, rel_ret['all_prd_labels_int32']) return_dict['losses']['loss_cls_ttl'] = loss_cls_ttl return_dict['metrics']['accuracy_cls_ttl'] = accuracy_cls_ttl else: loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses( prd_scores, rel_ret['all_prd_labels_int32']) return_dict['losses']['loss_cls_prd'] = loss_cls_prd return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS: # sbj rel_feat_sbj_pos = self.Prd_RCNN.Box_Head( blob_conv_prd, rel_ret, rois_name='rel_rois_sbj_pos', use_relu=use_relu) spo_feat_sbj_pos = torch.cat( (sbj_feat_sbj_pos, rel_feat_sbj_pos, obj_feat_sbj_pos), dim=1) if cfg.MODEL.USE_SPATIAL_FEAT: spt_feat_sbj_pos = rel_ret['spt_feat_sbj_pos'] else: spt_feat_sbj_pos = None if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE: sbj_labels_sbj_pos_fg = rel_ret[ 'sbj_labels_sbj_pos_fg_int32'] obj_labels_sbj_pos_fg = rel_ret[ 'obj_labels_sbj_pos_fg_int32'] else: sbj_labels_sbj_pos_fg = None obj_labels_sbj_pos_fg = None _, prd_bias_scores_sbj_pos, _, ttl_cls_scores_sbj_pos, _, _ = \ self.RelDN(spo_feat_sbj_pos, spt_feat_sbj_pos, sbj_labels_sbj_pos_fg, obj_labels_sbj_pos_fg, sbj_feat_sbj_pos, obj_feat_sbj_pos) # obj rel_feat_obj_pos = self.Prd_RCNN.Box_Head( blob_conv_prd, rel_ret, rois_name='rel_rois_obj_pos', use_relu=use_relu) spo_feat_obj_pos = torch.cat( (sbj_feat_obj_pos, rel_feat_obj_pos, obj_feat_obj_pos), dim=1) if cfg.MODEL.USE_SPATIAL_FEAT: spt_feat_obj_pos = rel_ret['spt_feat_obj_pos'] else: spt_feat_obj_pos = None if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE: sbj_labels_obj_pos_fg = rel_ret[ 'sbj_labels_obj_pos_fg_int32'] obj_labels_obj_pos_fg = rel_ret[ 'obj_labels_obj_pos_fg_int32'] else: sbj_labels_obj_pos_fg = None obj_labels_obj_pos_fg = None _, prd_bias_scores_obj_pos, _, ttl_cls_scores_obj_pos, _, _ = \ self.RelDN(spo_feat_obj_pos, spt_feat_obj_pos, sbj_labels_obj_pos_fg, obj_labels_obj_pos_fg, sbj_feat_obj_pos, obj_feat_obj_pos) if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS: loss_contrastive_sbj, loss_contrastive_obj = reldn_heads.reldn_contrastive_losses( ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos, rel_ret) return_dict['losses'][ 'loss_contrastive_sbj'] = loss_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT return_dict['losses'][ 'loss_contrastive_obj'] = loss_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT if cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS: loss_so_contrastive_sbj, loss_so_contrastive_obj = reldn_heads.reldn_so_contrastive_losses( ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos, rel_ret) return_dict['losses'][ 'loss_so_contrastive_sbj'] = loss_so_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT return_dict['losses'][ 'loss_so_contrastive_obj'] = loss_so_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT if cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS: loss_p_contrastive_sbj, loss_p_contrastive_obj = reldn_heads.reldn_p_contrastive_losses( ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos, prd_bias_scores_sbj_pos, prd_bias_scores_obj_pos, rel_ret) return_dict['losses'][ 'loss_p_contrastive_sbj'] = loss_p_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT return_dict['losses'][ 'loss_p_contrastive_obj'] = loss_p_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT # pytorch0.4 bug on gathering scalar(0-dim) tensors for k, v in return_dict['losses'].items(): return_dict['losses'][k] = v.unsqueeze(0) for k, v in return_dict['metrics'].items(): return_dict['metrics'][k] = v.unsqueeze(0) else: # Testing return_dict['sbj_rois'] = rel_ret['sbj_rois'] return_dict['obj_rois'] = rel_ret['obj_rois'] return_dict['sbj_labels'] = rel_ret['sbj_labels'] return_dict['obj_labels'] = rel_ret['obj_labels'] return_dict['sbj_scores'] = rel_ret['sbj_scores'] return_dict['obj_scores'] = rel_ret['obj_scores'] return_dict['prd_scores'] = prd_scores if cfg.MODEL.USE_FREQ_BIAS: return_dict['prd_scores_bias'] = prd_bias_scores if cfg.MODEL.USE_SPATIAL_FEAT: return_dict['prd_scores_spt'] = prd_spt_scores if cfg.MODEL.ADD_SCORES_ALL: return_dict['prd_ttl_scores'] = ttl_cls_scores if do_vis: return_dict['blob_conv'] = blob_conv return_dict['blob_conv_prd'] = blob_conv_prd return return_dict
def forward(self, det_rois, det_labels, det_scores, im_info, dataset_name, roidb=None): """ det_rois: feature maps from the backbone network. (Variable) im_info: (CPU Variable) roidb: (list of ndarray) """ # Get pairwise proposals first if roidb is not None: # we always feed one image per batch during training assert len(roidb) == 1 sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0]) obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0]) # remove self paired rois if det_rois.shape[ 0] > 1: # no pairs to remove when there is at most one detection sbj_inds, obj_inds = self.remove_self_pairs( det_rois.shape[0], sbj_inds, obj_inds) sbj_rois = det_rois[sbj_inds] obj_rois = det_rois[obj_inds] im_scale = im_info.data.numpy()[:, 2][0] sbj_boxes = sbj_rois[:, 1:] / im_scale obj_boxes = obj_rois[:, 1:] / im_scale # filters out those roi pairs whose boxes are not overlapping in the original scales if cfg.MODEL.USE_OVLP_FILTER: ovlp_so = box_utils.bbox_pair_overlaps( sbj_boxes.astype(dtype=np.float32, copy=False), obj_boxes.astype(dtype=np.float32, copy=False)) ovlp_inds = np.where(ovlp_so > 0)[0] sbj_inds = sbj_inds[ovlp_inds] obj_inds = obj_inds[ovlp_inds] sbj_rois = sbj_rois[ovlp_inds] obj_rois = obj_rois[ovlp_inds] sbj_boxes = sbj_boxes[ovlp_inds] obj_boxes = obj_boxes[ovlp_inds] return_dict = {} if self.training: # Add binary relationships blobs_out = self.RelPN_GenerateProposalLabels( sbj_rois, obj_rois, det_rois, roidb, im_info) return_dict.update(blobs_out) else: sbj_labels = det_labels[sbj_inds] obj_labels = det_labels[obj_inds] sbj_scores = det_scores[sbj_inds] obj_scores = det_scores[obj_inds] rel_rois = box_utils.rois_union(sbj_rois, obj_rois) return_dict['det_rois'] = det_rois return_dict['sbj_inds'] = sbj_inds return_dict['obj_inds'] = obj_inds return_dict['sbj_rois'] = sbj_rois return_dict['obj_rois'] = obj_rois return_dict['rel_rois'] = rel_rois return_dict['sbj_labels'] = sbj_labels return_dict['obj_labels'] = obj_labels return_dict['sbj_scores'] = sbj_scores return_dict['obj_scores'] = obj_scores return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32) im_scale = im_info.data.numpy()[:, 2][0] im_w = im_info.data.numpy()[:, 1][0] im_h = im_info.data.numpy()[:, 0][0] if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT: return_dict['all_sbj_labels_int32'] = sbj_labels.astype( np.int32, copy=False) - 1 # det_labels start from 1 return_dict['all_obj_labels_int32'] = obj_labels.astype( np.int32, copy=False) - 1 # det_labels start from 1 if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL # when use min_rel_area, the same sbj/obj area could be mapped to different feature levels # when they are associated with different relationships # Thus we cannot get det_rois features then gather sbj/obj features # The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois rois_blob_names = ['det_rois', 'rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( return_dict, rois_blob_name, return_dict[rois_blob_name], target_lvls, lvl_min, lvl_max) return return_dict
def _forward(self, data, im_info, dataset_name=None, roidb=None, use_gt_labels=False, include_feat=False, **rpn_kwargs): im_data = data if self.training: roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) if dataset_name is not None: dataset_name = blob_utils.deserialize(dataset_name) else: dataset_name = cfg.TRAIN.DATASETS[ 0] if self.training else cfg.TEST.DATASETS[ 0] # assuming only one dataset per run device_id = im_data.get_device() return_dict = {} # A dict to collect return variables blob_conv = self.Conv_Body(im_data) blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data) if cfg.FPN.FPN_ON: # Retain only the blobs that will be used for RoI heads. `blob_conv` may include # extra blobs that are used for RPN proposals, but not for RoI heads. blob_conv = blob_conv[-self.num_roi_levels:] blob_conv_prd = blob_conv_prd[-self.num_roi_levels:] if not cfg.TRAIN.USE_GT_BOXES: rpn_ret = self.RPN(blob_conv, im_info, roidb) if cfg.MODEL.SHARE_RES5 and self.training: box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) else: box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) cls_score, bbox_pred = self.Box_Outs(box_feat) # now go through the predicate branch use_relu = False if cfg.MODEL.NO_FC7_RELU else True if self.training: if cfg.TRAIN.USE_GT_BOXES: # we always feed one image per batch during training assert len(roidb) == 1 im_scale = im_info.data.numpy()[:, 2][0] im_w = im_info.data.numpy()[:, 1][0] im_h = im_info.data.numpy()[:, 0][0] sbj_boxes = roidb[0]['sbj_gt_boxes'] obj_boxes = roidb[0]['obj_gt_boxes'] sbj_all_boxes = _augment_gt_boxes_by_perturbation( sbj_boxes, im_w, im_h) obj_all_boxes = _augment_gt_boxes_by_perturbation( obj_boxes, im_w, im_h) det_all_boxes = np.vstack((sbj_all_boxes, obj_all_boxes)) det_all_boxes = np.unique(det_all_boxes, axis=0) det_all_rois = det_all_boxes * im_scale repeated_batch_idx = 0 * blob_utils.ones( (det_all_rois.shape[0], 1)) det_all_rois = np.hstack((repeated_batch_idx, det_all_rois)) rel_ret = self.RelPN(det_all_rois, None, None, im_info, dataset_name, roidb) else: fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0] det_rois = rpn_ret['rois'][fg_inds] det_labels = rpn_ret['labels_int32'][fg_inds] det_scores = F.softmax(cls_score[fg_inds], dim=1) rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) sbj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) else: if roidb is not None: im_scale = im_info.data.numpy()[:, 2][0] im_w = im_info.data.numpy()[:, 1][0] im_h = im_info.data.numpy()[:, 0][0] sbj_boxes = roidb['sbj_gt_boxes'] obj_boxes = roidb['obj_gt_boxes'] sbj_rois = sbj_boxes * im_scale obj_rois = obj_boxes * im_scale repeated_batch_idx = 0 * blob_utils.ones( (sbj_rois.shape[0], 1)) sbj_rois = np.hstack((repeated_batch_idx, sbj_rois)) obj_rois = np.hstack((repeated_batch_idx, obj_rois)) rel_rois = box_utils.rois_union(sbj_rois, obj_rois) rel_ret = {} rel_ret['sbj_rois'] = sbj_rois rel_ret['obj_rois'] = obj_rois rel_ret['rel_rois'] = rel_rois if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( rel_ret, rois_blob_name, rel_ret[rois_blob_name], target_lvls, lvl_min, lvl_max) if use_gt_labels: sbj_labels = roidb['sbj_gt_classes'] # start from 0 obj_labels = roidb['obj_gt_classes'] # start from 0 sbj_scores = np.ones_like(sbj_labels, dtype=np.float32) obj_scores = np.ones_like(obj_labels, dtype=np.float32) else: sbj_det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=True) sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat) sbj_cls_scores = sbj_cls_scores.data.cpu().numpy() obj_det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=True) obj_cls_scores, _ = self.Box_Outs(obj_det_feat) obj_cls_scores = obj_cls_scores.data.cpu().numpy() sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1) obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1) sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1) obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1) rel_ret['sbj_scores'] = sbj_scores.astype(np.float32, copy=False) rel_ret['obj_scores'] = obj_scores.astype(np.float32, copy=False) rel_ret['sbj_labels'] = sbj_labels.astype( np.int32, copy=False) + 1 # need to start from 1 rel_ret['obj_labels'] = obj_labels.astype( np.int32, copy=False) + 1 # need to start from 1 rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32, copy=False) rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32, copy=False) sbj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='sbj_rois', use_relu=use_relu) obj_feat = self.Box_Head(blob_conv, rel_ret, rois_name='obj_rois', use_relu=use_relu) else: score_thresh = cfg.TEST.SCORE_THRESH while score_thresh >= -1e-06: # a negative value very close to 0.0 det_rois, det_labels, det_scores = \ self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh) rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) valid_len = len(rel_ret['rel_rois']) if valid_len > 0: break logger.info( 'Got {} rel_rois when score_thresh={}, changing to {}'. format(valid_len, score_thresh, score_thresh - 0.01)) score_thresh -= 0.01 det_feat = self.Box_Head(blob_conv, rel_ret, rois_name='det_rois', use_relu=use_relu) sbj_feat = det_feat[rel_ret['sbj_inds']] obj_feat = det_feat[rel_ret['obj_inds']] rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd, rel_ret, rois_name='rel_rois', use_relu=use_relu) concat_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1) if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT: sbj_labels = rel_ret['all_sbj_labels_int32'] obj_labels = rel_ret['all_obj_labels_int32'] else: sbj_labels = None obj_labels = None # when MODEL.USE_SEM_CONCAT, memory runs out if the whole batch is fed once # so we need to feed the batch twice if it's big gn_size = 1000 if cfg.MODEL.USE_SEM_CONCAT and concat_feat.shape[0] > gn_size: group = int(math.ceil(concat_feat.shape[0] / gn_size)) prd_cls_scores = None sbj_cls_scores = None obj_cls_scores = None for i in range(group): end = int(min((i + 1) * gn_size, concat_feat.shape[0])) concat_feat_i = concat_feat[i * gn_size:end] sbj_labels_i = sbj_labels[ i * gn_size:end] if sbj_labels is not None else None obj_labels_i = obj_labels[ i * gn_size:end] if obj_labels is not None else None sbj_feat_i = sbj_feat[i * gn_size:end] obj_feat_i = obj_feat[i * gn_size:end] prd_cls_scores_i, sbj_cls_scores_i, obj_cls_scores_i = \ self.RelDN(concat_feat_i, sbj_labels_i, obj_labels_i, sbj_feat_i, obj_feat_i) if prd_cls_scores is None: prd_cls_scores = prd_cls_scores_i sbj_cls_scores = sbj_cls_scores_i obj_cls_scores = obj_cls_scores_i else: prd_cls_scores = torch.cat( (prd_cls_scores, prd_cls_scores_i)) sbj_cls_scores = torch.cat( (sbj_cls_scores, sbj_cls_scores_i )) if sbj_cls_scores_i is not None else sbj_cls_scores obj_cls_scores = torch.cat( (obj_cls_scores, obj_cls_scores_i )) if obj_cls_scores_i is not None else obj_cls_scores else: prd_cls_scores, sbj_cls_scores, obj_cls_scores = \ self.RelDN(concat_feat, sbj_labels, obj_labels, sbj_feat, obj_feat) if self.training: return_dict['losses'] = {} return_dict['metrics'] = {} if not cfg.TRAIN.USE_GT_BOXES: # rpn loss rpn_kwargs.update( dict((k, rpn_ret[k]) for k in rpn_ret.keys() if (k.startswith('rpn_cls_logits') or k.startswith('rpn_bbox_pred')))) loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses( **rpn_kwargs) if cfg.FPN.FPN_ON: for i, lvl in enumerate( range(cfg.FPN.RPN_MIN_LEVEL, cfg.FPN.RPN_MAX_LEVEL + 1)): return_dict['losses']['loss_rpn_cls_fpn%d' % lvl] = loss_rpn_cls[i] return_dict['losses']['loss_rpn_bbox_fpn%d' % lvl] = loss_rpn_bbox[i] else: return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox # bbox loss loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses( cls_score, bbox_pred, rpn_ret['labels_int32'], rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'], rpn_ret['bbox_outside_weights']) return_dict['losses']['loss_cls'] = loss_cls return_dict['losses']['loss_bbox'] = loss_bbox return_dict['metrics']['accuracy_cls'] = accuracy_cls loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses( prd_cls_scores, rel_ret['all_prd_labels_int32'], weight=self.prd_weights) return_dict['losses']['loss_cls_prd'] = loss_cls_prd return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd if cfg.MODEL.USE_SEPARATE_SO_SCORES: loss_cls_sbj, accuracy_cls_sbj = reldn_heads.reldn_losses( sbj_cls_scores, rel_ret['all_sbj_labels_int32'], weight=self.obj_weights) return_dict['losses']['loss_cls_sbj'] = loss_cls_sbj return_dict['metrics']['accuracy_cls_sbj'] = accuracy_cls_sbj loss_cls_obj, accuracy_cls_obj = reldn_heads.reldn_losses( obj_cls_scores, rel_ret['all_obj_labels_int32'], weight=self.obj_weights) return_dict['losses']['loss_cls_obj'] = loss_cls_obj return_dict['metrics']['accuracy_cls_obj'] = accuracy_cls_obj if cfg.TRAIN.HUBNESS: loss_hubness_prd = reldn_heads.add_hubness_loss(prd_cls_scores) loss_hubness_sbj = reldn_heads.add_hubness_loss(sbj_cls_scores) loss_hubness_obj = reldn_heads.add_hubness_loss(obj_cls_scores) return_dict['losses']['loss_hubness_prd'] = loss_hubness_prd return_dict['losses']['loss_hubness_sbj'] = loss_hubness_sbj return_dict['losses']['loss_hubness_obj'] = loss_hubness_obj # pytorch0.4 bug on gathering scalar(0-dim) tensors for k, v in return_dict['losses'].items(): return_dict['losses'][k] = v.unsqueeze(0) for k, v in return_dict['metrics'].items(): return_dict['metrics'][k] = v.unsqueeze(0) else: # Testing return_dict['sbj_rois'] = rel_ret['sbj_rois'] return_dict['obj_rois'] = rel_ret['obj_rois'] return_dict['sbj_labels'] = rel_ret['sbj_labels'] return_dict['obj_labels'] = rel_ret['obj_labels'] return_dict['sbj_scores'] = rel_ret['sbj_scores'] return_dict['sbj_scores_out'] = sbj_cls_scores return_dict['obj_scores'] = rel_ret['obj_scores'] return_dict['obj_scores_out'] = obj_cls_scores return_dict['prd_scores'] = prd_cls_scores if include_feat: return_dict['sbj_feat'] = sbj_feat return_dict['obj_feat'] = obj_feat return_dict['prd_feat'] = concat_feat return return_dict
def _forward(self, data, im_info, do_vis=False, dataset_name=None, roidb=None, use_gt_labels=False, **rpn_kwargs): im_data = data if self.training: # if not isinstance(roidb[0], np.array): # roidb = roidb[0] roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) # only support one gpu if dataset_name is not None: dataset_name = blob_utils.deserialize(dataset_name) else: dataset_name = cfg.TRAIN.DATASETS[0] if self.training else cfg.TEST.DATASETS[0] # assuming only one dataset per run device_id = im_data.get_device() return_dict = {} # A dict to collect return variables blob_conv = self.Conv_Body(im_data) # if not cfg.MODEL.USE_REL_PYRAMID: # blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data) if self.training: gt_rois = np.empty((0, 5), dtype=np.float32) gt_classes = np.empty((0), dtype=np.int64) for i, r in enumerate(roidb): rois_i = r['boxes'] * im_info[i, 2] rois_i = np.hstack((i * blob_utils.ones((rois_i.shape[0], 1)), rois_i)) gt_rois = np.append(gt_rois, rois_i, axis=0) gt_classes = np.append(gt_classes, r['gt_classes'], axis=0) if self.training or roidb is None: rpn_ret = self.RPN(blob_conv, im_info, roidb) if cfg.FPN.FPN_ON: # Retain only the blobs that will be used for RoI heads. `blob_conv` may include # extra blobs that are used for RPN proposals, but not for RoI heads. blob_conv = blob_conv[-self.num_roi_levels:] # if not cfg.MODEL.USE_REL_PYRAMID: # blob_conv_prd = blob_conv_prd[-self.num_roi_levels:] # else: # blob_conv_prd = self.RelPyramid(blob_conv) if self.training or roidb is None: if cfg.MODEL.SHARE_RES5 and self.training: box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) else: box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True) cls_score, bbox_pred = self.Box_Outs(box_feat) # now go through the predicate branch use_relu = False if cfg.MODEL.NO_FC7_RELU else True if self.training: score_thresh = cfg.TEST.SCORE_THRESH cls_score = F.softmax(cls_score, -1) while score_thresh >= -1e-06: # a negative value very close to 0.0 det_rois, det_labels, det_scores, det_dists, det_boxes_all = \ self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh) real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2]) non_zero_area_inds = np.where(real_area > 0)[0] det_rois = det_rois[non_zero_area_inds] det_labels = det_labels[non_zero_area_inds] det_scores = det_scores[non_zero_area_inds] det_dists = det_dists[non_zero_area_inds] det_boxes_all = det_boxes_all[non_zero_area_inds] # rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) valid_len = len(det_rois) if valid_len > 0: break logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format( valid_len, score_thresh, score_thresh - 0.01)) score_thresh -= 0.01 det_labels_gt = [] ious = box_utils.bbox_overlaps(det_rois[:, 1:], gt_rois[:, 1:]) * \ (det_rois[:, 0][:,None] == gt_rois[:, 0][None, :]) det_labels_gt = gt_classes[ious.argmax(-1)] det_labels_gt[ious.max(-1) < cfg.TRAIN.FG_THRESH] = 0 else: if roidb is not None: # raise FError('not support this mode!') # assert len(roidb) == 1 im_scale = im_info.data.numpy()[:, 2][0] im_w = im_info.data.numpy()[:, 1][0] im_h = im_info.data.numpy()[:, 0][0] fpn_ret = {'gt_rois': gt_rois} if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL rois_blob_names = ['gt_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs( fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls, lvl_min, lvl_max) det_feats = self.Box_Head(blob_conv, fpn_ret, rois_name='det_rois', use_relu=True) det_dists, _ = self.Box_Outs(det_feats) det_boxes_all = None if use_gt_labels: det_labels_gt = gt_classes det_labels = gt_classes else: score_thresh = cfg.TEST.SCORE_THRESH while score_thresh >= -1e-06: # a negative value very close to 0.0 det_rois, det_labels, det_scores, det_dists, det_boxes_all = \ self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh) real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2]) non_zero_area_inds = np.where(real_area > 0)[0] det_rois = det_rois[non_zero_area_inds] det_labels = det_labels[non_zero_area_inds] det_scores = det_scores[non_zero_area_inds] det_dists = det_dists[non_zero_area_inds] det_boxes_all = det_boxes_all[non_zero_area_inds] # rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb) valid_len = len(det_rois) if valid_len > 0: break logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format( valid_len, score_thresh, score_thresh - 0.01)) score_thresh -= 0.01 return_dict['det_rois'] = det_rois num_rois = det_rois.shape[0] if not isinstance(det_dists, torch.Tensor): assert det_dists.shape[0] == num_rois det_dists = torch.from_numpy(det_dists).float().cuda(device_id) return_dict['det_dists'] = det_dists return_dict['det_scores'] = det_scores return_dict['blob_conv'] = blob_conv return_dict['det_boxes_all'] = det_boxes_all assert det_boxes_all.shape[0] == num_rois return_dict['det_labels'] = det_labels # return_dict['blob_conv_prd'] = blob_conv_prd if self.training or use_gt_labels: return_dict['det_labels_gt'] = det_labels_gt return return_dict
def forward(self, det_rois, det_labels, det_scores, im_info, dataset_name, roidb=None): """ det_rois: feature maps from the backbone network. (Variable) im_info: (CPU Variable) roidb: (list of ndarray) """ # Get pairwise proposals first im_inds = det_rois[:, 0] is_cand = im_inds[:, None] == im_inds[None, :] is_cand.reshape(-1)[diagonal_inds(is_cand)] = False is_empty = np.where(is_cand.any(1) == 0)[0] if self.overlap: is_cand = is_cand & (box_utils.bbox_overlaps( det_rois[:, 1:], det_rois[:, 1:]) > 0) if is_empty.size > 0: is_cand[is_empty, is_empty] = True sbj_inds, obj_inds = np.where(is_cand) # remove self paired rois sbj_rois = det_rois[sbj_inds] obj_rois = det_rois[obj_inds] im_scale = im_info.data.numpy()[:, 2][0] sbj_boxes = sbj_rois[:, 1:] / im_scale obj_boxes = obj_rois[:, 1:] / im_scale # filters out those roi pairs whose boxes are not overlapping in the original scales return_dict = {} sbj_labels = det_labels[sbj_inds] obj_labels = det_labels[obj_inds] sbj_scores = det_scores[sbj_inds] obj_scores = det_scores[obj_inds] rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois) return_dict['det_rois'] = det_rois return_dict['sbj_inds'] = sbj_inds return_dict['obj_inds'] = obj_inds return_dict['sbj_rois'] = sbj_rois return_dict['obj_rois'] = obj_rois return_dict['rel_rois'] = rel_rois return_dict['sbj_labels'] = sbj_labels return_dict['obj_labels'] = obj_labels return_dict['sbj_scores'] = sbj_scores return_dict['obj_scores'] = obj_scores return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32) if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL # when use min_rel_area, the same sbj/obj area could be mapped to different feature levels # when they are associated with different relationships # Thus we cannot get det_rois features then gather sbj/obj features # The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois rois_blob_names = ['det_rois', 'rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name, return_dict[rois_blob_name], target_lvls, lvl_min, lvl_max) return return_dict
def _forward(self, data, im_info, roidb=None, **rpn_kwargs): im_data = data if self.training: roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) device_id = im_data.get_device() return_dict = {} # A dict to collect return variables blob_conv = self.Conv_Body(im_data) rpn_ret = self.RPN(blob_conv, im_info, roidb) # rpn proposals # if self.training: # # can be used to infer fg/bg ratio # return_dict['rois_label'] = rpn_ret['labels_int32'] rois_certification = False if cfg.FPN.FPN_ON: # Retain only the blobs that will be used for RoI heads. `blob_conv` may include # extra blobs that are used for RPN proposals, but not for RoI heads. blob_conv = blob_conv[-self.num_roi_levels:] if not self.training: return_dict['blob_conv'] = blob_conv if rois_certification: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL test_rpn_ret = {'rois': rpn_ret['rois']} lvls = fpn_utils.map_rois_to_fpn_levels(test_rpn_ret['rois'], lvl_min, lvl_max) rois_idx_order = np.empty((0, )) test_rois = test_rpn_ret['rois'] for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] rois_lvl = test_rois[idx_lvl, :] rois_idx_order = np.concatenate((rois_idx_order, idx_lvl)) test_rpn_ret['rois_fpn{}'.format(lvl)] = rois_lvl rois_idx_restore = np.argsort(rois_idx_order).astype(np.int32, copy=False) test_rpn_ret['rois_idx_restore_int32'] = rois_idx_restore test_feat = self.Box_Head(blob_conv, test_rpn_ret) test_cls_score, test_bbox_pred = self.Box_Outs(test_feat) test_cls_score = test_cls_score.data.cpu().numpy().squeeze() test_bbox_pred = test_bbox_pred.data.cpu().numpy().squeeze() if not cfg.MODEL.RPN_ONLY: if cfg.MODEL.SHARE_RES5 and self.training: box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret) # bbox proposals else: box_feat = self.Box_Head(blob_conv, rpn_ret) cls_score, bbox_pred = self.Box_Outs(box_feat) else: # TODO: complete the returns for RPN only situation pass # 在这里开始计算loss if self.training: return_dict['losses'] = {} return_dict['metrics'] = {} # rpn loss rpn_kwargs.update( dict((k, rpn_ret[k]) for k in rpn_ret.keys() if (k.startswith('rpn_cls_logits') or k.startswith('rpn_bbox_pred')))) loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses( **rpn_kwargs) if cfg.FPN.FPN_ON: for i, lvl in enumerate( range(cfg.FPN.RPN_MIN_LEVEL, cfg.FPN.RPN_MAX_LEVEL + 1)): return_dict['losses']['loss_rpn_cls_fpn%d' % lvl] = loss_rpn_cls[i] return_dict['losses']['loss_rpn_bbox_fpn%d' % lvl] = loss_rpn_bbox[i] else: return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox # bbox loss loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses( cls_score, bbox_pred, rpn_ret['labels_int32'], rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'], rpn_ret['bbox_outside_weights']) return_dict['losses']['loss_cls'] = loss_cls return_dict['losses']['loss_bbox'] = loss_bbox return_dict['metrics']['accuracy_cls'] = accuracy_cls if cfg.MODEL.MASK_ON: if getattr(self.Mask_Head, 'SHARE_RES5', False): mask_feat = self.Mask_Head( res5_feat, rpn_ret, roi_has_mask_int32=rpn_ret['roi_has_mask_int32']) else: mask_feat = self.Mask_Head(blob_conv, rpn_ret) mask_pred = self.Mask_Outs(mask_feat) # return_dict['mask_pred'] = mask_pred # mask loss loss_mask = mask_rcnn_heads.mask_rcnn_losses( mask_pred, rpn_ret['masks_int32']) return_dict['losses']['loss_mask'] = loss_mask if cfg.MODEL.KEYPOINTS_ON: if getattr(self.Keypoint_Head, 'SHARE_RES5', False): # No corresponding keypoint head implemented yet (Neither in Detectron) # Also, rpn need to generate the label 'roi_has_keypoints_int32' kps_feat = self.Keypoint_Head( res5_feat, rpn_ret, roi_has_keypoints_int32=rpn_ret[ 'roi_has_keypoint_int32']) else: kps_feat = self.Keypoint_Head(blob_conv, rpn_ret) kps_pred = self.Keypoint_Outs(kps_feat) # return_dict['keypoints_pred'] = kps_pred # keypoints loss if cfg.KRCNN.NORMALIZE_BY_VISIBLE_KEYPOINTS: loss_keypoints = keypoint_rcnn_heads.keypoint_losses( kps_pred, rpn_ret['keypoint_locations_int32'], rpn_ret['keypoint_weights']) else: loss_keypoints = keypoint_rcnn_heads.keypoint_losses( kps_pred, rpn_ret['keypoint_locations_int32'], rpn_ret['keypoint_weights'], rpn_ret['keypoint_loss_normalizer']) return_dict['losses']['loss_kps'] = loss_keypoints # pytorch0.4 bug on gathering scalar(0-dim) tensors for k, v in return_dict['losses'].items(): return_dict['losses'][k] = v.unsqueeze(0) for k, v in return_dict['metrics'].items(): return_dict['metrics'][k] = v.unsqueeze(0) else: # Testing return_dict['rois'] = rpn_ret['rois'] import json if cfg.TEST.IOU_OUT: # 直接通过rpn_ret可以取出rois with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/raw_roi.json", 'w') as f: json.dump((return_dict['rois'][:, 1:] / im_info.numpy()[0][2]).tolist(), f) # 如果在FPN模式下,需要进到一个collect_and_distribute...的函数去取出分配后的scores # ,我直接在collect_and_distribute_fpn_rpn_proposals.py里把json输出 # 因此这里直接考虑RPN_ONLY模式的取值。 if not cfg.FPN.FPN_ON: with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/rois_score.json", 'w') as f: score_2_json = [] for item in rpn_ret['rpn_roi_probs']: score_2_json.append(item.item()) json.dump(score_2_json, f) # 开始第二个fast_head阶段,首先通过rois和bbox_delta计算pred_box if cfg.FAST_RCNN.FAST_HEAD2_DEBUG: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL if cfg.FPN.FPN_ON: im_scale = im_info.data.cpu().numpy().squeeze()[2] rois = rpn_ret['rois'][:, 1:5] / im_scale bbox_pred = bbox_pred.data.cpu().numpy().squeeze() box_deltas = bbox_pred.reshape([-1, bbox_pred.shape[-1]]) shift_boxes = box_utils.bbox_transform( rois, box_deltas, cfg.MODEL.BBOX_REG_WEIGHTS) shift_boxes = box_utils.clip_tiled_boxes( shift_boxes, im_info.data.cpu().numpy().squeeze()[0:2]) num_classes = cfg.MODEL.NUM_CLASSES onecls_pred_boxes = [] onecls_score = [] dets_cls = {} count = 0 for j in range(1, num_classes): inds = np.where( cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0] boxes_j = shift_boxes[inds, j * 4:(j + 1) * 4] score_j = cls_score[inds, j] onecls_pred_boxes += boxes_j.tolist() onecls_score += score_j.tolist() dets_cls.update({j: []}) for k in range(len(boxes_j.tolist())): dets_cls[j].append(count) count += 1 assert count == len(onecls_pred_boxes) stage2_rois_score = np.array(onecls_score, dtype=np.float32) stage2_rois = np.array(onecls_pred_boxes, dtype=np.float32) # Redistribute stage2_rois using fpn_utils module provided functions # calculate by formula cls_tracker = {} if not stage2_rois.tolist(): stage1_pred_iou = stage2_rois_score.tolist() stage2_final_boxes = np.empty((0, )) stage2_final_score = np.empty((0, )) logger.info("Detections above threshold is null.") else: alter_rpn = {} unresize_stage2_rois = stage2_rois * im_scale # unresize_stage2_rois = np.concatenate((unresize_stage2_rois, unresized_rois[:, 1:5])) lvls = fpn_utils.map_rois_to_fpn_levels( unresize_stage2_rois, lvl_min, lvl_max) # TAG: We might need to visualize "stage2_rois" to make sure. rois_idx_order = np.empty((0, )) dummy_batch = np.zeros( (unresize_stage2_rois.shape[0], 1), dtype=np.float32) alter_rpn["rois"] = np.hstack( (dummy_batch, unresize_stage2_rois)).astype(np.float32, copy=False) # alter_rpn['rois'] = np.concatenate((alter_rpn['rois'], unresized_rois)) for output_idx, lvl in enumerate( range(lvl_min, lvl_max + 1)): idx_lvl = np.where(lvls == lvl)[0] rois_lvl = unresize_stage2_rois[idx_lvl, :] rois_idx_order = np.concatenate( (rois_idx_order, idx_lvl)) _ = np.zeros((rois_lvl.shape[0], 1), dtype=np.float32) alter_rpn['rois_fpn{}'.format(lvl)] = np.hstack( (_, rois_lvl)).astype(np.float32, copy=False) rois_idx_restore = np.argsort(rois_idx_order).astype( np.int32, copy=False) alter_rpn['rois_idx_restore_int32'] = rois_idx_restore # Go through 2nd stage of FPN and fast_head stage2_feat = self.Box_Head(blob_conv, alter_rpn) stage2_cls_score, stage2_bbox_pred = self.Box_Outs( stage2_feat) # Transform shift value to original one to get final pred boxes coordinates stage2_bbox_pred = stage2_bbox_pred.data.cpu().numpy( ).squeeze() stage2_cls_score = stage2_cls_score.data.cpu().numpy() stage2_box_deltas = stage2_bbox_pred.reshape( [-1, bbox_pred.shape[-1]]) # Add some variance to box delta if cfg.FAST_RCNN.STAGE1_TURBULENCE: import random for i in range(len(stage2_box_deltas)): for j in range(len(stage2_box_deltas[i])): stage2_box_deltas[i][j] *= random.uniform( 0.9, 1.1) stage2_cls_out = box_utils.bbox_transform( stage2_rois, stage2_box_deltas, cfg.MODEL.BBOX_REG_WEIGHTS) stage2_cls_out = box_utils.clip_tiled_boxes( stage2_cls_out, im_info.data.cpu().numpy().squeeze()[0:2]) onecls_pred_boxes = [] onecls_score = [] count = 0 for j in range(1, num_classes): inds = np.where( stage2_cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0] boxes_j = stage2_cls_out[inds, j * 4:(j + 1) * 4] score_j = stage2_cls_score[inds, j] dets_j = np.hstack( (boxes_j, score_j[:, np.newaxis])).astype(np.float32, copy=False) keep = box_utils.nms(dets_j, cfg.TEST.NMS) boxes_j = boxes_j[keep] score_j = score_j[keep] # 用于记录每个框属于第几类 onecls_score += score_j.tolist() onecls_pred_boxes += boxes_j.tolist() for k in range(len(score_j)): cls_tracker.update({count: j}) count += 1 assert count == len(onecls_score) stage2_final_boxes = np.array(onecls_pred_boxes, dtype=np.float32) stage2_final_score = np.array(onecls_score, dtype=np.float32) inds = np.where(stage2_final_score > 0.3)[0] # Filtered by keep index... preserve_stage2_final_boxes = copy.deepcopy( stage2_final_boxes) preserve_stage2_final_score = copy.deepcopy( stage2_final_score) stage2_final_boxes = stage2_final_boxes[inds] stage2_final_score = stage2_final_score[inds] # if nothing left after 0.3 threshold filter, reserve whole boxes to original. if stage2_final_boxes.size == 0: lower_inds = np.where( preserve_stage2_final_score > 0.1)[0] stage2_final_boxes = preserve_stage2_final_boxes[ lower_inds] stage2_final_score = preserve_stage2_final_score[ lower_inds] else: del preserve_stage2_final_boxes del preserve_stage2_final_score # if all boxes are clsfied into bg class. if stage2_final_boxes.size == 0: stage1_pred_iou = stage2_rois_score.tolist() stage2_final_boxes = np.empty((0, )) stage2_final_score = np.empty((0, )) logger.info("Detections above threshold is null.") else: # Restore stage2_pred_boxes to match the index with stage2_rois, Compute IOU between # final_boxes and stage2_rois, one by one flag = "cross_product" if flag == "element_wise": if stage2_final_boxes.shape[ 0] == stage2_rois.shape[0]: restored_stage2_final_boxes = stage2_final_boxes[ rois_idx_restore] stage1_pred_iou = [] for ind, item in enumerate(stage2_rois): stage1 = np.array( item, dtype=np.float32).reshape( (1, 4)) stage2 = np.array( restored_stage2_final_boxes[ind], dtype=np.float32).reshape((1, 4)) iou = box_utils.bbox_overlaps( stage1, stage2) stage1_pred_iou.append( iou.squeeze().item()) else: logger.info( "Mistake while processing {}".format( str(im_info))) elif flag == "cross_product": iou = box_utils.bbox_overlaps( stage2_rois, stage2_final_boxes) stage1_pred_iou = iou.max(axis=1).tolist() # stage1_pred is another name of stage2_rois assert len(stage1_pred_iou) == len(stage2_rois) if cfg.FAST_RCNN.IOU_NMS: with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_score.json", "w") as f: json.dump(stage2_rois_score.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage2_score.json", "w") as f: json.dump(stage2_final_score.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_pred_boxes.json", 'w') as f: json.dump(stage2_rois.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_pred_iou.json", 'w') as f: json.dump(stage1_pred_iou, f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage2_pred_boxes.json", 'w') as f: json.dump(stage2_final_boxes.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_dets_cls.json", 'w') as f: json.dump(dets_cls, f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_cls_tracker.json", 'w') as f: json.dump(cls_tracker, f) elif cfg.FAST_RCNN.SCORE_NMS: with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_score.json", "w") as f: json.dump(stage2_rois_score.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage2_score.json", "w") as f: json.dump(stage2_final_score.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_pred_boxes.json", 'w') as f: json.dump(stage2_rois.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_pred_iou.json", 'w') as f: json.dump(stage1_pred_iou, f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage2_pred_boxes.json", 'w') as f: json.dump(stage2_final_boxes.tolist(), f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_dets_cls.json", 'w') as f: json.dump(dets_cls, f) with open( "/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_cls_tracker.json", 'w') as f: json.dump(cls_tracker, f) else: im_scale = im_info.data.cpu().numpy().squeeze()[2] rois = rpn_ret['rois'][:, 1:5] / im_scale # unscale back to raw image space box_deltas = bbox_pred.data.cpu().numpy().squeeze() fast_stage1_score = cls_score.data.cpu().numpy().squeeze() box_deltas = box_deltas.reshape([-1, bbox_pred.shape[-1]]) stage2_rois = box_utils.bbox_transform( rois, box_deltas, cfg.MODEL.BBOX_REG_WEIGHTS) stage2_rois = box_utils.clip_tiled_boxes( stage2_rois, im_info.data.cpu().numpy().squeeze()[0:2]) num_classes = cfg.MODEL.NUM_CLASSES onecls_pred_boxes = [] onecls_cls_score = [] for j in range(1, num_classes): inds = np.where( cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0] boxes_j = stage2_rois[inds, j * 4:(j + 1) * 4] score_j = fast_stage1_score[inds, j] onecls_pred_boxes += boxes_j.tolist() onecls_cls_score += score_j.tolist() stage2_rois = np.array(onecls_pred_boxes, dtype=np.float32) stage2_rois_score = np.array(onecls_cls_score, dtype=np.float32) assert len(stage2_rois) == len(stage2_rois_score) # Send stage2 rois to next stage fast head, do ROI ALIGN again # to modify rpn_ret['rois] , rpn_ret['rpn_rois'] and rpn['rois_rpn_score'] rpn_ret['rois'] = stage2_rois rpn_ret['rpn_rois'] = stage2_rois rpn_ret['rpn_roi_probs'] = stage2_rois_score stage2_box_feat = self.Box_Head(blob_conv, rpn_ret) stage2_cls_score, stage2_bbox_pred = self.Box_Outs( stage2_box_feat) stage2_bbox_pred = stage2_bbox_pred.data.cpu().numpy( ).squeeze() stage2_bbox_pred = stage2_bbox_pred.reshape( [-1, bbox_pred.shape[-1]]) stage2_cls_pred_boxes = box_utils.bbox_transform( stage2_rois, stage2_bbox_pred, cfg.MODEL.BBOX_REG_WEIGHTS) stage2_cls_pred_boxes = box_utils.clip_tiled_boxes( stage2_cls_pred_boxes, im_info.data.cpu().numpy().squeeze()[0:2]) onecls_pred_boxes = [] onecls_cls_score = [] for j in range(1, num_classes): inds = np.where( stage2_cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0] if len(inds) != 0: print("KKKKK") boxes_j = stage2_cls_pred_boxes[inds, j * 4:(j + 1) * 4] score_j = stage2_cls_score[inds, j] onecls_pred_boxes += boxes_j.tolist() onecls_cls_score += score_j.tolist() stage2_bbox_pred = np.array(onecls_pred_boxes, dtype=np.float32) stage2_bbox_pred_score = np.array(onecls_cls_score, dtype=np.float32) # get stage2 pred_boxes here return_dict['cls_score'] = cls_score return_dict['bbox_pred'] = bbox_pred return return_dict
def forward(self, det_rois, det_labels, det_scores, im_info, dataset_name, roidb=None): """ det_rois: feature maps from the backbone network. (Variable) im_info: (CPU Variable) roidb: (list of ndarray) """ sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0]) obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0]) # remove self paired rois if det_rois.shape[ 0] > 1: # no pairs to remove when there is at most one detection sbj_inds, obj_inds = self.remove_self_pairs( det_rois.shape[0], sbj_inds, obj_inds) sbj_rois = det_rois[sbj_inds] obj_rois = det_rois[obj_inds] im_scale = im_info.data.numpy()[:, 2][0] sbj_boxes = sbj_rois[:, 1:] / im_scale obj_boxes = obj_rois[:, 1:] / im_scale # filters out those roi pairs whose boxes are not overlapping in the original scales if self.overlap: ovlp_so = box_utils_rel.bbox_pair_overlaps( sbj_boxes.astype(dtype=np.float32, copy=False), obj_boxes.astype(dtype=np.float32, copy=False)) ovlp_inds = np.where((ovlp_so > 0) & (ovlp_so < 0.5))[0] if ovlp_inds.size > 0: sbj_inds = sbj_inds[ovlp_inds] obj_inds = obj_inds[ovlp_inds] sbj_rois = sbj_rois[ovlp_inds] obj_rois = obj_rois[ovlp_inds] return_dict = {} sbj_labels = det_labels[sbj_inds] obj_labels = det_labels[obj_inds] sbj_scores = det_scores[sbj_inds] obj_scores = det_scores[obj_inds] rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois) return_dict['det_rois'] = det_rois return_dict['sbj_inds'] = sbj_inds return_dict['obj_inds'] = obj_inds return_dict['sbj_rois'] = sbj_rois return_dict['obj_rois'] = obj_rois return_dict['rel_rois'] = rel_rois return_dict['sbj_labels'] = sbj_labels return_dict['obj_labels'] = obj_labels return_dict['sbj_scores'] = sbj_scores return_dict['obj_scores'] = obj_scores return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32) if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS: lvl_min = cfg.FPN.ROI_MIN_LEVEL lvl_max = cfg.FPN.ROI_MAX_LEVEL # when use min_rel_area, the same sbj/obj area could be mapped to different feature levels # when they are associated with different relationships # Thus we cannot get det_rois features then gather sbj/obj features # The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois rois_blob_names = ['det_rois', 'rel_rois'] for rois_blob_name in rois_blob_names: # Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl> target_lvls = fpn_utils.map_rois_to_fpn_levels( return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max) fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name, return_dict[rois_blob_name], target_lvls, lvl_min, lvl_max) return return_dict