def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn_utils.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn_utils.add_multilevel_roi_blobs(blobs, rois_blob_name,
blobs[rois_blob_name], target_lvls,
lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn_utils.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max
)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn_utils.add_multilevel_roi_blobs(
blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min,
lvl_max
)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min,
lvl_max)
fpn_utils.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min,
lvl_max)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
blobs, name, blobs[name], lvls, lvl_min, lvl_max
)
def obj_feature_map(self, blob_conv, det_rois, use_relu=True):
fpn_ret = {'rois': det_rois}
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
return self.Box_Head_sg(blob_conv, fpn_ret, rois_name='rois', use_relu=use_relu)
def visual_rep(self, blob_conv_prd, rois, pair_inds, device_id, use_relu=False):
assert pair_inds.shape[1] == 2
rel_rois = box_utils_rel.rois_union(rois[pair_inds[:, 0]], rois[pair_inds[:, 1]])
rel_ret = {'sbj_rois': rois[pair_inds[:, 0]], 'obj_rois': rois[pair_inds[:, 1]],
'rel_rois': rel_rois}
union_mask = self.union_mask(rel_ret, device_id)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
return self.Box_Head_prd(blob_conv_prd, rel_ret, union_mask, rois_name='rel_rois', use_relu=use_relu)
def _forward(self,
data,
im_info,
do_vis=False,
dataset_name=None,
roidb=None,
use_gt_labels=False,
**rpn_kwargs):
im_data = data
if self.training:
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[
0] if self.training else cfg.TEST.DATASETS[
0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
if not cfg.MODEL.USE_REL_PYRAMID:
blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
if not cfg.MODEL.USE_REL_PYRAMID:
blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
else:
blob_conv_prd = self.RelPyramid(blob_conv)
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv,
rpn_ret,
use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0]
det_rois = rpn_ret['rois'][fg_inds]
det_labels = rpn_ret['labels_int32'][fg_inds]
det_scores = F.softmax(cls_score[fg_inds], dim=1)
rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info,
dataset_name, roidb)
if cfg.MODEL.ADD_SO_SCORES:
sbj_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
if cfg.MODEL.ADD_SO_SCORES:
# sbj
sbj_feat_sbj_pos = self.S_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_sbj_pos',
use_relu=use_relu)
obj_feat_sbj_pos = self.O_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_sbj_pos',
use_relu=use_relu)
# obj
sbj_feat_obj_pos = self.S_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_obj_pos',
use_relu=use_relu)
obj_feat_obj_pos = self.O_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_obj_pos',
use_relu=use_relu)
else:
# sbj
sbj_feat_sbj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_sbj_pos',
use_relu=use_relu)
obj_feat_sbj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_sbj_pos',
use_relu=use_relu)
# obj
sbj_feat_obj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_obj_pos',
use_relu=use_relu)
obj_feat_obj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_obj_pos',
use_relu=use_relu)
else:
if roidb is not None:
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb['sbj_gt_boxes']
obj_boxes = roidb['obj_gt_boxes']
sbj_rois = sbj_boxes * im_scale
obj_rois = obj_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(sbj_rois.shape[0], 1))
sbj_rois = np.hstack((repeated_batch_idx, sbj_rois))
obj_rois = np.hstack((repeated_batch_idx, obj_rois))
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
rel_ret = {}
rel_ret['sbj_rois'] = sbj_rois
rel_ret['obj_rois'] = obj_rois
rel_ret['rel_rois'] = rel_rois
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name],
target_lvls, lvl_min, lvl_max)
sbj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=True)
sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat)
sbj_cls_scores = sbj_cls_scores.data.cpu().numpy()
obj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=True)
obj_cls_scores, _ = self.Box_Outs(obj_det_feat)
obj_cls_scores = obj_cls_scores.data.cpu().numpy()
if use_gt_labels:
sbj_labels = roidb['sbj_gt_classes'] # start from 0
obj_labels = roidb['obj_gt_classes'] # start from 0
sbj_scores = np.ones_like(sbj_labels, dtype=np.float32)
obj_scores = np.ones_like(obj_labels, dtype=np.float32)
else:
sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1)
obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1)
sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1)
obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1)
rel_ret['sbj_scores'] = sbj_scores.astype(np.float32,
copy=False)
rel_ret['obj_scores'] = obj_scores.astype(np.float32,
copy=False)
rel_ret['sbj_labels'] = sbj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['obj_labels'] = obj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32,
copy=False)
rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32,
copy=False)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat = box_utils_rel.get_spt_features(
sbj_boxes, obj_boxes, im_w, im_h)
rel_ret['spt_feat'] = spt_feat
if cfg.MODEL.ADD_SO_SCORES:
sbj_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
rel_ret = self.RelPN(det_rois, det_labels, det_scores,
im_info, dataset_name, roidb)
valid_len = len(rel_ret['rel_rois'])
if valid_len > 0:
break
logger.info(
'Got {} rel_rois when score_thresh={}, changing to {}'.
format(valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
if cfg.MODEL.ADD_SO_SCORES:
det_s_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
det_o_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_s_feat[rel_ret['sbj_inds']]
obj_feat = det_o_feat[rel_ret['obj_inds']]
else:
det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_feat[rel_ret['sbj_inds']]
obj_feat = det_feat[rel_ret['obj_inds']]
rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd,
rel_ret,
rois_name='rel_rois',
use_relu=use_relu)
spo_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat = rel_ret['spt_feat']
else:
spt_feat = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels = rel_ret['all_sbj_labels_int32']
obj_labels = rel_ret['all_obj_labels_int32']
else:
sbj_labels = None
obj_labels = None
# prd_scores is the visual scores. See reldn_heads.py
prd_scores, prd_bias_scores, prd_spt_scores, ttl_cls_scores, sbj_cls_scores, obj_cls_scores = \
self.RelDN(spo_feat, spt_feat, sbj_labels, obj_labels, sbj_feat, obj_feat)
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
# rpn loss
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
if cfg.MODEL.USE_FREQ_BIAS and not cfg.MODEL.ADD_SCORES_ALL:
loss_cls_bias, accuracy_cls_bias = reldn_heads.reldn_losses(
prd_bias_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_bias'] = loss_cls_bias
return_dict['metrics']['accuracy_cls_bias'] = accuracy_cls_bias
if cfg.MODEL.USE_SPATIAL_FEAT and not cfg.MODEL.ADD_SCORES_ALL:
loss_cls_spt, accuracy_cls_spt = reldn_heads.reldn_losses(
prd_spt_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_spt'] = loss_cls_spt
return_dict['metrics']['accuracy_cls_spt'] = accuracy_cls_spt
if cfg.MODEL.ADD_SCORES_ALL:
loss_cls_ttl, accuracy_cls_ttl = reldn_heads.reldn_losses(
ttl_cls_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_ttl'] = loss_cls_ttl
return_dict['metrics']['accuracy_cls_ttl'] = accuracy_cls_ttl
else:
loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses(
prd_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_prd'] = loss_cls_prd
return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
# sbj
rel_feat_sbj_pos = self.Prd_RCNN.Box_Head(
blob_conv_prd,
rel_ret,
rois_name='rel_rois_sbj_pos',
use_relu=use_relu)
spo_feat_sbj_pos = torch.cat(
(sbj_feat_sbj_pos, rel_feat_sbj_pos, obj_feat_sbj_pos),
dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat_sbj_pos = rel_ret['spt_feat_sbj_pos']
else:
spt_feat_sbj_pos = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels_sbj_pos_fg = rel_ret[
'sbj_labels_sbj_pos_fg_int32']
obj_labels_sbj_pos_fg = rel_ret[
'obj_labels_sbj_pos_fg_int32']
else:
sbj_labels_sbj_pos_fg = None
obj_labels_sbj_pos_fg = None
_, prd_bias_scores_sbj_pos, _, ttl_cls_scores_sbj_pos, _, _ = \
self.RelDN(spo_feat_sbj_pos, spt_feat_sbj_pos, sbj_labels_sbj_pos_fg, obj_labels_sbj_pos_fg, sbj_feat_sbj_pos, obj_feat_sbj_pos)
# obj
rel_feat_obj_pos = self.Prd_RCNN.Box_Head(
blob_conv_prd,
rel_ret,
rois_name='rel_rois_obj_pos',
use_relu=use_relu)
spo_feat_obj_pos = torch.cat(
(sbj_feat_obj_pos, rel_feat_obj_pos, obj_feat_obj_pos),
dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat_obj_pos = rel_ret['spt_feat_obj_pos']
else:
spt_feat_obj_pos = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels_obj_pos_fg = rel_ret[
'sbj_labels_obj_pos_fg_int32']
obj_labels_obj_pos_fg = rel_ret[
'obj_labels_obj_pos_fg_int32']
else:
sbj_labels_obj_pos_fg = None
obj_labels_obj_pos_fg = None
_, prd_bias_scores_obj_pos, _, ttl_cls_scores_obj_pos, _, _ = \
self.RelDN(spo_feat_obj_pos, spt_feat_obj_pos, sbj_labels_obj_pos_fg, obj_labels_obj_pos_fg, sbj_feat_obj_pos, obj_feat_obj_pos)
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS:
loss_contrastive_sbj, loss_contrastive_obj = reldn_heads.reldn_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_contrastive_sbj'] = loss_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT
return_dict['losses'][
'loss_contrastive_obj'] = loss_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT
if cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS:
loss_so_contrastive_sbj, loss_so_contrastive_obj = reldn_heads.reldn_so_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_so_contrastive_sbj'] = loss_so_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT
return_dict['losses'][
'loss_so_contrastive_obj'] = loss_so_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT
if cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
loss_p_contrastive_sbj, loss_p_contrastive_obj = reldn_heads.reldn_p_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
prd_bias_scores_sbj_pos, prd_bias_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_p_contrastive_sbj'] = loss_p_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT
return_dict['losses'][
'loss_p_contrastive_obj'] = loss_p_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
# Testing
return_dict['sbj_rois'] = rel_ret['sbj_rois']
return_dict['obj_rois'] = rel_ret['obj_rois']
return_dict['sbj_labels'] = rel_ret['sbj_labels']
return_dict['obj_labels'] = rel_ret['obj_labels']
return_dict['sbj_scores'] = rel_ret['sbj_scores']
return_dict['obj_scores'] = rel_ret['obj_scores']
return_dict['prd_scores'] = prd_scores
if cfg.MODEL.USE_FREQ_BIAS:
return_dict['prd_scores_bias'] = prd_bias_scores
if cfg.MODEL.USE_SPATIAL_FEAT:
return_dict['prd_scores_spt'] = prd_spt_scores
if cfg.MODEL.ADD_SCORES_ALL:
return_dict['prd_ttl_scores'] = ttl_cls_scores
if do_vis:
return_dict['blob_conv'] = blob_conv
return_dict['blob_conv_prd'] = blob_conv_prd
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
# Get pairwise proposals first
if roidb is not None:
# we always feed one image per batch during training
assert len(roidb) == 1
sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0])
obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0])
# remove self paired rois
if det_rois.shape[
0] > 1: # no pairs to remove when there is at most one detection
sbj_inds, obj_inds = self.remove_self_pairs(
det_rois.shape[0], sbj_inds, obj_inds)
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
if cfg.MODEL.USE_OVLP_FILTER:
ovlp_so = box_utils.bbox_pair_overlaps(
sbj_boxes.astype(dtype=np.float32, copy=False),
obj_boxes.astype(dtype=np.float32, copy=False))
ovlp_inds = np.where(ovlp_so > 0)[0]
sbj_inds = sbj_inds[ovlp_inds]
obj_inds = obj_inds[ovlp_inds]
sbj_rois = sbj_rois[ovlp_inds]
obj_rois = obj_rois[ovlp_inds]
sbj_boxes = sbj_boxes[ovlp_inds]
obj_boxes = obj_boxes[ovlp_inds]
return_dict = {}
if self.training:
# Add binary relationships
blobs_out = self.RelPN_GenerateProposalLabels(
sbj_rois, obj_rois, det_rois, roidb, im_info)
return_dict.update(blobs_out)
else:
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]],
dtype=np.int32)
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT:
return_dict['all_sbj_labels_int32'] = sbj_labels.astype(
np.int32, copy=False) - 1 # det_labels start from 1
return_dict['all_obj_labels_int32'] = obj_labels.astype(
np.int32, copy=False) - 1 # det_labels start from 1
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
return_dict, rois_blob_name,
return_dict[rois_blob_name], target_lvls, lvl_min,
lvl_max)
return return_dict
def _forward(self,
data,
im_info,
dataset_name=None,
roidb=None,
use_gt_labels=False,
include_feat=False,
**rpn_kwargs):
im_data = data
if self.training:
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[
0] if self.training else cfg.TEST.DATASETS[
0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
if not cfg.TRAIN.USE_GT_BOXES:
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv,
rpn_ret,
use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
if cfg.TRAIN.USE_GT_BOXES:
# we always feed one image per batch during training
assert len(roidb) == 1
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb[0]['sbj_gt_boxes']
obj_boxes = roidb[0]['obj_gt_boxes']
sbj_all_boxes = _augment_gt_boxes_by_perturbation(
sbj_boxes, im_w, im_h)
obj_all_boxes = _augment_gt_boxes_by_perturbation(
obj_boxes, im_w, im_h)
det_all_boxes = np.vstack((sbj_all_boxes, obj_all_boxes))
det_all_boxes = np.unique(det_all_boxes, axis=0)
det_all_rois = det_all_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(det_all_rois.shape[0], 1))
det_all_rois = np.hstack((repeated_batch_idx, det_all_rois))
rel_ret = self.RelPN(det_all_rois, None, None, im_info,
dataset_name, roidb)
else:
fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0]
det_rois = rpn_ret['rois'][fg_inds]
det_labels = rpn_ret['labels_int32'][fg_inds]
det_scores = F.softmax(cls_score[fg_inds], dim=1)
rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info,
dataset_name, roidb)
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
if roidb is not None:
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb['sbj_gt_boxes']
obj_boxes = roidb['obj_gt_boxes']
sbj_rois = sbj_boxes * im_scale
obj_rois = obj_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(sbj_rois.shape[0], 1))
sbj_rois = np.hstack((repeated_batch_idx, sbj_rois))
obj_rois = np.hstack((repeated_batch_idx, obj_rois))
rel_rois = box_utils.rois_union(sbj_rois, obj_rois)
rel_ret = {}
rel_ret['sbj_rois'] = sbj_rois
rel_ret['obj_rois'] = obj_rois
rel_ret['rel_rois'] = rel_rois
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name],
target_lvls, lvl_min, lvl_max)
if use_gt_labels:
sbj_labels = roidb['sbj_gt_classes'] # start from 0
obj_labels = roidb['obj_gt_classes'] # start from 0
sbj_scores = np.ones_like(sbj_labels, dtype=np.float32)
obj_scores = np.ones_like(obj_labels, dtype=np.float32)
else:
sbj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=True)
sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat)
sbj_cls_scores = sbj_cls_scores.data.cpu().numpy()
obj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=True)
obj_cls_scores, _ = self.Box_Outs(obj_det_feat)
obj_cls_scores = obj_cls_scores.data.cpu().numpy()
sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1)
obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1)
sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1)
obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1)
rel_ret['sbj_scores'] = sbj_scores.astype(np.float32,
copy=False)
rel_ret['obj_scores'] = obj_scores.astype(np.float32,
copy=False)
rel_ret['sbj_labels'] = sbj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['obj_labels'] = obj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32,
copy=False)
rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32,
copy=False)
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
rel_ret = self.RelPN(det_rois, det_labels, det_scores,
im_info, dataset_name, roidb)
valid_len = len(rel_ret['rel_rois'])
if valid_len > 0:
break
logger.info(
'Got {} rel_rois when score_thresh={}, changing to {}'.
format(valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_feat[rel_ret['sbj_inds']]
obj_feat = det_feat[rel_ret['obj_inds']]
rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd,
rel_ret,
rois_name='rel_rois',
use_relu=use_relu)
concat_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1)
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT:
sbj_labels = rel_ret['all_sbj_labels_int32']
obj_labels = rel_ret['all_obj_labels_int32']
else:
sbj_labels = None
obj_labels = None
# when MODEL.USE_SEM_CONCAT, memory runs out if the whole batch is fed once
# so we need to feed the batch twice if it's big
gn_size = 1000
if cfg.MODEL.USE_SEM_CONCAT and concat_feat.shape[0] > gn_size:
group = int(math.ceil(concat_feat.shape[0] / gn_size))
prd_cls_scores = None
sbj_cls_scores = None
obj_cls_scores = None
for i in range(group):
end = int(min((i + 1) * gn_size, concat_feat.shape[0]))
concat_feat_i = concat_feat[i * gn_size:end]
sbj_labels_i = sbj_labels[
i * gn_size:end] if sbj_labels is not None else None
obj_labels_i = obj_labels[
i * gn_size:end] if obj_labels is not None else None
sbj_feat_i = sbj_feat[i * gn_size:end]
obj_feat_i = obj_feat[i * gn_size:end]
prd_cls_scores_i, sbj_cls_scores_i, obj_cls_scores_i = \
self.RelDN(concat_feat_i, sbj_labels_i, obj_labels_i, sbj_feat_i, obj_feat_i)
if prd_cls_scores is None:
prd_cls_scores = prd_cls_scores_i
sbj_cls_scores = sbj_cls_scores_i
obj_cls_scores = obj_cls_scores_i
else:
prd_cls_scores = torch.cat(
(prd_cls_scores, prd_cls_scores_i))
sbj_cls_scores = torch.cat(
(sbj_cls_scores, sbj_cls_scores_i
)) if sbj_cls_scores_i is not None else sbj_cls_scores
obj_cls_scores = torch.cat(
(obj_cls_scores, obj_cls_scores_i
)) if obj_cls_scores_i is not None else obj_cls_scores
else:
prd_cls_scores, sbj_cls_scores, obj_cls_scores = \
self.RelDN(concat_feat, sbj_labels, obj_labels, sbj_feat, obj_feat)
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
if not cfg.TRAIN.USE_GT_BOXES:
# rpn loss
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses(
prd_cls_scores,
rel_ret['all_prd_labels_int32'],
weight=self.prd_weights)
return_dict['losses']['loss_cls_prd'] = loss_cls_prd
return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd
if cfg.MODEL.USE_SEPARATE_SO_SCORES:
loss_cls_sbj, accuracy_cls_sbj = reldn_heads.reldn_losses(
sbj_cls_scores,
rel_ret['all_sbj_labels_int32'],
weight=self.obj_weights)
return_dict['losses']['loss_cls_sbj'] = loss_cls_sbj
return_dict['metrics']['accuracy_cls_sbj'] = accuracy_cls_sbj
loss_cls_obj, accuracy_cls_obj = reldn_heads.reldn_losses(
obj_cls_scores,
rel_ret['all_obj_labels_int32'],
weight=self.obj_weights)
return_dict['losses']['loss_cls_obj'] = loss_cls_obj
return_dict['metrics']['accuracy_cls_obj'] = accuracy_cls_obj
if cfg.TRAIN.HUBNESS:
loss_hubness_prd = reldn_heads.add_hubness_loss(prd_cls_scores)
loss_hubness_sbj = reldn_heads.add_hubness_loss(sbj_cls_scores)
loss_hubness_obj = reldn_heads.add_hubness_loss(obj_cls_scores)
return_dict['losses']['loss_hubness_prd'] = loss_hubness_prd
return_dict['losses']['loss_hubness_sbj'] = loss_hubness_sbj
return_dict['losses']['loss_hubness_obj'] = loss_hubness_obj
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
# Testing
return_dict['sbj_rois'] = rel_ret['sbj_rois']
return_dict['obj_rois'] = rel_ret['obj_rois']
return_dict['sbj_labels'] = rel_ret['sbj_labels']
return_dict['obj_labels'] = rel_ret['obj_labels']
return_dict['sbj_scores'] = rel_ret['sbj_scores']
return_dict['sbj_scores_out'] = sbj_cls_scores
return_dict['obj_scores'] = rel_ret['obj_scores']
return_dict['obj_scores_out'] = obj_cls_scores
return_dict['prd_scores'] = prd_cls_scores
if include_feat:
return_dict['sbj_feat'] = sbj_feat
return_dict['obj_feat'] = obj_feat
return_dict['prd_feat'] = concat_feat
return return_dict
def _forward(self, data, im_info, do_vis=False, dataset_name=None, roidb=None, use_gt_labels=False, **rpn_kwargs):
im_data = data
if self.training:
# if not isinstance(roidb[0], np.array):
# roidb = roidb[0]
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) # only support one gpu
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[0] if self.training else cfg.TEST.DATASETS[0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
# if not cfg.MODEL.USE_REL_PYRAMID:
# blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
if self.training:
gt_rois = np.empty((0, 5), dtype=np.float32)
gt_classes = np.empty((0), dtype=np.int64)
for i, r in enumerate(roidb):
rois_i = r['boxes'] * im_info[i, 2]
rois_i = np.hstack((i * blob_utils.ones((rois_i.shape[0], 1)), rois_i))
gt_rois = np.append(gt_rois, rois_i, axis=0)
gt_classes = np.append(gt_classes, r['gt_classes'], axis=0)
if self.training or roidb is None:
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
# if not cfg.MODEL.USE_REL_PYRAMID:
# blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
# else:
# blob_conv_prd = self.RelPyramid(blob_conv)
if self.training or roidb is None:
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
score_thresh = cfg.TEST.SCORE_THRESH
cls_score = F.softmax(cls_score, -1)
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores, det_dists, det_boxes_all = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2])
non_zero_area_inds = np.where(real_area > 0)[0]
det_rois = det_rois[non_zero_area_inds]
det_labels = det_labels[non_zero_area_inds]
det_scores = det_scores[non_zero_area_inds]
det_dists = det_dists[non_zero_area_inds]
det_boxes_all = det_boxes_all[non_zero_area_inds]
# rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb)
valid_len = len(det_rois)
if valid_len > 0:
break
logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format(
valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
det_labels_gt = []
ious = box_utils.bbox_overlaps(det_rois[:, 1:], gt_rois[:, 1:]) * \
(det_rois[:, 0][:,None] == gt_rois[:, 0][None, :])
det_labels_gt = gt_classes[ious.argmax(-1)]
det_labels_gt[ious.max(-1) < cfg.TRAIN.FG_THRESH] = 0
else:
if roidb is not None:
# raise FError('not support this mode!')
# assert len(roidb) == 1
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
fpn_ret = {'gt_rois': gt_rois}
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['gt_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
det_feats = self.Box_Head(blob_conv, fpn_ret, rois_name='det_rois', use_relu=True)
det_dists, _ = self.Box_Outs(det_feats)
det_boxes_all = None
if use_gt_labels:
det_labels_gt = gt_classes
det_labels = gt_classes
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores, det_dists, det_boxes_all = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2])
non_zero_area_inds = np.where(real_area > 0)[0]
det_rois = det_rois[non_zero_area_inds]
det_labels = det_labels[non_zero_area_inds]
det_scores = det_scores[non_zero_area_inds]
det_dists = det_dists[non_zero_area_inds]
det_boxes_all = det_boxes_all[non_zero_area_inds]
# rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb)
valid_len = len(det_rois)
if valid_len > 0:
break
logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format(
valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
return_dict['det_rois'] = det_rois
num_rois = det_rois.shape[0]
if not isinstance(det_dists, torch.Tensor):
assert det_dists.shape[0] == num_rois
det_dists = torch.from_numpy(det_dists).float().cuda(device_id)
return_dict['det_dists'] = det_dists
return_dict['det_scores'] = det_scores
return_dict['blob_conv'] = blob_conv
return_dict['det_boxes_all'] = det_boxes_all
assert det_boxes_all.shape[0] == num_rois
return_dict['det_labels'] = det_labels
# return_dict['blob_conv_prd'] = blob_conv_prd
if self.training or use_gt_labels:
return_dict['det_labels_gt'] = det_labels_gt
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
# Get pairwise proposals first
im_inds = det_rois[:, 0]
is_cand = im_inds[:, None] == im_inds[None, :]
is_cand.reshape(-1)[diagonal_inds(is_cand)] = False
is_empty = np.where(is_cand.any(1) == 0)[0]
if self.overlap:
is_cand = is_cand & (box_utils.bbox_overlaps(
det_rois[:, 1:], det_rois[:, 1:]) > 0)
if is_empty.size > 0:
is_cand[is_empty, is_empty] = True
sbj_inds, obj_inds = np.where(is_cand)
# remove self paired rois
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
return_dict = {}
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name,
return_dict[rois_blob_name],
target_lvls, lvl_min,
lvl_max)
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0])
obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0])
# remove self paired rois
if det_rois.shape[
0] > 1: # no pairs to remove when there is at most one detection
sbj_inds, obj_inds = self.remove_self_pairs(
det_rois.shape[0], sbj_inds, obj_inds)
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
if self.overlap:
ovlp_so = box_utils_rel.bbox_pair_overlaps(
sbj_boxes.astype(dtype=np.float32, copy=False),
obj_boxes.astype(dtype=np.float32, copy=False))
ovlp_inds = np.where((ovlp_so > 0) & (ovlp_so < 0.5))[0]
if ovlp_inds.size > 0:
sbj_inds = sbj_inds[ovlp_inds]
obj_inds = obj_inds[ovlp_inds]
sbj_rois = sbj_rois[ovlp_inds]
obj_rois = obj_rois[ovlp_inds]
return_dict = {}
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name,
return_dict[rois_blob_name],
target_lvls, lvl_min,
lvl_max)
return return_dict