def distribute(rois, label_blobs):
"""To understand the output blob order see return value of
roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
# Delete roi entries that have negative area
# idx_neg = np.where(lvls == -1)[0]
# rois = np.delete(rois, idx_neg, axis=0)
# lvls = np.delete(lvls, idx_neg, axis=0)
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
outputs = [None] * len(output_blob_names)
outputs[0] = rois
# Create new roi blobs for each FPN level
# (See: utils.fpn.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
outputs[-1] = rois_idx_restore.astype(np.int32)
return dict(zip(output_blob_names, outputs))
def distribute(rois, label_blobs):
"""To understand the output blob order see return value of
roi_data.fast_rcnn.get_fast_rcnn_blob_names(is_training=False)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(rois[:, 1:5], lvl_min, lvl_max)
# Delete roi entries that have negative area
# idx_neg = np.where(lvls == -1)[0]
# rois = np.delete(rois, idx_neg, axis=0)
# lvls = np.delete(lvls, idx_neg, axis=0)
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names(
is_training=False)
outputs = [None] * len(output_blob_names)
outputs[0] = rois
# Create new roi blobs for each FPN level
# (See: utils.fpn.add_multilevel_roi_blobs which is similar but annoying
# to generalize to support this particular case.)
rois_idx_order = np.empty((0, ))
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
blob_roi_level = rois[idx_lvl, :]
outputs[output_idx + 1] = blob_roi_level
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
rois_idx_restore = np.argsort(rois_idx_order)
outputs[-1] = rois_idx_restore.astype(np.int32)
return dict(zip(output_blob_names, outputs))
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn_utils.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn_utils.add_multilevel_roi_blobs(blobs, rois_blob_name,
blobs[rois_blob_name], target_lvls,
lvl_min, lvl_max)
def _distribute_rois_over_fpn_levels(rois_blob_name):
"""Distribute rois over the different FPN levels."""
# Get target level for each roi
# Recall blob rois are in (batch_idx, x1, y1, x2, y2) format, hence take
# the box coordinates from columns 1:5
target_lvls = fpn_utils.map_rois_to_fpn_levels(
blobs[rois_blob_name][:, 1:5], lvl_min, lvl_max
)
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
fpn_utils.add_multilevel_roi_blobs(
blobs, rois_blob_name, blobs[rois_blob_name], target_lvls, lvl_min,
lvl_max
)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min,
lvl_max)
fpn_utils.add_multilevel_roi_blobs(blobs, name, blobs[name], lvls, lvl_min,
lvl_max)
def _add_multilevel_rois_for_test(blobs, name):
"""Distributes a set of RoIs across FPN pyramid levels by creating new level
specific RoI blobs.
Arguments:
blobs (dict): dictionary of blobs
name (str): a key in 'blobs' identifying the source RoI blob
Returns:
[by ref] blobs (dict): new keys named by `name + 'fpn' + level`
are added to dict each with a value that's an R_level x 5 ndarray of
RoIs (see _get_rois_blob for format)
"""
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
lvls = fpn_utils.map_rois_to_fpn_levels(blobs[name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
blobs, name, blobs[name], lvls, lvl_min, lvl_max
)
def obj_feature_map(self, blob_conv, det_rois, use_relu=True):
fpn_ret = {'rois': det_rois}
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
return self.Box_Head_sg(blob_conv, fpn_ret, rois_name='rois', use_relu=use_relu)
def visual_rep(self, blob_conv_prd, rois, pair_inds, device_id, use_relu=False):
assert pair_inds.shape[1] == 2
rel_rois = box_utils_rel.rois_union(rois[pair_inds[:, 0]], rois[pair_inds[:, 1]])
rel_ret = {'sbj_rois': rois[pair_inds[:, 0]], 'obj_rois': rois[pair_inds[:, 1]],
'rel_rois': rel_rois}
union_mask = self.union_mask(rel_ret, device_id)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
return self.Box_Head_prd(blob_conv_prd, rel_ret, union_mask, rois_name='rel_rois', use_relu=use_relu)
def _forward(self,
data,
im_info,
do_vis=False,
dataset_name=None,
roidb=None,
use_gt_labels=False,
**rpn_kwargs):
im_data = data
if self.training:
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[
0] if self.training else cfg.TEST.DATASETS[
0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
if not cfg.MODEL.USE_REL_PYRAMID:
blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
if not cfg.MODEL.USE_REL_PYRAMID:
blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
else:
blob_conv_prd = self.RelPyramid(blob_conv)
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv,
rpn_ret,
use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0]
det_rois = rpn_ret['rois'][fg_inds]
det_labels = rpn_ret['labels_int32'][fg_inds]
det_scores = F.softmax(cls_score[fg_inds], dim=1)
rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info,
dataset_name, roidb)
if cfg.MODEL.ADD_SO_SCORES:
sbj_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
if cfg.MODEL.ADD_SO_SCORES:
# sbj
sbj_feat_sbj_pos = self.S_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_sbj_pos',
use_relu=use_relu)
obj_feat_sbj_pos = self.O_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_sbj_pos',
use_relu=use_relu)
# obj
sbj_feat_obj_pos = self.S_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_obj_pos',
use_relu=use_relu)
obj_feat_obj_pos = self.O_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_obj_pos',
use_relu=use_relu)
else:
# sbj
sbj_feat_sbj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_sbj_pos',
use_relu=use_relu)
obj_feat_sbj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_sbj_pos',
use_relu=use_relu)
# obj
sbj_feat_obj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='sbj_rois_obj_pos',
use_relu=use_relu)
obj_feat_obj_pos = self.Box_Head(
blob_conv,
rel_ret,
rois_name='obj_rois_obj_pos',
use_relu=use_relu)
else:
if roidb is not None:
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb['sbj_gt_boxes']
obj_boxes = roidb['obj_gt_boxes']
sbj_rois = sbj_boxes * im_scale
obj_rois = obj_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(sbj_rois.shape[0], 1))
sbj_rois = np.hstack((repeated_batch_idx, sbj_rois))
obj_rois = np.hstack((repeated_batch_idx, obj_rois))
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
rel_ret = {}
rel_ret['sbj_rois'] = sbj_rois
rel_ret['obj_rois'] = obj_rois
rel_ret['rel_rois'] = rel_rois
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name],
target_lvls, lvl_min, lvl_max)
sbj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=True)
sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat)
sbj_cls_scores = sbj_cls_scores.data.cpu().numpy()
obj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=True)
obj_cls_scores, _ = self.Box_Outs(obj_det_feat)
obj_cls_scores = obj_cls_scores.data.cpu().numpy()
if use_gt_labels:
sbj_labels = roidb['sbj_gt_classes'] # start from 0
obj_labels = roidb['obj_gt_classes'] # start from 0
sbj_scores = np.ones_like(sbj_labels, dtype=np.float32)
obj_scores = np.ones_like(obj_labels, dtype=np.float32)
else:
sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1)
obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1)
sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1)
obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1)
rel_ret['sbj_scores'] = sbj_scores.astype(np.float32,
copy=False)
rel_ret['obj_scores'] = obj_scores.astype(np.float32,
copy=False)
rel_ret['sbj_labels'] = sbj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['obj_labels'] = obj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32,
copy=False)
rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32,
copy=False)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat = box_utils_rel.get_spt_features(
sbj_boxes, obj_boxes, im_w, im_h)
rel_ret['spt_feat'] = spt_feat
if cfg.MODEL.ADD_SO_SCORES:
sbj_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
rel_ret = self.RelPN(det_rois, det_labels, det_scores,
im_info, dataset_name, roidb)
valid_len = len(rel_ret['rel_rois'])
if valid_len > 0:
break
logger.info(
'Got {} rel_rois when score_thresh={}, changing to {}'.
format(valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
if cfg.MODEL.ADD_SO_SCORES:
det_s_feat = self.S_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
det_o_feat = self.O_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_s_feat[rel_ret['sbj_inds']]
obj_feat = det_o_feat[rel_ret['obj_inds']]
else:
det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_feat[rel_ret['sbj_inds']]
obj_feat = det_feat[rel_ret['obj_inds']]
rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd,
rel_ret,
rois_name='rel_rois',
use_relu=use_relu)
spo_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat = rel_ret['spt_feat']
else:
spt_feat = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels = rel_ret['all_sbj_labels_int32']
obj_labels = rel_ret['all_obj_labels_int32']
else:
sbj_labels = None
obj_labels = None
# prd_scores is the visual scores. See reldn_heads.py
prd_scores, prd_bias_scores, prd_spt_scores, ttl_cls_scores, sbj_cls_scores, obj_cls_scores = \
self.RelDN(spo_feat, spt_feat, sbj_labels, obj_labels, sbj_feat, obj_feat)
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
# rpn loss
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
if cfg.MODEL.USE_FREQ_BIAS and not cfg.MODEL.ADD_SCORES_ALL:
loss_cls_bias, accuracy_cls_bias = reldn_heads.reldn_losses(
prd_bias_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_bias'] = loss_cls_bias
return_dict['metrics']['accuracy_cls_bias'] = accuracy_cls_bias
if cfg.MODEL.USE_SPATIAL_FEAT and not cfg.MODEL.ADD_SCORES_ALL:
loss_cls_spt, accuracy_cls_spt = reldn_heads.reldn_losses(
prd_spt_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_spt'] = loss_cls_spt
return_dict['metrics']['accuracy_cls_spt'] = accuracy_cls_spt
if cfg.MODEL.ADD_SCORES_ALL:
loss_cls_ttl, accuracy_cls_ttl = reldn_heads.reldn_losses(
ttl_cls_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_ttl'] = loss_cls_ttl
return_dict['metrics']['accuracy_cls_ttl'] = accuracy_cls_ttl
else:
loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses(
prd_scores, rel_ret['all_prd_labels_int32'])
return_dict['losses']['loss_cls_prd'] = loss_cls_prd
return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS or cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
# sbj
rel_feat_sbj_pos = self.Prd_RCNN.Box_Head(
blob_conv_prd,
rel_ret,
rois_name='rel_rois_sbj_pos',
use_relu=use_relu)
spo_feat_sbj_pos = torch.cat(
(sbj_feat_sbj_pos, rel_feat_sbj_pos, obj_feat_sbj_pos),
dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat_sbj_pos = rel_ret['spt_feat_sbj_pos']
else:
spt_feat_sbj_pos = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels_sbj_pos_fg = rel_ret[
'sbj_labels_sbj_pos_fg_int32']
obj_labels_sbj_pos_fg = rel_ret[
'obj_labels_sbj_pos_fg_int32']
else:
sbj_labels_sbj_pos_fg = None
obj_labels_sbj_pos_fg = None
_, prd_bias_scores_sbj_pos, _, ttl_cls_scores_sbj_pos, _, _ = \
self.RelDN(spo_feat_sbj_pos, spt_feat_sbj_pos, sbj_labels_sbj_pos_fg, obj_labels_sbj_pos_fg, sbj_feat_sbj_pos, obj_feat_sbj_pos)
# obj
rel_feat_obj_pos = self.Prd_RCNN.Box_Head(
blob_conv_prd,
rel_ret,
rois_name='rel_rois_obj_pos',
use_relu=use_relu)
spo_feat_obj_pos = torch.cat(
(sbj_feat_obj_pos, rel_feat_obj_pos, obj_feat_obj_pos),
dim=1)
if cfg.MODEL.USE_SPATIAL_FEAT:
spt_feat_obj_pos = rel_ret['spt_feat_obj_pos']
else:
spt_feat_obj_pos = None
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE:
sbj_labels_obj_pos_fg = rel_ret[
'sbj_labels_obj_pos_fg_int32']
obj_labels_obj_pos_fg = rel_ret[
'obj_labels_obj_pos_fg_int32']
else:
sbj_labels_obj_pos_fg = None
obj_labels_obj_pos_fg = None
_, prd_bias_scores_obj_pos, _, ttl_cls_scores_obj_pos, _, _ = \
self.RelDN(spo_feat_obj_pos, spt_feat_obj_pos, sbj_labels_obj_pos_fg, obj_labels_obj_pos_fg, sbj_feat_obj_pos, obj_feat_obj_pos)
if cfg.MODEL.USE_NODE_CONTRASTIVE_LOSS:
loss_contrastive_sbj, loss_contrastive_obj = reldn_heads.reldn_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_contrastive_sbj'] = loss_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT
return_dict['losses'][
'loss_contrastive_obj'] = loss_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_WEIGHT
if cfg.MODEL.USE_NODE_CONTRASTIVE_SO_AWARE_LOSS:
loss_so_contrastive_sbj, loss_so_contrastive_obj = reldn_heads.reldn_so_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_so_contrastive_sbj'] = loss_so_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT
return_dict['losses'][
'loss_so_contrastive_obj'] = loss_so_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_SO_AWARE_WEIGHT
if cfg.MODEL.USE_NODE_CONTRASTIVE_P_AWARE_LOSS:
loss_p_contrastive_sbj, loss_p_contrastive_obj = reldn_heads.reldn_p_contrastive_losses(
ttl_cls_scores_sbj_pos, ttl_cls_scores_obj_pos,
prd_bias_scores_sbj_pos, prd_bias_scores_obj_pos,
rel_ret)
return_dict['losses'][
'loss_p_contrastive_sbj'] = loss_p_contrastive_sbj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT
return_dict['losses'][
'loss_p_contrastive_obj'] = loss_p_contrastive_obj * cfg.MODEL.NODE_CONTRASTIVE_P_AWARE_WEIGHT
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
# Testing
return_dict['sbj_rois'] = rel_ret['sbj_rois']
return_dict['obj_rois'] = rel_ret['obj_rois']
return_dict['sbj_labels'] = rel_ret['sbj_labels']
return_dict['obj_labels'] = rel_ret['obj_labels']
return_dict['sbj_scores'] = rel_ret['sbj_scores']
return_dict['obj_scores'] = rel_ret['obj_scores']
return_dict['prd_scores'] = prd_scores
if cfg.MODEL.USE_FREQ_BIAS:
return_dict['prd_scores_bias'] = prd_bias_scores
if cfg.MODEL.USE_SPATIAL_FEAT:
return_dict['prd_scores_spt'] = prd_spt_scores
if cfg.MODEL.ADD_SCORES_ALL:
return_dict['prd_ttl_scores'] = ttl_cls_scores
if do_vis:
return_dict['blob_conv'] = blob_conv
return_dict['blob_conv_prd'] = blob_conv_prd
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
# Get pairwise proposals first
if roidb is not None:
# we always feed one image per batch during training
assert len(roidb) == 1
sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0])
obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0])
# remove self paired rois
if det_rois.shape[
0] > 1: # no pairs to remove when there is at most one detection
sbj_inds, obj_inds = self.remove_self_pairs(
det_rois.shape[0], sbj_inds, obj_inds)
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
if cfg.MODEL.USE_OVLP_FILTER:
ovlp_so = box_utils.bbox_pair_overlaps(
sbj_boxes.astype(dtype=np.float32, copy=False),
obj_boxes.astype(dtype=np.float32, copy=False))
ovlp_inds = np.where(ovlp_so > 0)[0]
sbj_inds = sbj_inds[ovlp_inds]
obj_inds = obj_inds[ovlp_inds]
sbj_rois = sbj_rois[ovlp_inds]
obj_rois = obj_rois[ovlp_inds]
sbj_boxes = sbj_boxes[ovlp_inds]
obj_boxes = obj_boxes[ovlp_inds]
return_dict = {}
if self.training:
# Add binary relationships
blobs_out = self.RelPN_GenerateProposalLabels(
sbj_rois, obj_rois, det_rois, roidb, im_info)
return_dict.update(blobs_out)
else:
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]],
dtype=np.int32)
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT:
return_dict['all_sbj_labels_int32'] = sbj_labels.astype(
np.int32, copy=False) - 1 # det_labels start from 1
return_dict['all_obj_labels_int32'] = obj_labels.astype(
np.int32, copy=False) - 1 # det_labels start from 1
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
return_dict, rois_blob_name,
return_dict[rois_blob_name], target_lvls, lvl_min,
lvl_max)
return return_dict
def _forward(self,
data,
im_info,
dataset_name=None,
roidb=None,
use_gt_labels=False,
include_feat=False,
**rpn_kwargs):
im_data = data
if self.training:
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[
0] if self.training else cfg.TEST.DATASETS[
0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
if not cfg.TRAIN.USE_GT_BOXES:
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv,
rpn_ret,
use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
if cfg.TRAIN.USE_GT_BOXES:
# we always feed one image per batch during training
assert len(roidb) == 1
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb[0]['sbj_gt_boxes']
obj_boxes = roidb[0]['obj_gt_boxes']
sbj_all_boxes = _augment_gt_boxes_by_perturbation(
sbj_boxes, im_w, im_h)
obj_all_boxes = _augment_gt_boxes_by_perturbation(
obj_boxes, im_w, im_h)
det_all_boxes = np.vstack((sbj_all_boxes, obj_all_boxes))
det_all_boxes = np.unique(det_all_boxes, axis=0)
det_all_rois = det_all_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(det_all_rois.shape[0], 1))
det_all_rois = np.hstack((repeated_batch_idx, det_all_rois))
rel_ret = self.RelPN(det_all_rois, None, None, im_info,
dataset_name, roidb)
else:
fg_inds = np.where(rpn_ret['labels_int32'] > 0)[0]
det_rois = rpn_ret['rois'][fg_inds]
det_labels = rpn_ret['labels_int32'][fg_inds]
det_scores = F.softmax(cls_score[fg_inds], dim=1)
rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info,
dataset_name, roidb)
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
if roidb is not None:
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
sbj_boxes = roidb['sbj_gt_boxes']
obj_boxes = roidb['obj_gt_boxes']
sbj_rois = sbj_boxes * im_scale
obj_rois = obj_boxes * im_scale
repeated_batch_idx = 0 * blob_utils.ones(
(sbj_rois.shape[0], 1))
sbj_rois = np.hstack((repeated_batch_idx, sbj_rois))
obj_rois = np.hstack((repeated_batch_idx, obj_rois))
rel_rois = box_utils.rois_union(sbj_rois, obj_rois)
rel_ret = {}
rel_ret['sbj_rois'] = sbj_rois
rel_ret['obj_rois'] = obj_rois
rel_ret['rel_rois'] = rel_rois
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['sbj_rois', 'obj_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
rel_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
rel_ret, rois_blob_name, rel_ret[rois_blob_name],
target_lvls, lvl_min, lvl_max)
if use_gt_labels:
sbj_labels = roidb['sbj_gt_classes'] # start from 0
obj_labels = roidb['obj_gt_classes'] # start from 0
sbj_scores = np.ones_like(sbj_labels, dtype=np.float32)
obj_scores = np.ones_like(obj_labels, dtype=np.float32)
else:
sbj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=True)
sbj_cls_scores, _ = self.Box_Outs(sbj_det_feat)
sbj_cls_scores = sbj_cls_scores.data.cpu().numpy()
obj_det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=True)
obj_cls_scores, _ = self.Box_Outs(obj_det_feat)
obj_cls_scores = obj_cls_scores.data.cpu().numpy()
sbj_labels = np.argmax(sbj_cls_scores[:, 1:], axis=1)
obj_labels = np.argmax(obj_cls_scores[:, 1:], axis=1)
sbj_scores = np.amax(sbj_cls_scores[:, 1:], axis=1)
obj_scores = np.amax(obj_cls_scores[:, 1:], axis=1)
rel_ret['sbj_scores'] = sbj_scores.astype(np.float32,
copy=False)
rel_ret['obj_scores'] = obj_scores.astype(np.float32,
copy=False)
rel_ret['sbj_labels'] = sbj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['obj_labels'] = obj_labels.astype(
np.int32, copy=False) + 1 # need to start from 1
rel_ret['all_sbj_labels_int32'] = sbj_labels.astype(np.int32,
copy=False)
rel_ret['all_obj_labels_int32'] = obj_labels.astype(np.int32,
copy=False)
sbj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='sbj_rois',
use_relu=use_relu)
obj_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='obj_rois',
use_relu=use_relu)
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
rel_ret = self.RelPN(det_rois, det_labels, det_scores,
im_info, dataset_name, roidb)
valid_len = len(rel_ret['rel_rois'])
if valid_len > 0:
break
logger.info(
'Got {} rel_rois when score_thresh={}, changing to {}'.
format(valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
det_feat = self.Box_Head(blob_conv,
rel_ret,
rois_name='det_rois',
use_relu=use_relu)
sbj_feat = det_feat[rel_ret['sbj_inds']]
obj_feat = det_feat[rel_ret['obj_inds']]
rel_feat = self.Prd_RCNN.Box_Head(blob_conv_prd,
rel_ret,
rois_name='rel_rois',
use_relu=use_relu)
concat_feat = torch.cat((sbj_feat, rel_feat, obj_feat), dim=1)
if cfg.MODEL.USE_FREQ_BIAS or cfg.MODEL.RUN_BASELINE or cfg.MODEL.USE_SEM_CONCAT:
sbj_labels = rel_ret['all_sbj_labels_int32']
obj_labels = rel_ret['all_obj_labels_int32']
else:
sbj_labels = None
obj_labels = None
# when MODEL.USE_SEM_CONCAT, memory runs out if the whole batch is fed once
# so we need to feed the batch twice if it's big
gn_size = 1000
if cfg.MODEL.USE_SEM_CONCAT and concat_feat.shape[0] > gn_size:
group = int(math.ceil(concat_feat.shape[0] / gn_size))
prd_cls_scores = None
sbj_cls_scores = None
obj_cls_scores = None
for i in range(group):
end = int(min((i + 1) * gn_size, concat_feat.shape[0]))
concat_feat_i = concat_feat[i * gn_size:end]
sbj_labels_i = sbj_labels[
i * gn_size:end] if sbj_labels is not None else None
obj_labels_i = obj_labels[
i * gn_size:end] if obj_labels is not None else None
sbj_feat_i = sbj_feat[i * gn_size:end]
obj_feat_i = obj_feat[i * gn_size:end]
prd_cls_scores_i, sbj_cls_scores_i, obj_cls_scores_i = \
self.RelDN(concat_feat_i, sbj_labels_i, obj_labels_i, sbj_feat_i, obj_feat_i)
if prd_cls_scores is None:
prd_cls_scores = prd_cls_scores_i
sbj_cls_scores = sbj_cls_scores_i
obj_cls_scores = obj_cls_scores_i
else:
prd_cls_scores = torch.cat(
(prd_cls_scores, prd_cls_scores_i))
sbj_cls_scores = torch.cat(
(sbj_cls_scores, sbj_cls_scores_i
)) if sbj_cls_scores_i is not None else sbj_cls_scores
obj_cls_scores = torch.cat(
(obj_cls_scores, obj_cls_scores_i
)) if obj_cls_scores_i is not None else obj_cls_scores
else:
prd_cls_scores, sbj_cls_scores, obj_cls_scores = \
self.RelDN(concat_feat, sbj_labels, obj_labels, sbj_feat, obj_feat)
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
if not cfg.TRAIN.USE_GT_BOXES:
# rpn loss
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
loss_cls_prd, accuracy_cls_prd = reldn_heads.reldn_losses(
prd_cls_scores,
rel_ret['all_prd_labels_int32'],
weight=self.prd_weights)
return_dict['losses']['loss_cls_prd'] = loss_cls_prd
return_dict['metrics']['accuracy_cls_prd'] = accuracy_cls_prd
if cfg.MODEL.USE_SEPARATE_SO_SCORES:
loss_cls_sbj, accuracy_cls_sbj = reldn_heads.reldn_losses(
sbj_cls_scores,
rel_ret['all_sbj_labels_int32'],
weight=self.obj_weights)
return_dict['losses']['loss_cls_sbj'] = loss_cls_sbj
return_dict['metrics']['accuracy_cls_sbj'] = accuracy_cls_sbj
loss_cls_obj, accuracy_cls_obj = reldn_heads.reldn_losses(
obj_cls_scores,
rel_ret['all_obj_labels_int32'],
weight=self.obj_weights)
return_dict['losses']['loss_cls_obj'] = loss_cls_obj
return_dict['metrics']['accuracy_cls_obj'] = accuracy_cls_obj
if cfg.TRAIN.HUBNESS:
loss_hubness_prd = reldn_heads.add_hubness_loss(prd_cls_scores)
loss_hubness_sbj = reldn_heads.add_hubness_loss(sbj_cls_scores)
loss_hubness_obj = reldn_heads.add_hubness_loss(obj_cls_scores)
return_dict['losses']['loss_hubness_prd'] = loss_hubness_prd
return_dict['losses']['loss_hubness_sbj'] = loss_hubness_sbj
return_dict['losses']['loss_hubness_obj'] = loss_hubness_obj
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
# Testing
return_dict['sbj_rois'] = rel_ret['sbj_rois']
return_dict['obj_rois'] = rel_ret['obj_rois']
return_dict['sbj_labels'] = rel_ret['sbj_labels']
return_dict['obj_labels'] = rel_ret['obj_labels']
return_dict['sbj_scores'] = rel_ret['sbj_scores']
return_dict['sbj_scores_out'] = sbj_cls_scores
return_dict['obj_scores'] = rel_ret['obj_scores']
return_dict['obj_scores_out'] = obj_cls_scores
return_dict['prd_scores'] = prd_cls_scores
if include_feat:
return_dict['sbj_feat'] = sbj_feat
return_dict['obj_feat'] = obj_feat
return_dict['prd_feat'] = concat_feat
return return_dict
def _forward(self, data, im_info, do_vis=False, dataset_name=None, roidb=None, use_gt_labels=False, **rpn_kwargs):
im_data = data
if self.training:
# if not isinstance(roidb[0], np.array):
# roidb = roidb[0]
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb)) # only support one gpu
if dataset_name is not None:
dataset_name = blob_utils.deserialize(dataset_name)
else:
dataset_name = cfg.TRAIN.DATASETS[0] if self.training else cfg.TEST.DATASETS[0] # assuming only one dataset per run
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
# if not cfg.MODEL.USE_REL_PYRAMID:
# blob_conv_prd = self.Prd_RCNN.Conv_Body(im_data)
if self.training:
gt_rois = np.empty((0, 5), dtype=np.float32)
gt_classes = np.empty((0), dtype=np.int64)
for i, r in enumerate(roidb):
rois_i = r['boxes'] * im_info[i, 2]
rois_i = np.hstack((i * blob_utils.ones((rois_i.shape[0], 1)), rois_i))
gt_rois = np.append(gt_rois, rois_i, axis=0)
gt_classes = np.append(gt_classes, r['gt_classes'], axis=0)
if self.training or roidb is None:
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
# if not cfg.MODEL.USE_REL_PYRAMID:
# blob_conv_prd = blob_conv_prd[-self.num_roi_levels:]
# else:
# blob_conv_prd = self.RelPyramid(blob_conv)
if self.training or roidb is None:
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret, use_relu=True)
cls_score, bbox_pred = self.Box_Outs(box_feat)
# now go through the predicate branch
use_relu = False if cfg.MODEL.NO_FC7_RELU else True
if self.training:
score_thresh = cfg.TEST.SCORE_THRESH
cls_score = F.softmax(cls_score, -1)
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores, det_dists, det_boxes_all = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2])
non_zero_area_inds = np.where(real_area > 0)[0]
det_rois = det_rois[non_zero_area_inds]
det_labels = det_labels[non_zero_area_inds]
det_scores = det_scores[non_zero_area_inds]
det_dists = det_dists[non_zero_area_inds]
det_boxes_all = det_boxes_all[non_zero_area_inds]
# rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb)
valid_len = len(det_rois)
if valid_len > 0:
break
logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format(
valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
det_labels_gt = []
ious = box_utils.bbox_overlaps(det_rois[:, 1:], gt_rois[:, 1:]) * \
(det_rois[:, 0][:,None] == gt_rois[:, 0][None, :])
det_labels_gt = gt_classes[ious.argmax(-1)]
det_labels_gt[ious.max(-1) < cfg.TRAIN.FG_THRESH] = 0
else:
if roidb is not None:
# raise FError('not support this mode!')
# assert len(roidb) == 1
im_scale = im_info.data.numpy()[:, 2][0]
im_w = im_info.data.numpy()[:, 1][0]
im_h = im_info.data.numpy()[:, 0][0]
fpn_ret = {'gt_rois': gt_rois}
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
rois_blob_names = ['gt_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
fpn_ret[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(
fpn_ret, rois_blob_name, fpn_ret[rois_blob_name], target_lvls,
lvl_min, lvl_max)
det_feats = self.Box_Head(blob_conv, fpn_ret, rois_name='det_rois', use_relu=True)
det_dists, _ = self.Box_Outs(det_feats)
det_boxes_all = None
if use_gt_labels:
det_labels_gt = gt_classes
det_labels = gt_classes
else:
score_thresh = cfg.TEST.SCORE_THRESH
while score_thresh >= -1e-06: # a negative value very close to 0.0
det_rois, det_labels, det_scores, det_dists, det_boxes_all = \
self.prepare_det_rois(rpn_ret['rois'], cls_score, bbox_pred, im_info, score_thresh)
real_area = (det_rois[:, 3] - det_rois[:, 1]) * (det_rois[:, 4] - det_rois[:, 2])
non_zero_area_inds = np.where(real_area > 0)[0]
det_rois = det_rois[non_zero_area_inds]
det_labels = det_labels[non_zero_area_inds]
det_scores = det_scores[non_zero_area_inds]
det_dists = det_dists[non_zero_area_inds]
det_boxes_all = det_boxes_all[non_zero_area_inds]
# rel_ret = self.RelPN(det_rois, det_labels, det_scores, im_info, dataset_name, roidb)
valid_len = len(det_rois)
if valid_len > 0:
break
logger.info('Got {} det_rois when score_thresh={}, changing to {}'.format(
valid_len, score_thresh, score_thresh - 0.01))
score_thresh -= 0.01
return_dict['det_rois'] = det_rois
num_rois = det_rois.shape[0]
if not isinstance(det_dists, torch.Tensor):
assert det_dists.shape[0] == num_rois
det_dists = torch.from_numpy(det_dists).float().cuda(device_id)
return_dict['det_dists'] = det_dists
return_dict['det_scores'] = det_scores
return_dict['blob_conv'] = blob_conv
return_dict['det_boxes_all'] = det_boxes_all
assert det_boxes_all.shape[0] == num_rois
return_dict['det_labels'] = det_labels
# return_dict['blob_conv_prd'] = blob_conv_prd
if self.training or use_gt_labels:
return_dict['det_labels_gt'] = det_labels_gt
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
# Get pairwise proposals first
im_inds = det_rois[:, 0]
is_cand = im_inds[:, None] == im_inds[None, :]
is_cand.reshape(-1)[diagonal_inds(is_cand)] = False
is_empty = np.where(is_cand.any(1) == 0)[0]
if self.overlap:
is_cand = is_cand & (box_utils.bbox_overlaps(
det_rois[:, 1:], det_rois[:, 1:]) > 0)
if is_empty.size > 0:
is_cand[is_empty, is_empty] = True
sbj_inds, obj_inds = np.where(is_cand)
# remove self paired rois
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
return_dict = {}
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name,
return_dict[rois_blob_name],
target_lvls, lvl_min,
lvl_max)
return return_dict
def _forward(self, data, im_info, roidb=None, **rpn_kwargs):
im_data = data
if self.training:
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
rpn_ret = self.RPN(blob_conv, im_info, roidb)
# rpn proposals
# if self.training:
# # can be used to infer fg/bg ratio
# return_dict['rois_label'] = rpn_ret['labels_int32']
rois_certification = False
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
blob_conv = blob_conv[-self.num_roi_levels:]
if not self.training:
return_dict['blob_conv'] = blob_conv
if rois_certification:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
test_rpn_ret = {'rois': rpn_ret['rois']}
lvls = fpn_utils.map_rois_to_fpn_levels(test_rpn_ret['rois'],
lvl_min, lvl_max)
rois_idx_order = np.empty((0, ))
test_rois = test_rpn_ret['rois']
for output_idx, lvl in enumerate(range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
rois_lvl = test_rois[idx_lvl, :]
rois_idx_order = np.concatenate((rois_idx_order, idx_lvl))
test_rpn_ret['rois_fpn{}'.format(lvl)] = rois_lvl
rois_idx_restore = np.argsort(rois_idx_order).astype(np.int32,
copy=False)
test_rpn_ret['rois_idx_restore_int32'] = rois_idx_restore
test_feat = self.Box_Head(blob_conv, test_rpn_ret)
test_cls_score, test_bbox_pred = self.Box_Outs(test_feat)
test_cls_score = test_cls_score.data.cpu().numpy().squeeze()
test_bbox_pred = test_bbox_pred.data.cpu().numpy().squeeze()
if not cfg.MODEL.RPN_ONLY:
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret)
# bbox proposals
else:
box_feat = self.Box_Head(blob_conv, rpn_ret)
cls_score, bbox_pred = self.Box_Outs(box_feat)
else:
# TODO: complete the returns for RPN only situation
pass
# 在这里开始计算loss
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
# rpn loss
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
if cfg.MODEL.MASK_ON:
if getattr(self.Mask_Head, 'SHARE_RES5', False):
mask_feat = self.Mask_Head(
res5_feat,
rpn_ret,
roi_has_mask_int32=rpn_ret['roi_has_mask_int32'])
else:
mask_feat = self.Mask_Head(blob_conv, rpn_ret)
mask_pred = self.Mask_Outs(mask_feat)
# return_dict['mask_pred'] = mask_pred
# mask loss
loss_mask = mask_rcnn_heads.mask_rcnn_losses(
mask_pred, rpn_ret['masks_int32'])
return_dict['losses']['loss_mask'] = loss_mask
if cfg.MODEL.KEYPOINTS_ON:
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
# No corresponding keypoint head implemented yet (Neither in Detectron)
# Also, rpn need to generate the label 'roi_has_keypoints_int32'
kps_feat = self.Keypoint_Head(
res5_feat,
rpn_ret,
roi_has_keypoints_int32=rpn_ret[
'roi_has_keypoint_int32'])
else:
kps_feat = self.Keypoint_Head(blob_conv, rpn_ret)
kps_pred = self.Keypoint_Outs(kps_feat)
# return_dict['keypoints_pred'] = kps_pred
# keypoints loss
if cfg.KRCNN.NORMALIZE_BY_VISIBLE_KEYPOINTS:
loss_keypoints = keypoint_rcnn_heads.keypoint_losses(
kps_pred, rpn_ret['keypoint_locations_int32'],
rpn_ret['keypoint_weights'])
else:
loss_keypoints = keypoint_rcnn_heads.keypoint_losses(
kps_pred, rpn_ret['keypoint_locations_int32'],
rpn_ret['keypoint_weights'],
rpn_ret['keypoint_loss_normalizer'])
return_dict['losses']['loss_kps'] = loss_keypoints
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
# Testing
return_dict['rois'] = rpn_ret['rois']
import json
if cfg.TEST.IOU_OUT:
# 直接通过rpn_ret可以取出rois
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/raw_roi.json",
'w') as f:
json.dump((return_dict['rois'][:, 1:] /
im_info.numpy()[0][2]).tolist(), f)
# 如果在FPN模式下，需要进到一个collect_and_distribute...的函数去取出分配后的scores
# ，我直接在collect_and_distribute_fpn_rpn_proposals.py里把json输出
# 因此这里直接考虑RPN_ONLY模式的取值。
if not cfg.FPN.FPN_ON:
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/rois_score.json",
'w') as f:
score_2_json = []
for item in rpn_ret['rpn_roi_probs']:
score_2_json.append(item.item())
json.dump(score_2_json, f)
# 开始第二个fast_head阶段，首先通过rois和bbox_delta计算pred_box
if cfg.FAST_RCNN.FAST_HEAD2_DEBUG:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
if cfg.FPN.FPN_ON:
im_scale = im_info.data.cpu().numpy().squeeze()[2]
rois = rpn_ret['rois'][:, 1:5] / im_scale
bbox_pred = bbox_pred.data.cpu().numpy().squeeze()
box_deltas = bbox_pred.reshape([-1, bbox_pred.shape[-1]])
shift_boxes = box_utils.bbox_transform(
rois, box_deltas, cfg.MODEL.BBOX_REG_WEIGHTS)
shift_boxes = box_utils.clip_tiled_boxes(
shift_boxes,
im_info.data.cpu().numpy().squeeze()[0:2])
num_classes = cfg.MODEL.NUM_CLASSES
onecls_pred_boxes = []
onecls_score = []
dets_cls = {}
count = 0
for j in range(1, num_classes):
inds = np.where(
cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0]
boxes_j = shift_boxes[inds, j * 4:(j + 1) * 4]
score_j = cls_score[inds, j]
onecls_pred_boxes += boxes_j.tolist()
onecls_score += score_j.tolist()
dets_cls.update({j: []})
for k in range(len(boxes_j.tolist())):
dets_cls[j].append(count)
count += 1
assert count == len(onecls_pred_boxes)
stage2_rois_score = np.array(onecls_score,
dtype=np.float32)
stage2_rois = np.array(onecls_pred_boxes, dtype=np.float32)
# Redistribute stage2_rois using fpn_utils module provided functions
# calculate by formula
cls_tracker = {}
if not stage2_rois.tolist():
stage1_pred_iou = stage2_rois_score.tolist()
stage2_final_boxes = np.empty((0, ))
stage2_final_score = np.empty((0, ))
logger.info("Detections above threshold is null.")
else:
alter_rpn = {}
unresize_stage2_rois = stage2_rois * im_scale
# unresize_stage2_rois = np.concatenate((unresize_stage2_rois, unresized_rois[:, 1:5]))
lvls = fpn_utils.map_rois_to_fpn_levels(
unresize_stage2_rois, lvl_min, lvl_max)
# TAG: We might need to visualize "stage2_rois" to make sure.
rois_idx_order = np.empty((0, ))
dummy_batch = np.zeros(
(unresize_stage2_rois.shape[0], 1),
dtype=np.float32)
alter_rpn["rois"] = np.hstack(
(dummy_batch,
unresize_stage2_rois)).astype(np.float32,
copy=False)
# alter_rpn['rois'] = np.concatenate((alter_rpn['rois'], unresized_rois))
for output_idx, lvl in enumerate(
range(lvl_min, lvl_max + 1)):
idx_lvl = np.where(lvls == lvl)[0]
rois_lvl = unresize_stage2_rois[idx_lvl, :]
rois_idx_order = np.concatenate(
(rois_idx_order, idx_lvl))
_ = np.zeros((rois_lvl.shape[0], 1),
dtype=np.float32)
alter_rpn['rois_fpn{}'.format(lvl)] = np.hstack(
(_, rois_lvl)).astype(np.float32, copy=False)
rois_idx_restore = np.argsort(rois_idx_order).astype(
np.int32, copy=False)
alter_rpn['rois_idx_restore_int32'] = rois_idx_restore
# Go through 2nd stage of FPN and fast_head
stage2_feat = self.Box_Head(blob_conv, alter_rpn)
stage2_cls_score, stage2_bbox_pred = self.Box_Outs(
stage2_feat)
# Transform shift value to original one to get final pred boxes coordinates
stage2_bbox_pred = stage2_bbox_pred.data.cpu().numpy(
).squeeze()
stage2_cls_score = stage2_cls_score.data.cpu().numpy()
stage2_box_deltas = stage2_bbox_pred.reshape(
[-1, bbox_pred.shape[-1]])
# Add some variance to box delta
if cfg.FAST_RCNN.STAGE1_TURBULENCE:
import random
for i in range(len(stage2_box_deltas)):
for j in range(len(stage2_box_deltas[i])):
stage2_box_deltas[i][j] *= random.uniform(
0.9, 1.1)
stage2_cls_out = box_utils.bbox_transform(
stage2_rois, stage2_box_deltas,
cfg.MODEL.BBOX_REG_WEIGHTS)
stage2_cls_out = box_utils.clip_tiled_boxes(
stage2_cls_out,
im_info.data.cpu().numpy().squeeze()[0:2])
onecls_pred_boxes = []
onecls_score = []
count = 0
for j in range(1, num_classes):
inds = np.where(
stage2_cls_score[:,
j] > cfg.TEST.SCORE_THRESH)[0]
boxes_j = stage2_cls_out[inds, j * 4:(j + 1) * 4]
score_j = stage2_cls_score[inds, j]
dets_j = np.hstack(
(boxes_j,
score_j[:, np.newaxis])).astype(np.float32,
copy=False)
keep = box_utils.nms(dets_j, cfg.TEST.NMS)
boxes_j = boxes_j[keep]
score_j = score_j[keep]
# 用于记录每个框属于第几类
onecls_score += score_j.tolist()
onecls_pred_boxes += boxes_j.tolist()
for k in range(len(score_j)):
cls_tracker.update({count: j})
count += 1
assert count == len(onecls_score)
stage2_final_boxes = np.array(onecls_pred_boxes,
dtype=np.float32)
stage2_final_score = np.array(onecls_score,
dtype=np.float32)
inds = np.where(stage2_final_score > 0.3)[0]
# Filtered by keep index...
preserve_stage2_final_boxes = copy.deepcopy(
stage2_final_boxes)
preserve_stage2_final_score = copy.deepcopy(
stage2_final_score)
stage2_final_boxes = stage2_final_boxes[inds]
stage2_final_score = stage2_final_score[inds]
# if nothing left after 0.3 threshold filter, reserve whole boxes to original.
if stage2_final_boxes.size == 0:
lower_inds = np.where(
preserve_stage2_final_score > 0.1)[0]
stage2_final_boxes = preserve_stage2_final_boxes[
lower_inds]
stage2_final_score = preserve_stage2_final_score[
lower_inds]
else:
del preserve_stage2_final_boxes
del preserve_stage2_final_score
# if all boxes are clsfied into bg class.
if stage2_final_boxes.size == 0:
stage1_pred_iou = stage2_rois_score.tolist()
stage2_final_boxes = np.empty((0, ))
stage2_final_score = np.empty((0, ))
logger.info("Detections above threshold is null.")
else:
# Restore stage2_pred_boxes to match the index with stage2_rois, Compute IOU between
# final_boxes and stage2_rois, one by one
flag = "cross_product"
if flag == "element_wise":
if stage2_final_boxes.shape[
0] == stage2_rois.shape[0]:
restored_stage2_final_boxes = stage2_final_boxes[
rois_idx_restore]
stage1_pred_iou = []
for ind, item in enumerate(stage2_rois):
stage1 = np.array(
item, dtype=np.float32).reshape(
(1, 4))
stage2 = np.array(
restored_stage2_final_boxes[ind],
dtype=np.float32).reshape((1, 4))
iou = box_utils.bbox_overlaps(
stage1, stage2)
stage1_pred_iou.append(
iou.squeeze().item())
else:
logger.info(
"Mistake while processing {}".format(
str(im_info)))
elif flag == "cross_product":
iou = box_utils.bbox_overlaps(
stage2_rois, stage2_final_boxes)
stage1_pred_iou = iou.max(axis=1).tolist()
# stage1_pred is another name of stage2_rois
assert len(stage1_pred_iou) == len(stage2_rois)
if cfg.FAST_RCNN.IOU_NMS:
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_score.json",
"w") as f:
json.dump(stage2_rois_score.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage2_score.json",
"w") as f:
json.dump(stage2_final_score.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_pred_boxes.json",
'w') as f:
json.dump(stage2_rois.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage1_pred_iou.json",
'w') as f:
json.dump(stage1_pred_iou, f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_stage2_pred_boxes.json",
'w') as f:
json.dump(stage2_final_boxes.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_dets_cls.json",
'w') as f:
json.dump(dets_cls, f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/iou_cls_tracker.json",
'w') as f:
json.dump(cls_tracker, f)
elif cfg.FAST_RCNN.SCORE_NMS:
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_score.json",
"w") as f:
json.dump(stage2_rois_score.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage2_score.json",
"w") as f:
json.dump(stage2_final_score.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_pred_boxes.json",
'w') as f:
json.dump(stage2_rois.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage1_pred_iou.json",
'w') as f:
json.dump(stage1_pred_iou, f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_stage2_pred_boxes.json",
'w') as f:
json.dump(stage2_final_boxes.tolist(), f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_dets_cls.json",
'w') as f:
json.dump(dets_cls, f)
with open(
"/nfs/project/libo_i/IOU.pytorch/IOU_Validation/score_cls_tracker.json",
'w') as f:
json.dump(cls_tracker, f)
else:
im_scale = im_info.data.cpu().numpy().squeeze()[2]
rois = rpn_ret['rois'][:, 1:5] / im_scale
# unscale back to raw image space
box_deltas = bbox_pred.data.cpu().numpy().squeeze()
fast_stage1_score = cls_score.data.cpu().numpy().squeeze()
box_deltas = box_deltas.reshape([-1, bbox_pred.shape[-1]])
stage2_rois = box_utils.bbox_transform(
rois, box_deltas, cfg.MODEL.BBOX_REG_WEIGHTS)
stage2_rois = box_utils.clip_tiled_boxes(
stage2_rois,
im_info.data.cpu().numpy().squeeze()[0:2])
num_classes = cfg.MODEL.NUM_CLASSES
onecls_pred_boxes = []
onecls_cls_score = []
for j in range(1, num_classes):
inds = np.where(
cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0]
boxes_j = stage2_rois[inds, j * 4:(j + 1) * 4]
score_j = fast_stage1_score[inds, j]
onecls_pred_boxes += boxes_j.tolist()
onecls_cls_score += score_j.tolist()
stage2_rois = np.array(onecls_pred_boxes, dtype=np.float32)
stage2_rois_score = np.array(onecls_cls_score,
dtype=np.float32)
assert len(stage2_rois) == len(stage2_rois_score)
# Send stage2 rois to next stage fast head, do ROI ALIGN again
# to modify rpn_ret['rois] , rpn_ret['rpn_rois'] and rpn['rois_rpn_score']
rpn_ret['rois'] = stage2_rois
rpn_ret['rpn_rois'] = stage2_rois
rpn_ret['rpn_roi_probs'] = stage2_rois_score
stage2_box_feat = self.Box_Head(blob_conv, rpn_ret)
stage2_cls_score, stage2_bbox_pred = self.Box_Outs(
stage2_box_feat)
stage2_bbox_pred = stage2_bbox_pred.data.cpu().numpy(
).squeeze()
stage2_bbox_pred = stage2_bbox_pred.reshape(
[-1, bbox_pred.shape[-1]])
stage2_cls_pred_boxes = box_utils.bbox_transform(
stage2_rois, stage2_bbox_pred,
cfg.MODEL.BBOX_REG_WEIGHTS)
stage2_cls_pred_boxes = box_utils.clip_tiled_boxes(
stage2_cls_pred_boxes,
im_info.data.cpu().numpy().squeeze()[0:2])
onecls_pred_boxes = []
onecls_cls_score = []
for j in range(1, num_classes):
inds = np.where(
stage2_cls_score[:, j] > cfg.TEST.SCORE_THRESH)[0]
if len(inds) != 0:
print("KKKKK")
boxes_j = stage2_cls_pred_boxes[inds,
j * 4:(j + 1) * 4]
score_j = stage2_cls_score[inds, j]
onecls_pred_boxes += boxes_j.tolist()
onecls_cls_score += score_j.tolist()
stage2_bbox_pred = np.array(onecls_pred_boxes,
dtype=np.float32)
stage2_bbox_pred_score = np.array(onecls_cls_score,
dtype=np.float32)
# get stage2 pred_boxes here
return_dict['cls_score'] = cls_score
return_dict['bbox_pred'] = bbox_pred
return return_dict
def forward(self,
det_rois,
det_labels,
det_scores,
im_info,
dataset_name,
roidb=None):
"""
det_rois: feature maps from the backbone network. (Variable)
im_info: (CPU Variable)
roidb: (list of ndarray)
"""
sbj_inds = np.repeat(np.arange(det_rois.shape[0]), det_rois.shape[0])
obj_inds = np.tile(np.arange(det_rois.shape[0]), det_rois.shape[0])
# remove self paired rois
if det_rois.shape[
0] > 1: # no pairs to remove when there is at most one detection
sbj_inds, obj_inds = self.remove_self_pairs(
det_rois.shape[0], sbj_inds, obj_inds)
sbj_rois = det_rois[sbj_inds]
obj_rois = det_rois[obj_inds]
im_scale = im_info.data.numpy()[:, 2][0]
sbj_boxes = sbj_rois[:, 1:] / im_scale
obj_boxes = obj_rois[:, 1:] / im_scale
# filters out those roi pairs whose boxes are not overlapping in the original scales
if self.overlap:
ovlp_so = box_utils_rel.bbox_pair_overlaps(
sbj_boxes.astype(dtype=np.float32, copy=False),
obj_boxes.astype(dtype=np.float32, copy=False))
ovlp_inds = np.where((ovlp_so > 0) & (ovlp_so < 0.5))[0]
if ovlp_inds.size > 0:
sbj_inds = sbj_inds[ovlp_inds]
obj_inds = obj_inds[ovlp_inds]
sbj_rois = sbj_rois[ovlp_inds]
obj_rois = obj_rois[ovlp_inds]
return_dict = {}
sbj_labels = det_labels[sbj_inds]
obj_labels = det_labels[obj_inds]
sbj_scores = det_scores[sbj_inds]
obj_scores = det_scores[obj_inds]
rel_rois = box_utils_rel.rois_union(sbj_rois, obj_rois)
return_dict['det_rois'] = det_rois
return_dict['sbj_inds'] = sbj_inds
return_dict['obj_inds'] = obj_inds
return_dict['sbj_rois'] = sbj_rois
return_dict['obj_rois'] = obj_rois
return_dict['rel_rois'] = rel_rois
return_dict['sbj_labels'] = sbj_labels
return_dict['obj_labels'] = obj_labels
return_dict['sbj_scores'] = sbj_scores
return_dict['obj_scores'] = obj_scores
return_dict['fg_size'] = np.array([sbj_rois.shape[0]], dtype=np.int32)
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
lvl_min = cfg.FPN.ROI_MIN_LEVEL
lvl_max = cfg.FPN.ROI_MAX_LEVEL
# when use min_rel_area, the same sbj/obj area could be mapped to different feature levels
# when they are associated with different relationships
# Thus we cannot get det_rois features then gather sbj/obj features
# The only way is gather sbj/obj per relationship, thus need to return sbj_rois/obj_rois
rois_blob_names = ['det_rois', 'rel_rois']
for rois_blob_name in rois_blob_names:
# Add per FPN level roi blobs named like: <rois_blob_name>_fpn<lvl>
target_lvls = fpn_utils.map_rois_to_fpn_levels(
return_dict[rois_blob_name][:, 1:5], lvl_min, lvl_max)
fpn_utils.add_multilevel_roi_blobs(return_dict, rois_blob_name,
return_dict[rois_blob_name],
target_lvls, lvl_min,
lvl_max)
return return_dict
