def forward(self, inputs, roidb, im_info):
"""
Args:
inputs: a list of [rpn_rois_fpn2, ..., rpn_rois_fpn6,
rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
im_info: [[im_height, im_width, im_scale], ...]
"""
rois = collect(inputs, self.training)
if self.training:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
im_scales = im_info.data.numpy()[:, 2]
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
else:
# For inference we have a special code path that avoids some data
# loader overhead
blobs = distribute(rois, None)
return blobs
def forward(self, rpn_rois, roidb, im_info):
"""Op for generating training labels for RPN proposals. This is used
when training RPN jointly with Fast/Mask R-CNN (as in end-to-end
Faster R-CNN training).
blobs_in:
- 'rpn_rois': 2D tensor of RPN proposals output by GenerateProposals
- 'roidb': roidb entries that will be labeled
- 'im_info': See GenerateProposals doc.
blobs_out:
- (variable set of blobs): returns whatever blobs are required for
training the model. It does this by querying the data loader for
the list of blobs that are needed.
"""
im_scales = im_info.data.numpy()[:, 2]
# ForkedPdb().set_trace()
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
# Note: crowd_thresh=0 will ignore _filter_crowd_proposals
json_dataset.add_proposals(roidb, rpn_rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs
def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = self.collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
self.distribute(rois, None, outputs, self._train)
def forward(self, inputs, outputs):
"""See modeling.detector.CollectAndDistributeFpnRpnProposals for
inputs/outputs documentation.
"""
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def forward(self, rpn_rois, roidb, im_info):
"""Op for generating training labels for RPN proposals. This is used
when training RPN jointly with Fast/Mask R-CNN (as in end-to-end
Faster R-CNN training).
blobs_in:
- 'rpn_rois': 2D tensor of RPN proposals output by GenerateProposals
- 'roidb': roidb entries that will be labeled
- 'im_info': See GenerateProposals doc.
blobs_out:
- (variable set of blobs): returns whatever blobs are required for
training the model. It does this by querying the data loader for
the list of blobs that are needed.
"""
im_scales = im_info.data.numpy()[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
# Note: crowd_thresh=0 will ignore _filter_crowd_proposals
json_dataset.add_proposals(roidb, rpn_rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs
def forward(self, inputs, roidb, im_info):
"""
Args:
inputs: a list of [rpn_rois_fpn2, ..., rpn_rois_fpn6,
rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
im_info: [[im_height, im_width, im_scale], ...]
"""
rois = collect(inputs, self.training)
if self.training:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
im_scales = im_info.data.numpy()[:, 2]
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
else:
# For inference we have a special code path that avoids some data
# loader overhead
blobs = distribute(rois, None)
return blobs
def get_new_blobs(rois, im_info, roidb, pos_iou=0):
im_scales = im_info.data.numpy()[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
# pdb.set_trace()
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
add_fast_rcnn_blobs(blobs, im_scales, roidb, pos_iou)
return blobs
def forward(self, inputs, roidb, im_info):
"""
Args:
inputs: a list of [rpn_rois_fpn2, ..., rpn_rois_fpn6,
rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
im_info: [[im_height, im_width, im_scale], ...]
"""
rois = collect(inputs, self.training)
if self.training:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
im_scales = im_info.data.numpy()[:, 2]
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
if len(cfg.TRAIN.DATASETS) == 1:
dataset_name = cfg.TRAIN.DATASETS[0]
if 'coco' in dataset_name:
json_dataset.add_proposals(roidb,
rois,
im_scales,
crowd_thresh=0)
elif 'davis' in dataset_name:
davis_db.add_proposals(roidb,
rois,
im_scales,
crowd_thresh=0)
else:
json_dataset.add_proposals(roidb,
rois,
im_scales,
crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
blobs = {k: [] for k in output_blob_names}
if cfg.CONVGRU.DYNAMIC_MODEL:
assert len(
roidb
) == 1, 'The batch size should be one, so the running graph can be the same.(has gt or no gt should be consistant).'
# No gt.
if not len(np.where(roidb[0]['gt_classes'] > 0)[0]) > 0:
return blobs
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
else:
# For inference we have a special code path that avoids some data
# loader overhead
blobs = distribute(rois, None)
return blobs
def forward(self, inputs, outputs):
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
json_dataset.add_proposals(roidb, rois, im_scales)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
json_dataset.add_proposals(roidb, rois, im_scales)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, outputs):
"""See modeling.detector.GenerateProposalLabels for inputs/outputs
documentation.
"""
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
rois = inputs[0].data
roidb = blob_utils.deserialize(inputs[1].data)
im_info = inputs[2].data
im_scales = im_info[:, 2]
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
def forward(self, inputs, roidb, im_info):
"""
Args:
inputs: a list of [rpn_rois_fpn2, ..., rpn_rois_fpn6,
rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6] # (2905, 5)
im_info: [[im_height, im_width, im_scale], ...]
"""
num_img = im_info.shape[0]
rois = collect(inputs, self.training, num_img)
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
im_scales = im_info.data.numpy()[:, 2]
# For historical consistency with the original Faster R-CNN
# implementation we are *not* filtering crowd proposals.
# This choice should be investigated in the future (it likely does
# not matter).
### add 2000 rois in roidb
json_dataset.add_proposals(roidb, rois, im_scales, crowd_thresh=0)
# Compute training labels for the RPN proposals; also handles
# distributing the proposals over FPN levels
# output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
output_blob_names = ['rois']
if cfg.FPN.FPN_ON and cfg.FPN.MULTILEVEL_ROIS:
# Support for FPN multi-level rois without bbox reg isn't
# implemented (... and may never be implemented)
k_max = cfg.FPN.ROI_MAX_LEVEL
k_min = cfg.FPN.ROI_MIN_LEVEL
# Same format as rois blob, but one per FPN level
for lvl in range(k_min, k_max + 1):
output_blob_names += ['rois_fpn' + str(lvl)]
output_blob_names += ['rois_idx_restore_int32']
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
return blobs
def forward(self, inputs, outputs):
# inputs is
# [rpn_rois_fpn2, ..., rpn_rois_fpn6,
# rpn_roi_probs_fpn2, ..., rpn_roi_probs_fpn6]
# If training with Faster R-CNN, then inputs will additionally include
# + [roidb, im_info]
rois = collect(inputs, self._train)
if self._train:
# During training we reuse the data loader code. We populate roidb
# entries on the fly using the rois generated by RPN.
# im_info: [[im_height, im_width, im_scale], ...]
im_info = inputs[-1].data
im_scales = im_info[:, 2]
roidb = blob_utils.deserialize(inputs[-2].data)
output_blob_names = roi_data.fast_rcnn.get_fast_rcnn_blob_names()
json_dataset.add_proposals(roidb, rois, im_scales)
blobs = {k: [] for k in output_blob_names}
roi_data.fast_rcnn.add_fast_rcnn_blobs(blobs, im_scales, roidb)
for i, k in enumerate(output_blob_names):
blob_utils.py_op_copy_blob(blobs[k], outputs[i])
else:
# For inference we have a special code path that avoids some data
# loader overhead
distribute(rois, None, outputs, self._train)
def _forward(self, data, im_info, roidb=None, **rpn_kwargs):
im_data = data
if self.training:
# roidb: list, length = batch size
# 'has_visible_keypoints': bool
# 'boxes' & 'gt_classes': object bboxes and classes
# 'segms', 'seg_areas', 'gt_overlaps', 'is_crowd', 'box_to_gt_ind_map': pass
# 'gt_actions': num_box*26
# 'gt_role_id': num_box*26*2
roidb = list(map(lambda x: blob_utils.deserialize(x)[0], roidb))
device_id = im_data.get_device()
return_dict = {} # A dict to collect return variables
blob_conv = self.Conv_Body(im_data)
# Original RPN module will generate proposals, and sample 256 positive/negative
# examples in a 1:3 ratio for r-cnn stage. For InteractNet(hoi), I set
# cfg.TRAIN.BATCH_SIZE_PER_IM and cfg.TRAIN.FG_FRACTION big value, to save
# every proposal in rpn_ret, then I will re-sample from rpn_ret for three branch
# of InteractNet, see roi_data/hoi_data.py for more information.
if not cfg.VCOCO.USE_PRECOMP_BOX:
rpn_ret = self.RPN(blob_conv, im_info, roidb)
if cfg.MODEL.VCOCO_ON and self.training:
# WARNING! always sample hoi branch before detection branch when training
hoi_blob_in = sample_for_hoi_branch(rpn_ret,
roidb,
im_info,
is_training=True)
# Re-sampling for RCNN head, rpn_ret will be modified inplace
sample_for_detection_branch(rpn_ret)
elif self.training:
json_dataset.add_proposals(roidb,
rois=None,
im_info=im_info.data.numpy(),
crowd_thresh=0) #[:, 2]
hoi_blob_in = sample_for_hoi_branch_precomp_box_train(
roidb, im_info, is_training=True)
if hoi_blob_in is None:
return_dict['losses'] = {}
return_dict['metrics'] = {}
return_dict['losses'][
'loss_hoi_interaction_action'] = torch.tensor(
[0.]).cuda(device_id)
return_dict['metrics'][
'accuracy_interaction_cls'] = torch.tensor(
[0.]).cuda(device_id)
return_dict['losses'][
'loss_hoi_interaction_affinity'] = torch.tensor(
[0.]).cuda(device_id)
return_dict['metrics'][
'accuracy_interaction_affinity'] = torch.tensor(
[0.]).cuda(device_id)
return return_dict
if cfg.FPN.FPN_ON:
# Retain only the blobs that will be used for RoI heads. `blob_conv` may include
# extra blobs that are used for RPN proposals, but not for RoI heads.
#blob_conv = blob_conv[-self.num_roi_levels:]
if cfg.FPN.MULTILEVEL_ROIS:
blob_conv = blob_conv[-self.num_roi_levels:]
else:
blob_conv = blob_conv[-1]
if not self.training:
return_dict['blob_conv'] = blob_conv
if not cfg.VCOCO.USE_PRECOMP_BOX:
if not cfg.MODEL.RPN_ONLY:
if cfg.MODEL.SHARE_RES5 and self.training:
box_feat, res5_feat = self.Box_Head(blob_conv, rpn_ret)
else:
box_feat = self.Box_Head(blob_conv, rpn_ret)
cls_score, bbox_pred = self.Box_Outs(box_feat)
else:
# TODO: complete the returns for RPN only situation
pass
if self.training:
return_dict['losses'] = {}
return_dict['metrics'] = {}
# rpn loss
if not cfg.VCOCO.USE_PRECOMP_BOX:
rpn_kwargs.update(
dict((k, rpn_ret[k]) for k in rpn_ret.keys()
if (k.startswith('rpn_cls_logits')
or k.startswith('rpn_bbox_pred'))))
loss_rpn_cls, loss_rpn_bbox = rpn_heads.generic_rpn_losses(
**rpn_kwargs)
if cfg.FPN.FPN_ON:
for i, lvl in enumerate(
range(cfg.FPN.RPN_MIN_LEVEL,
cfg.FPN.RPN_MAX_LEVEL + 1)):
return_dict['losses']['loss_rpn_cls_fpn%d' %
lvl] = loss_rpn_cls[i]
return_dict['losses']['loss_rpn_bbox_fpn%d' %
lvl] = loss_rpn_bbox[i]
else:
return_dict['losses']['loss_rpn_cls'] = loss_rpn_cls
return_dict['losses']['loss_rpn_bbox'] = loss_rpn_bbox
# bbox loss
loss_cls, loss_bbox, accuracy_cls = fast_rcnn_heads.fast_rcnn_losses(
cls_score, bbox_pred, rpn_ret['labels_int32'],
rpn_ret['bbox_targets'], rpn_ret['bbox_inside_weights'],
rpn_ret['bbox_outside_weights'])
return_dict['losses']['loss_cls'] = loss_cls
return_dict['losses']['loss_bbox'] = loss_bbox
return_dict['metrics']['accuracy_cls'] = accuracy_cls
if cfg.MODEL.MASK_ON:
if getattr(self.Mask_Head, 'SHARE_RES5', False):
mask_feat = self.Mask_Head(
res5_feat,
rpn_ret,
roi_has_mask_int32=rpn_ret['roi_has_mask_int32'])
else:
mask_feat = self.Mask_Head(blob_conv, rpn_ret)
mask_pred = self.Mask_Outs(mask_feat)
# return_dict['mask_pred'] = mask_pred
# mask loss
loss_mask = mask_rcnn_heads.mask_rcnn_losses(
mask_pred, rpn_ret['masks_int32'])
return_dict['losses']['loss_mask'] = loss_mask
if cfg.MODEL.KEYPOINTS_ON:
if getattr(self.Keypoint_Head, 'SHARE_RES5', False):
# No corresponding keypoint head implemented yet (Neither in Detectron)
# Also, rpn need to generate the label 'roi_has_keypoints_int32'
kps_feat = self.Keypoint_Head(
res5_feat,
rpn_ret,
roi_has_keypoints_int32=rpn_ret[
'roi_has_keypoint_int32'])
else:
kps_feat = self.Keypoint_Head(blob_conv, rpn_ret)
kps_pred = self.Keypoint_Outs(kps_feat)
# return_dict['keypoints_pred'] = kps_pred
# keypoints loss
if cfg.KRCNN.NORMALIZE_BY_VISIBLE_KEYPOINTS:
loss_keypoints = keypoint_rcnn_heads.keypoint_losses(
kps_pred, rpn_ret['keypoint_locations_int32'],
rpn_ret['keypoint_weights'])
else:
loss_keypoints = keypoint_rcnn_heads.keypoint_losses(
kps_pred, rpn_ret['keypoint_locations_int32'],
rpn_ret['keypoint_weights'],
rpn_ret['keypoint_loss_normalizer'])
return_dict['losses']['loss_kps'] = loss_keypoints
if cfg.MODEL.VCOCO_ON:
hoi_blob_out = self.HOI_Head(blob_conv, hoi_blob_in)
interaction_action_loss, interaction_affinity_loss, \
interaction_action_accuray_cls, interaction_affinity_cls = self.HOI_Head.loss(
hoi_blob_out)
return_dict['losses'][
'loss_hoi_interaction_action'] = interaction_action_loss
return_dict['metrics'][
'accuracy_interaction_cls'] = interaction_action_accuray_cls
return_dict['losses'][
'loss_hoi_interaction_affinity'] = interaction_affinity_loss
return_dict['metrics'][
'accuracy_interaction_affinity'] = interaction_affinity_cls
# pytorch0.4 bug on gathering scalar(0-dim) tensors
for k, v in return_dict['losses'].items():
return_dict['losses'][k] = v.unsqueeze(0)
for k, v in return_dict['metrics'].items():
return_dict['metrics'][k] = v.unsqueeze(0)
else:
if not cfg.VCOCO.USE_PRECOMP_BOX:
return_dict['rois'] = rpn_ret['rois']
return_dict['cls_score'] = cls_score
return_dict['bbox_pred'] = bbox_pred
#print('return ready')
return return_dict
