def simple_test(self, img, img_meta, proposals=None, rescale=False):
"""Test without augmentation."""
assert self.with_bbox, "Bbox head must be implemented."
assert self.with_track, "Track head must be implemented"
x = self.extract_feat(img)
proposal_list = self.simple_test_rpn(
x, img_meta, self.test_cfg.rpn) if proposals is None else proposals
det_bboxes, det_labels, det_obj_ids = self.simple_test_bboxes(
x, img_meta, proposal_list, self.test_cfg.rcnn, rescale=rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
if not self.with_mask:
return bbox_results
else:
segm_results = self.simple_test_mask(x,
img_meta,
det_bboxes,
det_labels,
rescale=rescale,
det_obj_ids=det_obj_ids)
return bbox_results, segm_results
def simple_test(self, img, img_meta, proposals=None, rescale=False):
"""Test without augmentation."""
(det_bboxes,
det_labels), segm_results, det_obj_ids = self.simple_test_bboxes(
img, img_meta, proposals, rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
return bbox_results, segm_results
def low_test(self, img, img_meta, key_img, key_feats, proposals=None, rescale=False):
"""Test with Flow Head."""
full_img = key_img
current_img = img
feat_last = key_feats
size_full = full_img.shape
size_current = current_img.shape
if size_full[-1] >= size_current[-1]:
size = size_current[-2:]
full_img = self.resize(full_img, size)
else:
size = size_full[-2:]
current_img = self.resize(current_img, size)
# flow
flow_output = self.flow_head(full_img, current_img)
upsampling = None
if upsampling is not None:
flow_output = F.interpolate(flow_output, size=current_img.size()[-2:], mode='bilinear', align_corners=False)
# extract feat
x, _ = self.extract_feat(img)
# img_size = img.shape[-2:]
# x = self.warp_feat(feat_last, flow_output*20, (img_size[0], img_size[1]), mode='warp')
proposal_list = self.simple_test_rpn(
x, img_meta, self.test_cfg.rpn) if proposals is None else proposals
det_bboxes, det_labels, det_obj_ids = self.simple_test_bboxes(
x, img_meta, proposal_list, self.test_cfg.rcnn, rescale=rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
def feat_fusion(x, feat_last):
feat = []
for i in range(len(x)):
_size = x[i].shape[-2:]
_tmp = torch.nn.functional.interpolate(feat_last[i], _size, mode='bilinear', align_corners=True)
# feat.append(torch.max(x[i], _tmp))
# feat.append((x[i] + _tmp) / 2)
feat.append((x[i] + _tmp))
return tuple(feat)
img_size = img.shape[-2:]
x = self.warp_feat(feat_last, flow_output*20, (img_size[0], img_size[1]), mode='warp')
# x_flow = self.warp_feat(feat_last, flow_outputs1)
# x1 = [resize(x_f, x11.shape[-2:]) for x_f, x11 in zip(x_flow, x)]
# x1 = feat_fusion(x, x_flow)
if not self.with_mask:
return bbox_results
else:
segm_results = self.simple_test_mask(
x, img_meta, det_bboxes, det_labels,
rescale=rescale, det_obj_ids=det_obj_ids)
return bbox_results, segm_results, None
def simple_test(self, img, img_meta, rescale=False):
x = self.extract_feat(img)
outs = self.bbox_head(x, x, False)
bbox_inputs = outs + (img_meta, self.test_cfg, rescale)
bbox_list = self.bbox_head.get_bboxes(*bbox_inputs)
bbox_results = [
bbox2result_with_id(det_bboxes, det_labels, det_obj_ids)
for det_bboxes, det_labels, cls_segms, det_obj_ids in bbox_list
]
segm_results = [
cls_segms
for det_bboxes, det_labels, cls_segms, det_obj_ids in bbox_list
]
return bbox_results[0], segm_results[0]
def simple_test(self, img, img_meta, proposals=None, rescale=False):
"""Test without augmentation."""
assert self.with_bbox, "Bbox head must be implemented."
assert self.with_track, "Track head must be implemented"
import time
t0 = time.time()
x = self.extract_feat(img)
t1 = time.time()
# print("time cost from begin to feature map is: {}".format(t1 - t0))
proposal_list = self.simple_test_rpn(
x, img_meta, self.test_cfg.rpn) if proposals is None else proposals
t2 = time.time()
# print("time cost from feature map to proposal list is: {}".format(t2 - t1))
det_bboxes, det_labels, det_obj_ids = self.simple_test_bboxes(
x, img_meta, proposal_list, self.test_cfg.rcnn, rescale=rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
t3 = time.time()
# print("time cost from begin to bbox and label is: {}".format(t3 - t2))
if not self.with_mask:
return bbox_results
else:
segm_results = self.simple_test_mask(x,
img_meta,
det_bboxes,
det_labels,
rescale=rescale,
det_obj_ids=det_obj_ids)
t4 = time.time()
# print("time cost from bbox to mask is: {}".format(t4 - t3))
# print("time cost is: {}\t {}\t {}\t {}\t".format(t1 - t0, t2 - t1, t3 - t2, t4 - t3))
# return bbox_results, segm_results, np.array([t1 - t0, t2 - t1, t3 - t2, t4 - t3])
return bbox_results, segm_results
def full_test(self, img, img_meta, proposals=None, rescale=False):
"""Test without Flow Head."""
x, _ = self.extract_feat(img)
key_feat_maps = x
proposal_list = self.simple_test_rpn(
x, img_meta, self.test_cfg.rpn) if proposals is None else proposals
det_bboxes, det_labels, det_obj_ids = self.simple_test_bboxes(
x, img_meta, proposal_list, self.test_cfg.rcnn, rescale=rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
if not self.with_mask:
return bbox_results
else:
segm_results = self.simple_test_mask(
x, img_meta, det_bboxes, det_labels,
rescale=rescale, det_obj_ids=det_obj_ids)
return bbox_results, segm_results, key_feat_maps
def simple_test(self,
img,
img_meta,
proposals=None,
rescale=False,
ref_img=None):
"""Test without augmentation."""
im_info = np.array([[float(img.shape[2]), float(img.shape[3]), 1.0]])
x = self.extract_feat(img)
# **********************************
# FCN Semantic Head forward test
# **********************************
assert hasattr(
self, 'panopticFPN'
) and self.panopticFPN is not None, "FCN HEAD must be implemented."
fcn_output, fcn_score = self.panopticFPN(
x[0:self.panopticFPN.num_levels])
# ***************************
# RPN forward test
# ***************************
assert proposals is None
proposal_list = self.simple_test_rpn(x, img_meta, self.test_cfg.rpn)
# *******************************
# bbox head forward test
# *******************************
assert self.with_bbox, "Bbox head must be implemented."
assert self.with_track, "Track head must be implemented."
det_bboxes, det_labels, det_obj_ids, best_match_scores, cls_score, bbox_pred, cls_prob, mask_rois, cls_idx = \
self.simple_test_bboxes(
x, img_meta, proposal_list,
self.test_cfg.rcnn, im_info,
rescale=rescale, is_panoptic=True)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
# *******************************
# mask head forward test
# *******************************
assert self.with_bbox, "Mask head must be implemented."
mask_results = self.simple_test_mask(x,
img_meta,
det_bboxes,
det_labels,
rescale=rescale,
det_obj_ids=det_obj_ids)
# *******************************
# Panoptic head forward test
# *******************************
assert hasattr(self, 'panopticFPN') and self.panopticFPN is not None
assert hasattr(self.test_cfg, 'loss_pano_weight')
mask_feats = self.mask_roi_extractor(
x[:self.mask_roi_extractor.num_inputs], mask_rois)
# [#bbox,256,14,14]
mask_score = self.mask_head(mask_feats)
# [#bbox,#things+1,28,28], # things+1=9 (Cityscapes-thing)
nobj, _, H, W = mask_score.shape
mask_score = mask_score.gather(
1,
cls_idx.view(-1, 1, 1, 1).expand(-1, -1, H, W))
# [#bbox,1,28,28]
# compute panoptic logits
keep_inds, mask_logits = self.mask_removal(mask_rois[:, 1:], cls_prob,
mask_score, cls_idx,
fcn_output.shape[2:])
# convert det_obj_ids to torch tensor
det_obj_ids = torch.from_numpy(det_obj_ids).to(cls_idx.device)
# mask logits [1,k,1024,2048]
mask_rois = mask_rois[keep_inds] #[k,5]
cls_idx = cls_idx[keep_inds] #[k]
det_labels = det_labels[keep_inds] #[k]
det_obj_ids = det_obj_ids[keep_inds] #[k]
cls_prob = cls_prob[keep_inds] #[k]
# get semantic segm logits
seg_stuff_logits, seg_inst_logits = self.seg_term(
cls_idx, fcn_output, mask_rois * 4.0)
# seg_stuff_logits [1,11,1024,2048]
# seg_inst_logits [1,k,1024,2048]
panoptic_logits = torch.cat(
[seg_stuff_logits, (seg_inst_logits + mask_logits)], dim=1)
panoptic_output = torch.max(F.softmax(panoptic_logits, dim=1),
dim=1)[1]
# fcn_output [1,1024,2048]
fcn_output = torch.max(F.softmax(fcn_output, dim=1), dim=1)[1]
# trim out the padded boundaries --> original input shape
img_shape_withoutpad = img_meta[0]['img_shape']
fcn_output = fcn_output[:, 0:img_shape_withoutpad[0],
0:img_shape_withoutpad[1]]
panoptic_output = panoptic_output[:, 0:img_shape_withoutpad[0],
0:img_shape_withoutpad[1]]
pano_results = {
'fcn_outputs': fcn_output,
'panoptic_cls_inds': cls_idx,
'panoptic_cls_prob': cls_prob,
'panoptic_det_labels': det_labels,
'panoptic_det_obj_ids': det_obj_ids,
'panoptic_outputs': panoptic_output,
}
return bbox_results, mask_results, pano_results
def simple_test(self,
img,
img_meta,
proposals=None,
rescale=False,
key_frame=False):
"""Test without augmentation."""
assert self.with_bbox, "Bbox head must be implemented."
assert self.with_track, "Track head must be implemented"
is_first = img_meta[0]['is_first']
if is_first:
self.prev_img = img
# extract feature maps
x, feat_res0 = self.extract_feat(img)
# save feature maps of key frame.
if key_frame:
self.key_feat_maps = x
self.key_feat_res0 = feat_res0
feat_maps = list(x)
# get proposal list
proposal_list = self.simple_test_rpn(
x, img_meta, self.test_cfg.rpn) if proposals is None else proposals
# get bbox results
det_bboxes, det_labels, det_obj_ids = self.simple_test_bboxes(
x, img_meta, proposal_list, self.test_cfg.rcnn, rescale=rescale)
bbox_results = bbox2result_with_id(det_bboxes, det_labels, det_obj_ids,
self.bbox_head.num_classes)
if not self.with_mask:
return bbox_results
else:
# get segmentation results using original net if current frame is key frame.
if key_frame:
segm_results = self.simple_test_mask(x,
img_meta,
det_bboxes,
det_labels,
rescale=rescale,
det_obj_ids=det_obj_ids)
return bbox_results, segm_results
# get segmentation results using flownet and feature warping
# if current frame is not key frame.
else:
# Turn feature maps to certain size.
# (b, c, 48, 64) for feature map and (b, c, 96, 128) for resnet-50 layer0 feature.
current_feat_map = self.resize(feat_maps[0])
key_feat_map = self.resize(self.key_feat_maps[0])
key_feat_res0 = self.resize(self.key_feat_res0, size=(96, 128))
key_feat_res0 = torch.cat((key_feat_res0, key_feat_res0),
1) # channel from 64 to 128
# get flow results
flow_output = self.flow_head(current_feat_map,
[key_feat_res0, key_feat_map])
# print("max flow:{}\t min flow:{}".format(torch.max(flow_output), torch.min(flow_output)))
# visualization
# rgb_flow = self.flow_head.flow2rgb(20 * flow_output[0], max_value=20)
# to_save = (rgb_flow * 255).astype(np.uint8).transpose(1,2,0)
# mmcv.imwrite(to_save, '/home/ubuntu/code/fengda/MaskTrackRCNN/flow.jpg')
# feature warping
current_feat_warped = warp(key_feat_map, flow_output)
# rescale feature map
feat_maps[0] = self.resize(current_feat_warped,
size=x[0].shape[-2:])
# get segmentation results
segm_results = self.simple_test_mask(tuple(feat_maps),
img_meta,
det_bboxes,
det_labels,
rescale=rescale,
det_obj_ids=det_obj_ids)
return bbox_results, segm_results
def simple_test(self,
img,
img_meta,
inp_seq=None,
rescale=False,
show=False):
"""Test without augmentation."""
assert self.with_bbox, "Bbox head must be implemented."
bbox_result_seq = []
segm_result_seq = []
mask_cluster_seq = []
mask_cluster_orig_seq = []
M_seq = []
if self.bidirectional:
for inp in inp_seq:
fms = list(self.backbone(inp['img'][0]))
M_seq.append(fms)
fms_f = []
hidden = None
for fms in M_seq:
hidden = self.gru_model(fms[self.rnn_level], hidden)
fms_f.append(hidden[-1])
fms_b = []
hidden = None
for fms in reversed(M_seq):
hidden = self.gru_model(fms[self.rnn_level], hidden)
fms_b.append(hidden[-1])
fms_b.reverse()
for i, (fm_f, fm_b) in enumerate(zip(fms_f, fms_b)):
M_seq[i][self.rnn_level] = torch.cat((fm_f, fm_b), 1)
M_seq[i] = self.neck(M_seq[i])
else:
hidden = None
for im in img_seq:
fms = list(self.forward_to_neck(im))
hidden = self.gru_model(fms[self.rnn_level], hidden)
fms[self.rnn_level] = hidden[-1]
M_seq.append(fms)
#extract end
for e, (x, inp) in enumerate(zip(M_seq, inp_seq)):
img_meta = inp['img_meta'][0]
proposal_list = self.simple_test_rpn(x, img_meta,
self.test_cfg.rpn)
det_bboxes, det_labels = self.simple_test_bboxes(
x,
img_meta,
proposal_list,
self.test_cfg.rcnn,
rescale=rescale)
segm_results, mask_clusters_orig, roi_cluster_ids, roi_indeces, self.mask_roi_map = self.simple_video_test_mask(
x,
img_meta,
det_bboxes,
det_labels,
self.mask_roi_map,
rescale=rescale)
mask_clusters_o = np.concatenate(
(mask_clusters_orig, np.zeros_like(
mask_clusters_orig[:, :, :1])),
axis=2)
mask_clusters_o = mask_clusters_o.argmax(2)
if e == 0:
if self.prev_mask_clusters is not None:
matched_idx = matching(mask_clusters_o,
self.prev_mask_clusters, self.ids)
self.inv_mask = np.concatenate(
[np.zeros(1), self.ids[matched_idx]])
mask_clusters = self.inv_mask[mask_clusters_o]
def inv_roi(orig_roi_id):
key_list = list(self.mask_roi_map.keys())
val_list = list(self.mask_roi_map.values())
orig_mask_id = key_list[val_list.index(orig_roi_id)]
mask_id = self.inv_mask[orig_mask_id]
return self.prev_mask_roi_map[mask_id]
if self.prev_mask_roi_map:
if isinstance(segm_results, dict):
segm_results = {
self.inv_mask[roi_cluster_id]: obj_segm
for roi_cluster_id, obj_segm in segm_results.items()
}
roi_cluster_ids = [
inv_roi(roi_cluster_id)
for roi_cluster_id in roi_cluster_ids
]
else:
self.prev_mask_roi_map = self.mask_roi_map
det_bboxes = det_bboxes[roi_indeces]
det_labels = det_labels[roi_indeces]
bbox_results = bbox2result_with_id(det_bboxes, det_labels,
roi_cluster_ids,
self.bbox_head.num_classes)
bbox_result_seq.append(bbox_results)
segm_result_seq.append(segm_results)
mask_clusters_vis = np.concatenate(
((mask_clusters == 1)[:, :, None],
(mask_clusters == 2)[:, :, None],
(mask_clusters == 3)[:, :, None]),
axis=2)
mask_cluster_seq.append(mask_clusters_vis)
mask_cluster_orig_seq.append(mask_clusters_orig)
if e == len(M_seq) - 1:
self.prev_mask_clusters = mask_clusters
self.prev_mask_roi_map = self.mask_roi_map
if show:
return bbox_result_seq, segm_result_seq, mask_cluster_seq, mask_cluster_orig_seq
else:
return bbox_result_seq, segm_result_seq