def forward(self, temp, target, bboxes):
fmap_tmp = self.backbone(temp)
fmap_tmp = self.upsample(fmap_tmp)
fmap_tgt = self.backbone(target)
fmap_tgt = self.upsample(fmap_tgt)
map_tmp = self.extra1(fmap_tmp)
map_tgt = self.extra2(fmap_tgt)
# print(bboxes.shape)
# print(map_tmp.shape, map_tgt.shape)
small_maps = roi_align(map_tmp, bboxes, (7, 7), 1, -1)
shape = small_maps.shape
small_maps = small_maps.view(-1, shape[2] * shape[3])
small_maps = self.post_template(small_maps)
small_maps = small_maps.view(-1, config.num_pts_tracker, shape[1],
shape[2], shape[3])
res = []
map_tgt = map_tgt.unsqueeze(1)
for i in range(map_tgt.shape[0]):
buff = F.conv2d(map_tgt[i], small_maps[i], None, 1, 3, 1,
1) # 3 is padding
res.append(buff)
res = torch.cat(res, dim=0)
return res
def forward(self, xs, proposal):
'''
:param xs(list,len=5): [p2,p3,p4,p5,p6], that is fpn's output
:param proposal(list,len=bs): list(proposal) proposal.shape=[np+nn,4]
:return:
'''
# list(featuremap), len=len(filter_levl)=4 [p2,p3,p4,p5]
filter_features = list()
for i in self.filter_level:
filter_features.append(xs[i])
levels = self.mapper(
proposal) # shape=[sum_of_batch_proposals] 元素取值范围[0,1,2,3]
rois = self.convert_to_roi_format(
proposal
) # shape=[sum_proposal_of_bs_img,5] 5==>batch_id,x1,y1,x2,y2
result = torch.zeros(
(len(rois), xs[0].shape[1], self.output_size, self.output_size),
dtype=xs[0].dtype,
device=rois.device)
for i in range(len(self.filter_level)): #[0,1,2,3]
per_level_feature = filter_features[i]
scale = self.scales[i]
level_mask = levels == i
rois_per_level = rois[level_mask]
result_idx_in_level = roi_align(per_level_feature,
rois_per_level,
output_size=self.output_size,
spatial_scale=scale,
sampling_ratio=self.sampling_ratio)
result[level_mask, ...] = result_idx_in_level
return result # shape=[len(rois),c,output_size,output_size]
def __call__(self, feature, proposal, image_shape):
idx = proposal.new_full((proposal.shape[0], 1), 0)
roi = torch.cat((idx, proposal), dim=1)
self.setup_scale(feature.shape[-2:], image_shape)
return roi_align(feature.to(roi), roi, self.output_size, self.spatial_scale,
self.sampling_ratio)
def extract_src(self, temp, bboxes):
with torch.no_grad():
fmap_tmp = self.backbone(temp)
fmap_tmp = self.upsample(fmap_tmp)
map_tmp = self.extra1(fmap_tmp)
small_maps = roi_align(map_tmp, bboxes, (7, 7), 1, -1)
shape = small_maps.shape
small_maps = small_maps.view(-1, shape[2] * shape[3])
small_maps = self.post_template(small_maps)
small_maps = small_maps.view(-1, config.num_pts_tracker, shape[1],
shape[2], shape[3])
return small_maps
def mask_loss(mask_logit, proposal, matched_idx, label, gt_mask):
matched_idx = matched_idx[:, None].to(proposal)
roi = torch.cat((matched_idx, proposal), dim=1)
M = mask_logit.shape[-1]
gt_mask = gt_mask[:, None].to(roi)
mask_target = roi_align(gt_mask, roi, M, 1., -1)[:, 0]
idx = torch.arange(label.shape[0], device=label.device)
m_loss = F.binary_cross_entropy_with_logits(mask_logit[idx, label],
mask_target)
return m_loss
def build_region_feas(feature_maps,
boxes_list,
output_crop_size=[3, 3],
img_size=[224, 224]):
# Building feas for each bounding box by using RoI Align
# feature_maps:[N,C,H,W], where N=b*T
IH, IW = img_size
FH, FW = feature_maps.size()[-2:] # Feature_H, Feature_W
region_feas = roi_align(feature_maps,
boxes_list,
output_crop_size,
spatial_scale=float(FW) /
IW) # b*T*K, C, S, S; S denotes output_size
return region_feas.view(region_feas.size(0), -1) # b*T*K, D*S*S
def process_metadata(feats, anchors, regression, classification, objectness, regressBoxes, clipBoxes, input_shape):
transformed_anchors = regressBoxes(anchors, regression)
transformed_anchors = clipBoxes(transformed_anchors, input_shape)
scores = torch.max(classification, dim=2, keepdim=True)[0]
# obj_scores = torch.max(objectness, dim=2, keepdim=True)[0]
obj_scores = objectness[:, :]
scores_over_thresh = (scores > 0.2)[:, :, 0]
out = []
# for i in range(feats.shape[0]):
for i in range(input_shape[0]):
if scores_over_thresh[i].sum() == 0:
out.append({
'rois': np.array(()),
'class_ids': np.array(()),
'scores': np.array(()),
'features': np.array(()),
'emb_idx': np.array(()),
})
continue
classification_per = classification[i, scores_over_thresh[i, :], ...].permute(1, 0)
transformed_anchors_per = transformed_anchors[i, scores_over_thresh[i, :], ...]
transformed_anchors_obj = transformed_anchors[i, ...]
scores_per = scores[i, scores_over_thresh[i, :], ...]
scores_, classes_ = classification_per.max(dim=0)
anchors_nms_idx = batched_nms(transformed_anchors_per, scores_per[:, 0], classes_, iou_threshold=0.2)
transformed_anchors_per = transformed_anchors_per/input_shape[2]
# print("ras", roi_aligned.shape)
if anchors_nms_idx.shape[0] != 0:
classes_ = classes_[anchors_nms_idx]
scores_ = scores_[anchors_nms_idx]
boxes_ = transformed_anchors_per[anchors_nms_idx, :]
# nms_obj_scores = torch.zeros_like(obj_scores)
# nms_obj_scores = obj_scores[:, anchors_nms_idx, :]
# print("anmis", anchors_nms_idx.shape)
# obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(nms_obj_scores[i], dim=0, descending=True)
# obj_scores_over_thresh = (obj_scores > 0.7)[:, :, 0]
# obj_scores_over_thresh = obj_scores_over_thresh[:, :50, 0]
# obj_idx_over_thresh = obj_idx_over_thresh[:50, 0]
# print("taos0", transformed_anchors_obj.shape)
# transformed_anchors_obj = transformed_anchors_obj[obj_idx_over_thresh, :]
# print("taos1", transformed_anchors_obj.shape)
# print("cs0", classes_.shape)
cropped_box_feats = []
all_obj_idx_over_thresh = []
all_obj_scores_over_thresh = []
roi_size = (32, 32)
anchor_subsets = [0, 36864, 46080, 48384, 48960, 49104]
for id, f_map in enumerate(feats):
current_level_idx = anchors_nms_idx[anchors_nms_idx>anchor_subsets[id]]
current_level_idx = current_level_idx[current_level_idx<anchor_subsets[id+1]]
# print("oss", obj_scores.shape)
nms_obj_scores = obj_scores[:, current_level_idx, :]
# print("nmsos", nms_obj_scores.shape)
obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(nms_obj_scores[i], dim=0, descending=True)
# print("iods", obj_idx_over_thresh.shape)
obj_idx_over_thresh = obj_idx_over_thresh[:25, 0]
obj_scores_over_thresh = obj_scores_over_thresh[:25]
all_obj_idx_over_thresh.append(obj_idx_over_thresh)
all_obj_scores_over_thresh.append(obj_scores_over_thresh)
# print("xxx", obj_scores_over_thresh.shape)
current_level_anchors_obj = transformed_anchors_obj[obj_idx_over_thresh, :]
# print("fms", f_map.shape)
# print("oits", obj_idx_over_thresh.shape)
roi_aligned = roi_align(f_map, [current_level_anchors_obj], roi_size)
# print("ras", roi_aligned.shape)
cropped_box_feats.append(roi_aligned)
# cropped_box_feats = torch.from_numpy(np.array(cropped_box_feats)).cuda()
# cropped_box_feats = cropped_box_feats
all_obj_idx_over_thresh = torch.cat(all_obj_idx_over_thresh, dim=-1)
all_obj_scores_over_thresh = torch.cat(all_obj_scores_over_thresh, dim=0)
# print("aosots", all_obj_scores_over_thresh.shape)
# print("aoiots", all_obj_idx_over_thresh.shape)
obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(all_obj_scores_over_thresh, dim=0, descending=True)[:25]
# print("0", obj_scores_over_thresh)
# print("1", obj_idx_over_thresh)
# print("aoiots", all_obj_idx_over_thresh.shape)
# print("2", all_obj_idx_over_thresh[obj_idx_over_thresh])
cropped_box_feats = torch.cat(cropped_box_feats, dim=0)
# print("cbfs", cropped_box_feats.shape)
features_ = cropped_box_feats
classes_ = classes_[all_obj_idx_over_thresh]
scores_ = scores_[all_obj_idx_over_thresh]
boxes_ = boxes_[all_obj_idx_over_thresh, :]
# print("fs", features_.shape)
out.append({
'rois': boxes_,
'class_ids': classes_,
'scores': scores_,
'features': features_,
'emb_idx': all_obj_idx_over_thresh,
})
else:
out.append({
'rois': np.array(()),
'class_ids': np.array(()),
'scores': np.array(()),
'features': np.array(()),
'emb_idx': np.array(()),
})
return out