def forward(self, temp, target, bboxes): fmap_tmp = self.backbone(temp) fmap_tmp = self.upsample(fmap_tmp) fmap_tgt = self.backbone(target) fmap_tgt = self.upsample(fmap_tgt) map_tmp = self.extra1(fmap_tmp) map_tgt = self.extra2(fmap_tgt) # print(bboxes.shape) # print(map_tmp.shape, map_tgt.shape) small_maps = roi_align(map_tmp, bboxes, (7, 7), 1, -1) shape = small_maps.shape small_maps = small_maps.view(-1, shape[2] * shape[3]) small_maps = self.post_template(small_maps) small_maps = small_maps.view(-1, config.num_pts_tracker, shape[1], shape[2], shape[3]) res = [] map_tgt = map_tgt.unsqueeze(1) for i in range(map_tgt.shape[0]): buff = F.conv2d(map_tgt[i], small_maps[i], None, 1, 3, 1, 1) # 3 is padding res.append(buff) res = torch.cat(res, dim=0) return res
def forward(self, xs, proposal): ''' :param xs(list,len=5): [p2,p3,p4,p5,p6], that is fpn's output :param proposal(list,len=bs): list(proposal) proposal.shape=[np+nn,4] :return: ''' # list(featuremap), len=len(filter_levl)=4 [p2,p3,p4,p5] filter_features = list() for i in self.filter_level: filter_features.append(xs[i]) levels = self.mapper( proposal) # shape=[sum_of_batch_proposals] 元素取值范围[0,1,2,3] rois = self.convert_to_roi_format( proposal ) # shape=[sum_proposal_of_bs_img,5] 5==>batch_id,x1,y1,x2,y2 result = torch.zeros( (len(rois), xs[0].shape[1], self.output_size, self.output_size), dtype=xs[0].dtype, device=rois.device) for i in range(len(self.filter_level)): #[0,1,2,3] per_level_feature = filter_features[i] scale = self.scales[i] level_mask = levels == i rois_per_level = rois[level_mask] result_idx_in_level = roi_align(per_level_feature, rois_per_level, output_size=self.output_size, spatial_scale=scale, sampling_ratio=self.sampling_ratio) result[level_mask, ...] = result_idx_in_level return result # shape=[len(rois),c,output_size,output_size]
def __call__(self, feature, proposal, image_shape): idx = proposal.new_full((proposal.shape[0], 1), 0) roi = torch.cat((idx, proposal), dim=1) self.setup_scale(feature.shape[-2:], image_shape) return roi_align(feature.to(roi), roi, self.output_size, self.spatial_scale, self.sampling_ratio)
def extract_src(self, temp, bboxes): with torch.no_grad(): fmap_tmp = self.backbone(temp) fmap_tmp = self.upsample(fmap_tmp) map_tmp = self.extra1(fmap_tmp) small_maps = roi_align(map_tmp, bboxes, (7, 7), 1, -1) shape = small_maps.shape small_maps = small_maps.view(-1, shape[2] * shape[3]) small_maps = self.post_template(small_maps) small_maps = small_maps.view(-1, config.num_pts_tracker, shape[1], shape[2], shape[3]) return small_maps
def mask_loss(mask_logit, proposal, matched_idx, label, gt_mask): matched_idx = matched_idx[:, None].to(proposal) roi = torch.cat((matched_idx, proposal), dim=1) M = mask_logit.shape[-1] gt_mask = gt_mask[:, None].to(roi) mask_target = roi_align(gt_mask, roi, M, 1., -1)[:, 0] idx = torch.arange(label.shape[0], device=label.device) m_loss = F.binary_cross_entropy_with_logits(mask_logit[idx, label], mask_target) return m_loss
def build_region_feas(feature_maps, boxes_list, output_crop_size=[3, 3], img_size=[224, 224]): # Building feas for each bounding box by using RoI Align # feature_maps:[N,C,H,W], where N=b*T IH, IW = img_size FH, FW = feature_maps.size()[-2:] # Feature_H, Feature_W region_feas = roi_align(feature_maps, boxes_list, output_crop_size, spatial_scale=float(FW) / IW) # b*T*K, C, S, S; S denotes output_size return region_feas.view(region_feas.size(0), -1) # b*T*K, D*S*S
def process_metadata(feats, anchors, regression, classification, objectness, regressBoxes, clipBoxes, input_shape): transformed_anchors = regressBoxes(anchors, regression) transformed_anchors = clipBoxes(transformed_anchors, input_shape) scores = torch.max(classification, dim=2, keepdim=True)[0] # obj_scores = torch.max(objectness, dim=2, keepdim=True)[0] obj_scores = objectness[:, :] scores_over_thresh = (scores > 0.2)[:, :, 0] out = [] # for i in range(feats.shape[0]): for i in range(input_shape[0]): if scores_over_thresh[i].sum() == 0: out.append({ 'rois': np.array(()), 'class_ids': np.array(()), 'scores': np.array(()), 'features': np.array(()), 'emb_idx': np.array(()), }) continue classification_per = classification[i, scores_over_thresh[i, :], ...].permute(1, 0) transformed_anchors_per = transformed_anchors[i, scores_over_thresh[i, :], ...] transformed_anchors_obj = transformed_anchors[i, ...] scores_per = scores[i, scores_over_thresh[i, :], ...] scores_, classes_ = classification_per.max(dim=0) anchors_nms_idx = batched_nms(transformed_anchors_per, scores_per[:, 0], classes_, iou_threshold=0.2) transformed_anchors_per = transformed_anchors_per/input_shape[2] # print("ras", roi_aligned.shape) if anchors_nms_idx.shape[0] != 0: classes_ = classes_[anchors_nms_idx] scores_ = scores_[anchors_nms_idx] boxes_ = transformed_anchors_per[anchors_nms_idx, :] # nms_obj_scores = torch.zeros_like(obj_scores) # nms_obj_scores = obj_scores[:, anchors_nms_idx, :] # print("anmis", anchors_nms_idx.shape) # obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(nms_obj_scores[i], dim=0, descending=True) # obj_scores_over_thresh = (obj_scores > 0.7)[:, :, 0] # obj_scores_over_thresh = obj_scores_over_thresh[:, :50, 0] # obj_idx_over_thresh = obj_idx_over_thresh[:50, 0] # print("taos0", transformed_anchors_obj.shape) # transformed_anchors_obj = transformed_anchors_obj[obj_idx_over_thresh, :] # print("taos1", transformed_anchors_obj.shape) # print("cs0", classes_.shape) cropped_box_feats = [] all_obj_idx_over_thresh = [] all_obj_scores_over_thresh = [] roi_size = (32, 32) anchor_subsets = [0, 36864, 46080, 48384, 48960, 49104] for id, f_map in enumerate(feats): current_level_idx = anchors_nms_idx[anchors_nms_idx>anchor_subsets[id]] current_level_idx = current_level_idx[current_level_idx<anchor_subsets[id+1]] # print("oss", obj_scores.shape) nms_obj_scores = obj_scores[:, current_level_idx, :] # print("nmsos", nms_obj_scores.shape) obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(nms_obj_scores[i], dim=0, descending=True) # print("iods", obj_idx_over_thresh.shape) obj_idx_over_thresh = obj_idx_over_thresh[:25, 0] obj_scores_over_thresh = obj_scores_over_thresh[:25] all_obj_idx_over_thresh.append(obj_idx_over_thresh) all_obj_scores_over_thresh.append(obj_scores_over_thresh) # print("xxx", obj_scores_over_thresh.shape) current_level_anchors_obj = transformed_anchors_obj[obj_idx_over_thresh, :] # print("fms", f_map.shape) # print("oits", obj_idx_over_thresh.shape) roi_aligned = roi_align(f_map, [current_level_anchors_obj], roi_size) # print("ras", roi_aligned.shape) cropped_box_feats.append(roi_aligned) # cropped_box_feats = torch.from_numpy(np.array(cropped_box_feats)).cuda() # cropped_box_feats = cropped_box_feats all_obj_idx_over_thresh = torch.cat(all_obj_idx_over_thresh, dim=-1) all_obj_scores_over_thresh = torch.cat(all_obj_scores_over_thresh, dim=0) # print("aosots", all_obj_scores_over_thresh.shape) # print("aoiots", all_obj_idx_over_thresh.shape) obj_scores_over_thresh, obj_idx_over_thresh = torch.sort(all_obj_scores_over_thresh, dim=0, descending=True)[:25] # print("0", obj_scores_over_thresh) # print("1", obj_idx_over_thresh) # print("aoiots", all_obj_idx_over_thresh.shape) # print("2", all_obj_idx_over_thresh[obj_idx_over_thresh]) cropped_box_feats = torch.cat(cropped_box_feats, dim=0) # print("cbfs", cropped_box_feats.shape) features_ = cropped_box_feats classes_ = classes_[all_obj_idx_over_thresh] scores_ = scores_[all_obj_idx_over_thresh] boxes_ = boxes_[all_obj_idx_over_thresh, :] # print("fs", features_.shape) out.append({ 'rois': boxes_, 'class_ids': classes_, 'scores': scores_, 'features': features_, 'emb_idx': all_obj_idx_over_thresh, }) else: out.append({ 'rois': np.array(()), 'class_ids': np.array(()), 'scores': np.array(()), 'features': np.array(()), 'emb_idx': np.array(()), }) return out