def _get_bboxes_single(self, cls_score, bbox_pred, img_shape, scale_factor, rescale=False): """Transform outputs from the last decoder layer into bbox predictions for each image. Args: cls_score (Tensor): Box score logits from the last decoder layer for each image. Shape [num_query, cls_out_channels]. bbox_pred (Tensor): Sigmoid outputs from the last decoder layer for each image, with coordinate format (cx, cy, w, h) and shape [num_query, 4]. img_shape (tuple[int]): Shape of input image, (height, width, 3). scale_factor (ndarray, optional): Scale factor of the image arange as (w_scale, h_scale, w_scale, h_scale). rescale (bool, optional): If True, return boxes in original image space. Default False. Returns: tuple[Tensor]: Results of detected bboxes and labels. - det_bboxes: Predicted bboxes with shape [num_query, 5], \ where the first 4 columns are bounding box positions \ (tl_x, tl_y, br_x, br_y) and the 5-th column are scores \ between 0 and 1. - det_labels: Predicted labels of the corresponding box with \ shape [num_query]. """ assert len(cls_score) == len(bbox_pred) max_per_img = self.test_cfg.get('max_per_img', self.num_query) # exclude background if self.loss_cls.use_sigmoid: cls_score = cls_score.sigmoid() scores, indexs = cls_score.view(-1).topk(max_per_img) det_labels = indexs % self.num_classes bbox_index = indexs // self.num_classes bbox_pred = bbox_pred[bbox_index] else: scores, det_labels = F.softmax(cls_score, dim=-1)[..., :-1].max(-1) scores, bbox_index = scores.topk(max_per_img) bbox_pred = bbox_pred[bbox_index] det_labels = det_labels[bbox_index] det_bboxes = bbox_cxcywh_to_xyxy(bbox_pred) det_bboxes[:, 0::2] = det_bboxes[:, 0::2] * img_shape[1] det_bboxes[:, 1::2] = det_bboxes[:, 1::2] * img_shape[0] det_bboxes[:, 0::2].clamp_(min=0, max=img_shape[1]) det_bboxes[:, 1::2].clamp_(min=0, max=img_shape[0]) if rescale: det_bboxes /= det_bboxes.new_tensor(scale_factor) det_bboxes = torch.cat((det_bboxes, scores.unsqueeze(1)), -1) return det_bboxes, det_labels
def avg_iou_cost(anchor_params, bboxes): assert len(anchor_params) % 2 == 0 anchor_whs = torch.tensor([ [w, h] for w, h in zip(anchor_params[::2], anchor_params[1::2]) ]).to(bboxes.device, dtype=bboxes.dtype) anchor_boxes = bbox_cxcywh_to_xyxy( torch.cat([torch.zeros_like(anchor_whs), anchor_whs], dim=1)) ious = bbox_overlaps(bboxes, anchor_boxes) max_ious, _ = ious.max(1) cost = 1 - max_ious.mean().item() return cost
def get_zero_center_bbox_tensor(self): """Get a tensor of bboxes centered at (0, 0). Returns: Tensor: Tensor of bboxes with shape (num_bboxes, 4) in [xmin, ymin, xmax, ymax] format. """ whs = torch.from_numpy(self.bbox_whs).to(self.device, dtype=torch.float32) bboxes = bbox_cxcywh_to_xyxy( torch.cat([torch.zeros_like(whs), whs], dim=1)) return bboxes
def loss_single(self, cls_scores, bbox_preds, gt_bboxes_list, gt_labels_list, img_metas, gt_bboxes_ignore_list=None): """"Loss function for outputs from a single decoder layer of a single feature level. Args: cls_scores (Tensor): Box score logits from a single decoder layer for all images. Shape [bs, num_query, cls_out_channels]. bbox_preds (Tensor): Sigmoid outputs from a single decoder layer for all images, with normalized coordinate (cx, cy, w, h) and shape [bs, num_query, 4]. gt_bboxes_list (list[Tensor]): Ground truth bboxes for each image with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. gt_labels_list (list[Tensor]): Ground truth class indices for each image with shape (num_gts, ). img_metas (list[dict]): List of image meta information. gt_bboxes_ignore_list (list[Tensor], optional): Bounding boxes which can be ignored for each image. Default None. Returns: dict[str, Tensor]: A dictionary of loss components for outputs from a single decoder layer. """ num_imgs = cls_scores.size(0) cls_scores_list = [cls_scores[i] for i in range(num_imgs)] bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] cls_reg_targets = self.get_targets(cls_scores_list, bbox_preds_list, gt_bboxes_list, gt_labels_list, img_metas, gt_bboxes_ignore_list) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets labels = torch.cat(labels_list, 0) label_weights = torch.cat(label_weights_list, 0) bbox_targets = torch.cat(bbox_targets_list, 0) bbox_weights = torch.cat(bbox_weights_list, 0) # classification loss cls_scores = cls_scores.reshape(-1, self.cls_out_channels) # construct weighted avg_factor to match with the official DETR repo cls_avg_factor = num_total_pos * 1.0 + \ num_total_neg * self.bg_cls_weight loss_cls = self.loss_cls( cls_scores, labels, label_weights, avg_factor=cls_avg_factor) # Compute the average number of gt boxes accross all gpus, for # normalization purposes num_total_pos = loss_cls.new_tensor([num_total_pos]) num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() # construct factors used for rescale bboxes factors = [] for img_meta, bbox_pred in zip(img_metas, bbox_preds): img_h, img_w, _ = img_meta['img_shape'] factor = bbox_pred.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0).repeat( bbox_pred.size(0), 1) factors.append(factor) factors = torch.cat(factors, 0) # DETR regress the relative position of boxes (cxcywh) in the image, # thus the learning target is normalized by the image size. So here # we need to re-scale them for calculating IoU loss bbox_preds = bbox_preds.reshape(-1, 4) bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors # regression IoU loss, defaultly GIoU loss loss_iou = self.loss_iou( bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) # regression L1 loss loss_bbox = self.loss_bbox( bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) return loss_cls, loss_bbox, loss_iou
def onnx_export(self, all_cls_scores_list, all_bbox_preds_list, img_metas): """Transform network outputs into bbox predictions, with ONNX exportation. Args: all_cls_scores_list (list[Tensor]): Classification outputs for each feature level. Each is a 4D-tensor with shape [nb_dec, bs, num_query, cls_out_channels]. all_bbox_preds_list (list[Tensor]): Sigmoid regression outputs for each feature level. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and shape [nb_dec, bs, num_query, 4]. img_metas (list[dict]): Meta information of each image. Returns: tuple[Tensor, Tensor]: dets of shape [N, num_det, 5] and class labels of shape [N, num_det]. """ assert len(img_metas) == 1, \ 'Only support one input image while in exporting to ONNX' cls_scores = all_cls_scores_list[-1][-1] bbox_preds = all_bbox_preds_list[-1][-1] # Note `img_shape` is not dynamically traceable to ONNX, # here `img_shape_for_onnx` (padded shape of image tensor) # is used. img_shape = img_metas[0]['img_shape_for_onnx'] max_per_img = self.test_cfg.get('max_per_img', self.num_query) batch_size = cls_scores.size(0) # `batch_index_offset` is used for the gather of concatenated tensor batch_index_offset = torch.arange(batch_size).to( cls_scores.device) * max_per_img batch_index_offset = batch_index_offset.unsqueeze(1).expand( batch_size, max_per_img) # supports dynamical batch inference if self.loss_cls.use_sigmoid: cls_scores = cls_scores.sigmoid() scores, indexes = cls_scores.view(batch_size, -1).topk(max_per_img, dim=1) det_labels = indexes % self.num_classes bbox_index = indexes // self.num_classes bbox_index = (bbox_index + batch_index_offset).view(-1) bbox_preds = bbox_preds.view(-1, 4)[bbox_index] bbox_preds = bbox_preds.view(batch_size, -1, 4) else: scores, det_labels = F.softmax(cls_scores, dim=-1)[..., :-1].max(-1) scores, bbox_index = scores.topk(max_per_img, dim=1) bbox_index = (bbox_index + batch_index_offset).view(-1) bbox_preds = bbox_preds.view(-1, 4)[bbox_index] det_labels = det_labels.view(-1)[bbox_index] bbox_preds = bbox_preds.view(batch_size, -1, 4) det_labels = det_labels.view(batch_size, -1) det_bboxes = bbox_cxcywh_to_xyxy(bbox_preds) # use `img_shape_tensor` for dynamically exporting to ONNX img_shape_tensor = img_shape.flip(0).repeat(2) # [w,h,w,h] img_shape_tensor = img_shape_tensor.unsqueeze(0).unsqueeze(0).expand( batch_size, det_bboxes.size(1), 4) det_bboxes = det_bboxes * img_shape_tensor # dynamically clip bboxes x1, y1, x2, y2 = det_bboxes.split((1, 1, 1, 1), dim=-1) from mmdet.core.export import dynamic_clip_for_onnx x1, y1, x2, y2 = dynamic_clip_for_onnx(x1, y1, x2, y2, img_shape) det_bboxes = torch.cat([x1, y1, x2, y2], dim=-1) det_bboxes = torch.cat((det_bboxes, scores.unsqueeze(-1)), -1) return det_bboxes, det_labels