def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (dict[str,Tensor]): mapping from names to backbone features instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances features = [features[f] for f in self.in_features] if self.training: proposals = instances if self._embedding_gt_box: # Use GT box to train proposal_boxes = [x.gt_boxes for x in proposals] else: # Use Pred box to train proposal_boxes = [x.proposal_boxes for x in proposals] proposal_boxes = [x.proposal_boxes for x in proposals] mask_features = self.mask_pooler(features, proposal_boxes) mask_logits = self.mask_head.layers(mask_features) return {"loss_mask": mask_rcnn_loss(mask_logits, proposals)} else: pred_boxes = [x.pred_boxes for x in instances] mask_features = self.mask_pooler(features, pred_boxes) return self.mask_head(mask_features, instances)
def forward(self, features, instances): """ Args: features (dict[str, Tensor]): a dict of image-level features instances (list[Instances]): proposals in training; detected instances in inference """ if self.training: proposal_boxes = [x.proposal_boxes for x in instances] mask_coarse_logits = self.coarse_head( self._roi_pooler(features, proposal_boxes)) losses = { "loss_mask": mask_rcnn_loss(mask_coarse_logits, instances) } losses.update( self._forward_mask_point(features, mask_coarse_logits, instances)) return losses else: pred_boxes = [x.pred_boxes for x in instances] mask_coarse_logits = self.coarse_head( self._roi_pooler(features, pred_boxes)) mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances) mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, features, instances): """ Args: features (dict[str, Tensor]): a dict of image-level features instances (list[Instances]): proposals in training; detected instances in inference """ if self.training: proposal_boxes = [x.proposal_boxes for x in instances] coarse_mask = self.coarse_head( self._roi_pooler(features, proposal_boxes)) losses = {"loss_mask": mask_rcnn_loss(coarse_mask, instances)} if not self.mask_point_on: return losses point_coords, point_labels = self._sample_train_points( coarse_mask, instances) point_fine_grained_features = self._point_pooler( features, proposal_boxes, point_coords) point_logits = self._get_point_logits(point_fine_grained_features, point_coords, coarse_mask) losses["loss_mask_point"] = roi_mask_point_loss( point_logits, instances, point_labels) return losses else: pred_boxes = [x.pred_boxes for x in instances] coarse_mask = self.coarse_head( self._roi_pooler(features, pred_boxes)) return self._subdivision_inference(features, coarse_mask, instances)
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (list[Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: # The loss is only defined on positive proposals. proposals, _ = select_foreground_proposals(self.train_part, instances, self.num_classes) proposal_boxes = [x.proposal_boxes for x in proposals] mask_features = self.mask_pooler(features, proposal_boxes) #if len(mask_features) == 0: # print('No X.............................') # return 0.0, 0.0, 0.0 #else: mask_logits = self.mask_head(mask_features) return mask_rcnn_loss(mask_logits, proposals) else: pred_boxes = [x.pred_boxes for x in instances] mask_features = self.mask_pooler(features, pred_boxes) mask_logits = self.mask_head(mask_features) mask_rcnn_inference(mask_logits, instances) return instances
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (list[Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: # NOTE: instances = foreground proposals mask_logits = self.mask_head(features) return {"loss_mask": mask_rcnn_loss(mask_logits, instances)} else: mask_logits = self.mask_head(features) mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, x, instances, similarity=None, base_classes=None, novel_classes=None): x, x_delta = self.layers(x) if similarity is not None: if x.numel() > 0: similarity_mask = similarity['seg'] mask_base = x.index_select(1, index=base_classes) mask_base_reshaped = mask_base.view(*mask_base.size()[:2], -1) if len(similarity_mask.size()) > 2: mask_combination = torch.bmm(similarity_mask, mask_base_reshaped) else: mask_combination = torch.matmul( mask_base_reshaped.transpose(1, 2), similarity_mask.transpose(0, 1)).transpose(1, 2) mask_novel = mask_combination.view(mask_base.size(0), -1, *mask_base.size()[2:]) mask_final = torch.zeros_like(x) mask_final = mask_final.index_copy(1, novel_classes, mask_novel) mask_final = mask_final.index_copy(1, base_classes, mask_base) x = mask_final x = x + x_delta if self.training: assert not torch.jit.is_scripting() return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period)} else: mask_rcnn_inference(x, instances) return instances
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (dict[str, Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: proposals, _ = select_foreground_proposals(instances, self.num_classes) proposal_boxes = [x.proposal_boxes for x in proposals] mask_coarse_logits = self._forward_mask_coarse(features, proposal_boxes) losses = {"loss_mask": mask_rcnn_loss(mask_coarse_logits, proposals)} losses.update(self._forward_mask_point(features, mask_coarse_logits, proposals)) return losses else: pred_boxes = [x.pred_boxes for x in instances] mask_coarse_logits = self._forward_mask_coarse(features, pred_boxes) mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances) mask_rcnn_inference(mask_logits, instances) return instances