def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (dict[str,Tensor]): mapping from names to backbone features
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
features = [features[f] for f in self.in_features]
if self.training:
proposals = instances
if self._embedding_gt_box:
# Use GT box to train
proposal_boxes = [x.gt_boxes for x in proposals]
else:
# Use Pred box to train
proposal_boxes = [x.proposal_boxes for x in proposals]
proposal_boxes = [x.proposal_boxes for x in proposals]
mask_features = self.mask_pooler(features, proposal_boxes)
mask_logits = self.mask_head.layers(mask_features)
return {"loss_mask": mask_rcnn_loss(mask_logits, proposals)}
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_features = self.mask_pooler(features, pred_boxes)
return self.mask_head(mask_features, instances)
def forward(self, features, instances):
"""
Args:
features (dict[str, Tensor]): a dict of image-level features
instances (list[Instances]): proposals in training; detected
instances in inference
"""
if self.training:
proposal_boxes = [x.proposal_boxes for x in instances]
mask_coarse_logits = self.coarse_head(
self._roi_pooler(features, proposal_boxes))
losses = {
"loss_mask": mask_rcnn_loss(mask_coarse_logits, instances)
}
losses.update(
self._forward_mask_point(features, mask_coarse_logits,
instances))
return losses
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_coarse_logits = self.coarse_head(
self._roi_pooler(features, pred_boxes))
mask_logits = self._forward_mask_point(features,
mask_coarse_logits,
instances)
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self, features, instances):
"""
Args:
features (dict[str, Tensor]): a dict of image-level features
instances (list[Instances]): proposals in training; detected
instances in inference
"""
if self.training:
proposal_boxes = [x.proposal_boxes for x in instances]
coarse_mask = self.coarse_head(
self._roi_pooler(features, proposal_boxes))
losses = {"loss_mask": mask_rcnn_loss(coarse_mask, instances)}
if not self.mask_point_on:
return losses
point_coords, point_labels = self._sample_train_points(
coarse_mask, instances)
point_fine_grained_features = self._point_pooler(
features, proposal_boxes, point_coords)
point_logits = self._get_point_logits(point_fine_grained_features,
point_coords, coarse_mask)
losses["loss_mask_point"] = roi_mask_point_loss(
point_logits, instances, point_labels)
return losses
else:
pred_boxes = [x.pred_boxes for x in instances]
coarse_mask = self.coarse_head(
self._roi_pooler(features, pred_boxes))
return self._subdivision_inference(features, coarse_mask,
instances)
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (list[Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
# The loss is only defined on positive proposals.
proposals, _ = select_foreground_proposals(self.train_part, instances, self.num_classes)
proposal_boxes = [x.proposal_boxes for x in proposals]
mask_features = self.mask_pooler(features, proposal_boxes)
#if len(mask_features) == 0:
# print('No X.............................')
# return 0.0, 0.0, 0.0
#else:
mask_logits = self.mask_head(mask_features)
return mask_rcnn_loss(mask_logits, proposals)
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_features = self.mask_pooler(features, pred_boxes)
mask_logits = self.mask_head(mask_features)
mask_rcnn_inference(mask_logits, instances)
return instances
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (list[Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
# NOTE: instances = foreground proposals
mask_logits = self.mask_head(features)
return {"loss_mask": mask_rcnn_loss(mask_logits, instances)}
else:
mask_logits = self.mask_head(features)
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self,
x,
instances,
similarity=None,
base_classes=None,
novel_classes=None):
x, x_delta = self.layers(x)
if similarity is not None:
if x.numel() > 0:
similarity_mask = similarity['seg']
mask_base = x.index_select(1, index=base_classes)
mask_base_reshaped = mask_base.view(*mask_base.size()[:2], -1)
if len(similarity_mask.size()) > 2:
mask_combination = torch.bmm(similarity_mask,
mask_base_reshaped)
else:
mask_combination = torch.matmul(
mask_base_reshaped.transpose(1, 2),
similarity_mask.transpose(0, 1)).transpose(1, 2)
mask_novel = mask_combination.view(mask_base.size(0), -1,
*mask_base.size()[2:])
mask_final = torch.zeros_like(x)
mask_final = mask_final.index_copy(1, novel_classes,
mask_novel)
mask_final = mask_final.index_copy(1, base_classes, mask_base)
x = mask_final
x = x + x_delta
if self.training:
assert not torch.jit.is_scripting()
return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period)}
else:
mask_rcnn_inference(x, instances)
return instances
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (dict[str, Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
proposals, _ = select_foreground_proposals(instances, self.num_classes)
proposal_boxes = [x.proposal_boxes for x in proposals]
mask_coarse_logits = self._forward_mask_coarse(features, proposal_boxes)
losses = {"loss_mask": mask_rcnn_loss(mask_coarse_logits, proposals)}
losses.update(self._forward_mask_point(features, mask_coarse_logits, proposals))
return losses
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_coarse_logits = self._forward_mask_coarse(features, pred_boxes)
mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances)
mask_rcnn_inference(mask_logits, instances)
return instances
