def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (list[Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: # NOTE: instances = foreground proposals mask_logits = self.mask_head(features) return {"loss_mask": mask_rcnn_loss(mask_logits, instances)} else: mask_logits = self.mask_head(features) mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, features, instances): """ Args: features (dict[str, Tensor]): a dict of image-level features instances (list[Instances]): proposals in training; detected instances in inference """ if self.training: proposal_boxes = [x.proposal_boxes for x in instances] mask_coarse_logits = self.coarse_head( self._roi_pooler(features, proposal_boxes)) losses = { "loss_mask": mask_rcnn_loss(mask_coarse_logits, instances) } losses.update( self._forward_mask_point(features, mask_coarse_logits, instances)) return losses else: pred_boxes = [x.pred_boxes for x in instances] mask_coarse_logits = self.coarse_head( self._roi_pooler(features, pred_boxes)) mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances) mask_rcnn_inference(mask_logits, instances) return instances
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (list[Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: # The loss is only defined on positive proposals. proposals, _ = select_foreground_proposals(self.train_part, instances, self.num_classes) proposal_boxes = [x.proposal_boxes for x in proposals] mask_features = self.mask_pooler(features, proposal_boxes) #if len(mask_features) == 0: # print('No X.............................') # return 0.0, 0.0, 0.0 #else: mask_logits = self.mask_head(mask_features) return mask_rcnn_loss(mask_logits, proposals) else: pred_boxes = [x.pred_boxes for x in instances] mask_features = self.mask_pooler(features, pred_boxes) mask_logits = self.mask_head(mask_features) mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, mask_features, boundary_features, instances): #(27,256,28,28),(27,256,28,28) x1 = self.inc(mask_features) #(27,256,28,28) boundary_features = boundary_features + self.mask_to_boundary( x1) #27,256,28,28 boundary_features = self.inc(boundary_features) #27,256,28,28 x2 = self.down1(x1) #(27,512,14,14) # x2 = F.interpolate(self.boundary_to_mask1(boundary_features),scale_factor=1/2, mode='bilinear', align_corners=True) + x2 #插值合适还是下采样卷积合适 boundary_features_tmp = self.boundary_to_mask1( boundary_features) #27,512,14,14 x2 = boundary_features_tmp + x2 #27,512,14,14 x3 = self.down2(x2) #(27,1024,7,7) x3 = self.boundary_to_mask2(boundary_features_tmp) + x3 #27,1024,7,7 x = self.up1(x3, x2) #(27,512,14,14) x = self.up2(x, x1) #(27,256,28,28) mask_logits = self.outc(x) #(27,2,28,28) boundary_logits = self.outc(boundary_features) if self.training: loss_mask, loss_boundary = boundary_preserving_mask_loss( mask_logits, boundary_logits, instances) return {"loss_mask": loss_mask, "loss_boundary": loss_boundary} else: mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, mask_features, context_feature, instances: List[Instances]): # self.mask_fcns # self.downsample # self.context_fcns # self.context_to_mask # self.mask_final_fusion # self.mask_deconv # self.mask_predictor for layer in self.mask_fcns: mask_features = layer(mask_features) # downsample context_feature = self.downsample(context_feature) for layer in self.context_fcns: context_feature = layer(context_feature) # context to mask fusion mask_features = self.context_to_mask(context_feature) + mask_features mask_features = self.mask_final_fusion(mask_features) # mask prediction mask_features = F.relu(self.mask_deconv(mask_features)) mask_logits = self.mask_predictor(mask_features) if self.training: return { "loss_mask": mask_rcnn_loss(mask_logits, instances, self.vis_period) * self.loss_weight } else: mask_rcnn_inference(mask_logits, instances) return instances
def forward(self, x, instances, similarity=None, base_classes=None, novel_classes=None): x, x_delta = self.layers(x) if similarity is not None: if x.numel() > 0: similarity_mask = similarity['seg'] mask_base = x.index_select(1, index=base_classes) mask_base_reshaped = mask_base.view(*mask_base.size()[:2], -1) if len(similarity_mask.size()) > 2: mask_combination = torch.bmm(similarity_mask, mask_base_reshaped) else: mask_combination = torch.matmul( mask_base_reshaped.transpose(1, 2), similarity_mask.transpose(0, 1)).transpose(1, 2) mask_novel = mask_combination.view(mask_base.size(0), -1, *mask_base.size()[2:]) mask_final = torch.zeros_like(x) mask_final = mask_final.index_copy(1, novel_classes, mask_novel) mask_final = mask_final.index_copy(1, base_classes, mask_base) x = mask_final x = x + x_delta if self.training: assert not torch.jit.is_scripting() return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period)} else: mask_rcnn_inference(x, instances) return instances
def forward(self, mask_features, boundary_features, instances: List[Instances]): for layer in self.mask_fcns: mask_features = layer(mask_features) # downsample boundary_features = self.downsample(boundary_features) # mask to boundary fusion boundary_features = boundary_features + self.mask_to_boundary( mask_features) for layer in self.boundary_fcns: boundary_features = layer(boundary_features) # boundary to mask fusion mask_features = self.boundary_to_mask( boundary_features) + mask_features mask_features = self.mask_final_fusion(mask_features) # mask prediction mask_features = F.interpolate(mask_features, scale_factor=2, mode='bilinear') mask_logits = self.mask_predictor(mask_features) # boundary prediction boundary_features = F.interpolate(boundary_features, scale_factor=2, mode='bilinear') boundary_logits = self.boundary_predictor(boundary_features) if self.training: loss_mask, loss_boundary = boundary_preserving_mask_loss( mask_logits, boundary_logits, instances) return {"loss_mask": loss_mask, "loss_boundary": loss_boundary} else: mask_rcnn_inference(mask_logits, instances) return instances
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (dict[str, Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: proposals, _ = select_foreground_proposals(instances, self.num_classes) proposal_boxes = [x.proposal_boxes for x in proposals] mask_coarse_logits = self._forward_mask_coarse(features, proposal_boxes) losses = {"loss_mask": mask_rcnn_loss(mask_coarse_logits, proposals)} losses.update(self._forward_mask_point(features, mask_coarse_logits, proposals)) return losses else: pred_boxes = [x.pred_boxes for x in instances] mask_coarse_logits = self._forward_mask_coarse(features, pred_boxes) mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances) mask_rcnn_inference(mask_logits, instances) return instances
def _subdivision_inference(self, features, mask_representations, instances): assert not self.training pred_boxes = [x.pred_boxes for x in instances] pred_classes = cat([x.pred_classes for x in instances]) mask_logits = None # +1 here to include an initial step to generate the coarsest mask # prediction with init_resolution, when mask_logits is None. # We compute initial mask by sampling on a regular grid. coarse_mask # can be used as initial mask as well, but it's typically very low-res # so it will be completely overwritten during subdivision anyway. for _ in range(self.mask_point_subdivision_steps + 1): if mask_logits is None: point_coords = generate_regular_grid_point_coords( pred_classes.size(0), self.mask_point_subdivision_init_resolution, pred_boxes[0].device, ) else: mask_logits = interpolate( mask_logits, scale_factor=2, mode="bilinear", align_corners=False ) uncertainty_map = calculate_uncertainty(mask_logits, pred_classes) point_indices, point_coords = get_uncertain_point_coords_on_grid( uncertainty_map, self.mask_point_subdivision_num_points ) # Run the point head for every point in point_coords fine_grained_features = self._point_pooler(features, pred_boxes, point_coords) point_logits = self._get_point_logits( fine_grained_features, point_coords, mask_representations ) if mask_logits is None: # Create initial mask_logits using point_logits on this regular grid R, C, _ = point_logits.shape mask_logits = point_logits.reshape( R, C, self.mask_point_subdivision_init_resolution, self.mask_point_subdivision_init_resolution, ) # The subdivision code will fail with the empty list of boxes if len(pred_classes) == 0: mask_rcnn_inference(mask_logits, instances) return instances else: # Put point predictions to the right places on the upsampled grid. R, C, H, W = mask_logits.shape point_indices = point_indices.unsqueeze(1).expand(-1, C, -1) mask_logits = ( mask_logits.reshape(R, C, H * W) .scatter_(2, point_indices, point_logits) .view(R, C, H, W) ) mask_rcnn_inference(mask_logits, instances) return instances
def __call__(self, pred_mask_logits, pred_instances): """equivalent to mask_head.mask_rcnn_inference""" if all(isinstance(x, InstancesList) for x in pred_instances): assert len(pred_instances) == 1 mask_probs_pred = pred_mask_logits.sigmoid() mask_probs_pred = alias(mask_probs_pred, "mask_fcn_probs") pred_instances[0].pred_masks = mask_probs_pred else: mask_rcnn_inference(pred_mask_logits, pred_instances)
def _forward_mask(self, features, instances): """ Forward logic of the mask prediction branch. Args: features (dict[str, Tensor]): #level input features for mask prediction instances (list[Instances]): the per-image instances to train/predict masks. In training, they can be the proposals. In inference, they can be the predicted boxes. Returns: In training, a dict of losses. In inference, update `instances` with new fields "pred_masks" and return it. """ if not self.mask_on: return {} if self.training else instances if self.training: instances, _ = select_foreground_proposals(instances, self.num_classes) if self.mask_pooler is not None: features = [features[f] for f in self.mask_in_features] boxes = [ x.proposal_boxes if self.training else x.pred_boxes for x in instances ] features = self.mask_pooler(features, boxes) else: features = dict([(f, features[f]) for f in self.mask_in_features]) mask_logits = self._forward_mask_rcnn(features, instances) if self.training: return self._forward_parallel_border(mask_logits, features, self.points_num, instances) else: # add "od_activated_map" attribute for instances, which is used for visualization od_activated_map, border_mask_logits = self._forward_parallel_border( mask_logits, features, self.points_num, instances) instance_num = 0 for instance in instances: instance.chosenP = od_activated_map[instance_num:instance_num + len(instance), :, :] instance_num += len(instance) mask_rcnn_inference(border_mask_logits, instances) return instances
def loss_or_inference(self, mask_logits, proposals): """ Gets predictions of the mask head and calculates the loss or performs inference depending on whether it is training or inference time. """ if self.training: mask_losses = {} mask_losses.update({ f'mask_loss': mask_loss_ps(mask_logits, proposals, self.vis_period, self.ps_mode) }) return mask_losses else: mask_rcnn_inference(mask_logits, proposals) return proposals
def forward(self, x, instances: List[Instances]): self.input_dict = {} self.input_dict['net_input'] = x self.input_dict['crop_gt'] = instances n_batch = x.shape[0] base_point = self.matrix.init_point base_normalized_point_adjacent = self.matrix.init_normalized_point_adjacent base_point_mask = self.matrix.init_point_mask base_triangle2point = self.matrix.init_triangle2point base_area_mask = self.matrix.init_area_mask base_triangle_mask = self.matrix.init_triangle_mask self.input_dict['base_point'] = base_point.expand(n_batch, -1, -1) self.input_dict['base_normalized_point_adjacent'] = base_normalized_point_adjacent.expand( n_batch, -1, -1 ) self.input_dict['base_point_mask'] = base_point_mask.expand(n_batch, -1, -1) self.input_dict['base_triangle2point'] = base_triangle2point.expand(n_batch, -1, -1) self.input_dict['base_area_mask'] = base_area_mask.expand(n_batch, -1) self.input_dict['base_triangle_mask'] = base_triangle_mask.expand(n_batch, -1) self.input_dict['grid_size'] = np.max(self.grid_size) output = self.model(**self.input_dict) output['pred_points'] gcn_pred_points = output['gcn_pred_points'] mask_features = gcn_pred_points.reshape( -1, self.out_dim, self.grid_size[0], self.grid_size[1] ) mask_features = F.relu(self.mask_deconv(mask_features)) output['mask_logits'] = self.mask_predictor(mask_features) if self.training: loss_mask, loss_defgrid = self.mask_head_loss( output, instances, self.to_three_channel(x) ) return {'loss_mask': loss_mask, 'loss_defgrid': loss_defgrid} else: mask_rcnn_inference(output['mask_logits'], instances) return instances
def forward(self, x, instances: List[Instances]) -> Any: """ Args: x: input region feature(s) provided by :class:`ROIHeads`. instances (list[Instances]): contains the boxes & labels corresponding to the input features. Exact format is up to its caller to decide. Typically, this is the foreground instances in training, with "proposal_boxes" field and other gt annotations. In inference, it contains boxes that are already predicted. Returns: A dict of losses in training. The predicted "instances" in inference. """ x = self.layers(x) if self.training: N, C, H, W = x.shape assert H == W proposal_boxes = [x.proposal_boxes for x in instances] assert N == np.sum(len(x) for x in proposal_boxes) if N == 0: return {"loss_mask": x.sum() * 0} # Training with point supervision # Sanity check: annotation should not contain gt_masks assert not instances[0].has("gt_masks") point_coords, point_labels = get_point_coords_from_point_annotation( instances) mask_logits = point_sample( x, point_coords, align_corners=False, ) return { "loss_mask": roi_mask_point_loss(mask_logits, instances, point_labels) } else: mask_rcnn_inference(x, instances) return instances