def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (list[Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
# NOTE: instances = foreground proposals
mask_logits = self.mask_head(features)
return {"loss_mask": mask_rcnn_loss(mask_logits, instances)}
else:
mask_logits = self.mask_head(features)
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self, features, instances):
"""
Args:
features (dict[str, Tensor]): a dict of image-level features
instances (list[Instances]): proposals in training; detected
instances in inference
"""
if self.training:
proposal_boxes = [x.proposal_boxes for x in instances]
mask_coarse_logits = self.coarse_head(
self._roi_pooler(features, proposal_boxes))
losses = {
"loss_mask": mask_rcnn_loss(mask_coarse_logits, instances)
}
losses.update(
self._forward_mask_point(features, mask_coarse_logits,
instances))
return losses
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_coarse_logits = self.coarse_head(
self._roi_pooler(features, pred_boxes))
mask_logits = self._forward_mask_point(features,
mask_coarse_logits,
instances)
mask_rcnn_inference(mask_logits, instances)
return instances
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (list[Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
# The loss is only defined on positive proposals.
proposals, _ = select_foreground_proposals(self.train_part, instances, self.num_classes)
proposal_boxes = [x.proposal_boxes for x in proposals]
mask_features = self.mask_pooler(features, proposal_boxes)
#if len(mask_features) == 0:
# print('No X.............................')
# return 0.0, 0.0, 0.0
#else:
mask_logits = self.mask_head(mask_features)
return mask_rcnn_loss(mask_logits, proposals)
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_features = self.mask_pooler(features, pred_boxes)
mask_logits = self.mask_head(mask_features)
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self, mask_features, boundary_features,
instances): #（27,256,28,28）,（27,256,28,28）
x1 = self.inc(mask_features) #（27,256,28,28）
boundary_features = boundary_features + self.mask_to_boundary(
x1) #27,256,28,28
boundary_features = self.inc(boundary_features) #27,256,28,28
x2 = self.down1(x1) #（27，512，14，14)
# x2 = F.interpolate(self.boundary_to_mask1(boundary_features),scale_factor=1/2, mode='bilinear', align_corners=True) + x2 #插值合适还是下采样卷积合适
boundary_features_tmp = self.boundary_to_mask1(
boundary_features) #27,512,14,14
x2 = boundary_features_tmp + x2 #27,512,14,14
x3 = self.down2(x2) #(27,1024,7,7）
x3 = self.boundary_to_mask2(boundary_features_tmp) + x3 #27,1024,7,7
x = self.up1(x3, x2) #（27,512,14,14）
x = self.up2(x, x1) #（27,256,28,28）
mask_logits = self.outc(x) #（27,2,28,28）
boundary_logits = self.outc(boundary_features)
if self.training:
loss_mask, loss_boundary = boundary_preserving_mask_loss(
mask_logits, boundary_logits, instances)
return {"loss_mask": loss_mask, "loss_boundary": loss_boundary}
else:
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self, mask_features, context_feature,
instances: List[Instances]):
# self.mask_fcns
# self.downsample
# self.context_fcns
# self.context_to_mask
# self.mask_final_fusion
# self.mask_deconv
# self.mask_predictor
for layer in self.mask_fcns:
mask_features = layer(mask_features)
# downsample
context_feature = self.downsample(context_feature)
for layer in self.context_fcns:
context_feature = layer(context_feature)
# context to mask fusion
mask_features = self.context_to_mask(context_feature) + mask_features
mask_features = self.mask_final_fusion(mask_features)
# mask prediction
mask_features = F.relu(self.mask_deconv(mask_features))
mask_logits = self.mask_predictor(mask_features)
if self.training:
return {
"loss_mask":
mask_rcnn_loss(mask_logits, instances, self.vis_period) *
self.loss_weight
}
else:
mask_rcnn_inference(mask_logits, instances)
return instances
def forward(self,
x,
instances,
similarity=None,
base_classes=None,
novel_classes=None):
x, x_delta = self.layers(x)
if similarity is not None:
if x.numel() > 0:
similarity_mask = similarity['seg']
mask_base = x.index_select(1, index=base_classes)
mask_base_reshaped = mask_base.view(*mask_base.size()[:2], -1)
if len(similarity_mask.size()) > 2:
mask_combination = torch.bmm(similarity_mask,
mask_base_reshaped)
else:
mask_combination = torch.matmul(
mask_base_reshaped.transpose(1, 2),
similarity_mask.transpose(0, 1)).transpose(1, 2)
mask_novel = mask_combination.view(mask_base.size(0), -1,
*mask_base.size()[2:])
mask_final = torch.zeros_like(x)
mask_final = mask_final.index_copy(1, novel_classes,
mask_novel)
mask_final = mask_final.index_copy(1, base_classes, mask_base)
x = mask_final
x = x + x_delta
if self.training:
assert not torch.jit.is_scripting()
return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period)}
else:
mask_rcnn_inference(x, instances)
return instances
def forward(self, mask_features, boundary_features,
instances: List[Instances]):
for layer in self.mask_fcns:
mask_features = layer(mask_features)
# downsample
boundary_features = self.downsample(boundary_features)
# mask to boundary fusion
boundary_features = boundary_features + self.mask_to_boundary(
mask_features)
for layer in self.boundary_fcns:
boundary_features = layer(boundary_features)
# boundary to mask fusion
mask_features = self.boundary_to_mask(
boundary_features) + mask_features
mask_features = self.mask_final_fusion(mask_features)
# mask prediction
mask_features = F.interpolate(mask_features,
scale_factor=2,
mode='bilinear')
mask_logits = self.mask_predictor(mask_features)
# boundary prediction
boundary_features = F.interpolate(boundary_features,
scale_factor=2,
mode='bilinear')
boundary_logits = self.boundary_predictor(boundary_features)
if self.training:
loss_mask, loss_boundary = boundary_preserving_mask_loss(
mask_logits, boundary_logits, instances)
return {"loss_mask": loss_mask, "loss_boundary": loss_boundary}
else:
mask_rcnn_inference(mask_logits, instances)
return instances
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (dict[str, Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
proposals, _ = select_foreground_proposals(instances, self.num_classes)
proposal_boxes = [x.proposal_boxes for x in proposals]
mask_coarse_logits = self._forward_mask_coarse(features, proposal_boxes)
losses = {"loss_mask": mask_rcnn_loss(mask_coarse_logits, proposals)}
losses.update(self._forward_mask_point(features, mask_coarse_logits, proposals))
return losses
else:
pred_boxes = [x.pred_boxes for x in instances]
mask_coarse_logits = self._forward_mask_coarse(features, pred_boxes)
mask_logits = self._forward_mask_point(features, mask_coarse_logits, instances)
mask_rcnn_inference(mask_logits, instances)
return instances
def _subdivision_inference(self, features, mask_representations, instances):
assert not self.training
pred_boxes = [x.pred_boxes for x in instances]
pred_classes = cat([x.pred_classes for x in instances])
mask_logits = None
# +1 here to include an initial step to generate the coarsest mask
# prediction with init_resolution, when mask_logits is None.
# We compute initial mask by sampling on a regular grid. coarse_mask
# can be used as initial mask as well, but it's typically very low-res
# so it will be completely overwritten during subdivision anyway.
for _ in range(self.mask_point_subdivision_steps + 1):
if mask_logits is None:
point_coords = generate_regular_grid_point_coords(
pred_classes.size(0),
self.mask_point_subdivision_init_resolution,
pred_boxes[0].device,
)
else:
mask_logits = interpolate(
mask_logits, scale_factor=2, mode="bilinear", align_corners=False
)
uncertainty_map = calculate_uncertainty(mask_logits, pred_classes)
point_indices, point_coords = get_uncertain_point_coords_on_grid(
uncertainty_map, self.mask_point_subdivision_num_points
)
# Run the point head for every point in point_coords
fine_grained_features = self._point_pooler(features, pred_boxes, point_coords)
point_logits = self._get_point_logits(
fine_grained_features, point_coords, mask_representations
)
if mask_logits is None:
# Create initial mask_logits using point_logits on this regular grid
R, C, _ = point_logits.shape
mask_logits = point_logits.reshape(
R,
C,
self.mask_point_subdivision_init_resolution,
self.mask_point_subdivision_init_resolution,
)
# The subdivision code will fail with the empty list of boxes
if len(pred_classes) == 0:
mask_rcnn_inference(mask_logits, instances)
return instances
else:
# Put point predictions to the right places on the upsampled grid.
R, C, H, W = mask_logits.shape
point_indices = point_indices.unsqueeze(1).expand(-1, C, -1)
mask_logits = (
mask_logits.reshape(R, C, H * W)
.scatter_(2, point_indices, point_logits)
.view(R, C, H, W)
)
mask_rcnn_inference(mask_logits, instances)
return instances
def __call__(self, pred_mask_logits, pred_instances):
"""equivalent to mask_head.mask_rcnn_inference"""
if all(isinstance(x, InstancesList) for x in pred_instances):
assert len(pred_instances) == 1
mask_probs_pred = pred_mask_logits.sigmoid()
mask_probs_pred = alias(mask_probs_pred, "mask_fcn_probs")
pred_instances[0].pred_masks = mask_probs_pred
else:
mask_rcnn_inference(pred_mask_logits, pred_instances)
def _forward_mask(self, features, instances):
"""
Forward logic of the mask prediction branch.
Args:
features (dict[str, Tensor]): #level input features for mask prediction
instances (list[Instances]): the per-image instances to train/predict masks.
In training, they can be the proposals.
In inference, they can be the predicted boxes.
Returns:
In training, a dict of losses.
In inference, update `instances` with new fields "pred_masks" and return it.
"""
if not self.mask_on:
return {} if self.training else instances
if self.training:
instances, _ = select_foreground_proposals(instances,
self.num_classes)
if self.mask_pooler is not None:
features = [features[f] for f in self.mask_in_features]
boxes = [
x.proposal_boxes if self.training else x.pred_boxes
for x in instances
]
features = self.mask_pooler(features, boxes)
else:
features = dict([(f, features[f]) for f in self.mask_in_features])
mask_logits = self._forward_mask_rcnn(features, instances)
if self.training:
return self._forward_parallel_border(mask_logits, features,
self.points_num, instances)
else:
# add "od_activated_map" attribute for instances, which is used for visualization
od_activated_map, border_mask_logits = self._forward_parallel_border(
mask_logits, features, self.points_num, instances)
instance_num = 0
for instance in instances:
instance.chosenP = od_activated_map[instance_num:instance_num +
len(instance), :, :]
instance_num += len(instance)
mask_rcnn_inference(border_mask_logits, instances)
return instances
def loss_or_inference(self, mask_logits, proposals):
"""
Gets predictions of the mask head and calculates the loss or performs inference depending
on whether it is training or inference time.
"""
if self.training:
mask_losses = {}
mask_losses.update({
f'mask_loss':
mask_loss_ps(mask_logits, proposals, self.vis_period,
self.ps_mode)
})
return mask_losses
else:
mask_rcnn_inference(mask_logits, proposals)
return proposals
def forward(self, x, instances: List[Instances]):
self.input_dict = {}
self.input_dict['net_input'] = x
self.input_dict['crop_gt'] = instances
n_batch = x.shape[0]
base_point = self.matrix.init_point
base_normalized_point_adjacent = self.matrix.init_normalized_point_adjacent
base_point_mask = self.matrix.init_point_mask
base_triangle2point = self.matrix.init_triangle2point
base_area_mask = self.matrix.init_area_mask
base_triangle_mask = self.matrix.init_triangle_mask
self.input_dict['base_point'] = base_point.expand(n_batch, -1, -1)
self.input_dict['base_normalized_point_adjacent'] = base_normalized_point_adjacent.expand(
n_batch, -1, -1
)
self.input_dict['base_point_mask'] = base_point_mask.expand(n_batch, -1, -1)
self.input_dict['base_triangle2point'] = base_triangle2point.expand(n_batch, -1, -1)
self.input_dict['base_area_mask'] = base_area_mask.expand(n_batch, -1)
self.input_dict['base_triangle_mask'] = base_triangle_mask.expand(n_batch, -1)
self.input_dict['grid_size'] = np.max(self.grid_size)
output = self.model(**self.input_dict)
output['pred_points']
gcn_pred_points = output['gcn_pred_points']
mask_features = gcn_pred_points.reshape(
-1, self.out_dim, self.grid_size[0], self.grid_size[1]
)
mask_features = F.relu(self.mask_deconv(mask_features))
output['mask_logits'] = self.mask_predictor(mask_features)
if self.training:
loss_mask, loss_defgrid = self.mask_head_loss(
output, instances, self.to_three_channel(x)
)
return {'loss_mask': loss_mask, 'loss_defgrid': loss_defgrid}
else:
mask_rcnn_inference(output['mask_logits'], instances)
return instances
def forward(self, x, instances: List[Instances]) -> Any:
"""
Args:
x: input region feature(s) provided by :class:`ROIHeads`.
instances (list[Instances]): contains the boxes & labels corresponding
to the input features.
Exact format is up to its caller to decide.
Typically, this is the foreground instances in training, with
"proposal_boxes" field and other gt annotations.
In inference, it contains boxes that are already predicted.
Returns:
A dict of losses in training. The predicted "instances" in inference.
"""
x = self.layers(x)
if self.training:
N, C, H, W = x.shape
assert H == W
proposal_boxes = [x.proposal_boxes for x in instances]
assert N == np.sum(len(x) for x in proposal_boxes)
if N == 0:
return {"loss_mask": x.sum() * 0}
# Training with point supervision
# Sanity check: annotation should not contain gt_masks
assert not instances[0].has("gt_masks")
point_coords, point_labels = get_point_coords_from_point_annotation(
instances)
mask_logits = point_sample(
x,
point_coords,
align_corners=False,
)
return {
"loss_mask":
roi_mask_point_loss(mask_logits, instances, point_labels)
}
else:
mask_rcnn_inference(x, instances)
return instances
