def label_and_sample_anchors(
self, anchors: List[RotatedBoxes], gt_instances: List[Instances]
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""
Args:
anchors (list[RotatedBoxes]): anchors for each feature map.
gt_instances: the ground-truth instances for each image.
Returns:
list[Tensor]:
List of #img tensors. i-th element is a vector of labels whose length is
the total number of anchors across feature maps. Label values are in {-1, 0, 1},
with meanings: -1 = ignore; 0 = negative class; 1 = positive class.
list[Tensor]:
i-th element is a Nx5 tensor, where N is the total number of anchors across
feature maps. The values are the matched gt boxes for each anchor.
Values are undefined for those anchors not labeled as 1.
"""
anchors = RotatedBoxes.cat(anchors)
gt_boxes = [x.gt_boxes for x in gt_instances]
del gt_instances
gt_labels = []
matched_gt_boxes = []
for gt_boxes_i in gt_boxes:
"""
gt_boxes_i: ground-truth boxes for i-th image
"""
match_quality_matrix = retry_if_cuda_oom(pairwise_iou_rotated)(
gt_boxes_i, anchors)
matched_idxs, gt_labels_i = retry_if_cuda_oom(
self.anchor_matcher)(match_quality_matrix)
# Matching is memory-expensive and may result in CPU tensors. But the result is small
gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device)
# A vector of labels (-1, 0, 1) for each anchor
gt_labels_i = self._subsample_labels(gt_labels_i)
if len(gt_boxes_i) == 0:
# These values won't be used anyway since the anchor is labeled as background
matched_gt_boxes_i = torch.zeros_like(anchors.tensor)
else:
# TODO wasted indexing computation for ignored boxes
matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor
gt_labels.append(gt_labels_i) # N,AHW
matched_gt_boxes.append(matched_gt_boxes_i)
return gt_labels, matched_gt_boxes
def _get_ground_truth(self):
"""
Returns:
gt_objectness_logits: list of N tensors. Tensor i is a vector whose length is the
total number of anchors in image i (i.e., len(anchors[i])). Label values are
in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative class; 1 = positive class.
gt_anchor_deltas: list of N tensors. Tensor i has shape (len(anchors[i]), 5).
"""
gt_objectness_logits = []
gt_anchor_deltas = []
# Concatenate anchors from all feature maps into a single RotatedBoxes per image
anchors = [RotatedBoxes.cat(anchors_i) for anchors_i in self.anchors]
for image_size_i, anchors_i, gt_boxes_i in zip(self.image_sizes,
anchors, self.gt_boxes):
"""
image_size_i: (h, w) for the i-th image
anchors_i: anchors for i-th image
gt_boxes_i: ground-truth boxes for i-th image
"""
# DEBUG
#assert torch.all(gt_boxes_i.tensor[:,2] > 1e-5)
#assert torch.all(gt_boxes_i.tensor[:,3] > 1e-5)
#assert torch.all(anchors_i.tensor[:,2] > 1e-5)
#assert torch.all(anchors_i.tensor[:,3] > 1e-5)
match_quality_matrix = pairwise_iou_rotated(gt_boxes_i, anchors_i)
matched_idxs, gt_objectness_logits_i = self.anchor_matcher(
match_quality_matrix)
if self.boundary_threshold >= 0:
# Discard anchors that go out of the boundaries of the image
# NOTE: This is legacy functionality that is turned off by default in Detectron2
anchors_inside_image = anchors_i.inside_box(
image_size_i, self.boundary_threshold)
gt_objectness_logits_i[~anchors_inside_image] = -1
if len(gt_boxes_i) == 0:
# These values won't be used anyway since the anchor is labeled as background
gt_anchor_deltas_i = torch.zeros_like(anchors_i.tensor)
else:
# TODO wasted computation for ignored boxes
matched_gt_boxes = gt_boxes_i[matched_idxs]
gt_anchor_deltas_i = self.box2box_transform.get_deltas(
anchors_i.tensor, matched_gt_boxes.tensor)
gt_objectness_logits.append(gt_objectness_logits_i)
gt_anchor_deltas.append(gt_anchor_deltas_i)
return gt_objectness_logits, gt_anchor_deltas
def label_anchors(self, anchors, gt_instances):
"""
Args:
anchors (list[Boxes]): A list of #feature level Boxes.
The Boxes contains anchors of this image on the specific feature level.
gt_instances (list[Instances]): a list of N `Instances`s. The i-th
`Instances` contains the ground-truth per-instance annotations
for the i-th input image.
Returns:
list[Tensor]:
List of #img tensors. i-th element is a vector of labels whose length is
the total number of anchors across all feature maps (sum(Hi * Wi * A)).
Label values are in {-1, 0, ..., K}, with -1 means ignore, and K means background.
list[Tensor]:
i-th element is a Rx4 tensor, where R is the total number of anchors across
feature maps. The values are the matched gt boxes for each anchor.
Values are undefined for those anchors not labeled as foreground.
"""
anchors = RotatedBoxes.cat(anchors) # Rx4
gt_labels = []
matched_gt_boxes = []
for gt_per_image in gt_instances:
match_quality_matrix = pairwise_iou(gt_per_image.gt_boxes, anchors)
matched_idxs, anchor_labels = self.anchor_matcher(match_quality_matrix)
del match_quality_matrix
if len(gt_per_image) > 0:
matched_gt_boxes_i = gt_per_image.gt_boxes.tensor[matched_idxs]
gt_labels_i = gt_per_image.gt_classes[matched_idxs]
# Anchors with label 0 are treated as background.
gt_labels_i[anchor_labels == 0] = self.num_classes
# Anchors with label -1 are ignored.
gt_labels_i[anchor_labels == -1] = -1
else:
matched_gt_boxes_i = torch.zeros_like(anchors.tensor)
gt_labels_i = torch.zeros_like(matched_idxs) + self.num_classes
gt_labels.append(gt_labels_i)
matched_gt_boxes.append(matched_gt_boxes_i)
return gt_labels, matched_gt_boxes
def get_ground_truth(self, anchors, targets):
"""
Args:
anchors (list[list[Boxes]]): a list of N=#image elements. Each is a
list of #feature level Boxes. The Boxes contains anchors of
this image on the specific feature level.
targets (list[Instances]): a list of N `Instances`s. The i-th
`Instances` contains the ground-truth per-instance annotations
for the i-th input image. Specify `targets` during training only.
Returns:
gt_classes (Tensor):
An integer tensor of shape (N, R) storing ground-truth
labels for each anchor.
R is the total number of anchors, i.e. the sum of Hi x Wi x A for all levels.
Anchors with an IoU with some target higher than the foreground threshold
are assigned their corresponding label in the [0, K-1] range.
Anchors whose IoU are below the background threshold are assigned
the label "K". Anchors whose IoU are between the foreground and background
thresholds are assigned a label "-1", i.e. ignore.
gt_anchors_deltas (Tensor):
Shape (N, R, 4).
The last dimension represents ground-truth box2box transform
targets (dx, dy, dw, dh) that map each anchor to its matched ground-truth box.
The values in the tensor are meaningful only when the corresponding
anchor is labeled as foreground.
"""
gt_classes = []
gt_anchors_deltas = []
anchors = [RotatedBoxes.cat(anchors_i) for anchors_i in anchors]
# list[Tensor(R, 4)], one for each image
for anchors_per_image, targets_per_image in zip(anchors, targets):
match_quality_matrix = pairwise_iou_rotated(
targets_per_image.gt_boxes, anchors_per_image)
# adjust the scores of 'relation' and 'complexes' cases in the matrix
# gt_matched_idxs, anchor_labels = self.matcher(match_quality_matrix, targets_per_image, anchors_per_image)
gt_matched_idxs, anchor_labels = self.matcher(match_quality_matrix)
# ground truth box regression
matched_gt_boxes = targets_per_image[gt_matched_idxs].gt_boxes
gt_anchors_reg_deltas_i = self.box2box_transform.get_deltas(
anchors_per_image.tensor, matched_gt_boxes.tensor)
# ground truth classes
has_gt = len(targets_per_image) > 0
if has_gt:
gt_classes_i = targets_per_image.gt_classes[gt_matched_idxs]
# Anchors with label 0 are treated as background.
gt_classes_i[anchor_labels == 0] = self.num_classes
# Anchors with label -1 are ignored.
gt_classes_i[anchor_labels == -1] = -1
else:
gt_classes_i = torch.zeros_like(
gt_matched_idxs) + self.num_classes
gt_classes.append(gt_classes_i)
gt_anchors_deltas.append(gt_anchors_reg_deltas_i)
del anchors
return torch.stack(gt_classes), torch.stack(gt_anchors_deltas)