def test_to(self):
x = Boxes(torch.rand(3, 4))
self.assertEqual(x.to(device="cpu").tensor.device.type, "cpu")
def func(x):
boxes = Boxes(x)
test = boxes.to(torch.device("cpu")).tensor
return boxes.area(), test
def process(self, input, output):
previous_len = len(self._partial_results)
for instance, output in zip(input, output):
input_image_id = instance['image_id']
instance_gt_annots = self._coco_api.loadAnns(
self._coco_api.getAnnIds(imgIds=input_image_id))
im_name = os.path.basename(instance['file_name'])
fields = output["instances"].get_fields()
pred_boxes = fields['pred_boxes'] # xyxy
scores = fields['scores'].cpu().numpy()
pred_class = fields['pred_classes']
if instance_gt_annots:
# GT but not preds --> FN
if len(pred_boxes) == 0:
for annot_dict in instance_gt_annots:
row = [im_name, "FN", "FN", "non-eval", -1, "NA"]
self._partial_results += [row]
# GT and preds --> TP or FP
else:
det_out = "TP"
from detectron2.structures import Boxes, pairwise_iou, BoxMode
gt_boxes = torch.tensor([
annot_dict['bbox'] for annot_dict in instance_gt_annots
])
gt_boxes = BoxMode.convert(gt_boxes, BoxMode.XYWH_ABS,
BoxMode.XYXY_ABS)
gt_boxes = Boxes(gt_boxes.to(pred_boxes.device))
ious = pairwise_iou(gt_boxes, pred_boxes)
paired_preds = []
for gt_idx, matches in enumerate(ious):
if matches.sum() == 0:
row = [im_name, "FN", "FN", "non-eval", -1, "NA"]
self._partial_results += [row]
else:
if self.eval_mode == "iou":
pred_idx = matches.argmax()
if pred_idx not in paired_preds:
paired_preds.append(pred_idx)
class_out = self._is_polyp_classified(
pred_class[pred_idx],
instance_gt_annots[gt_idx]
['category_id'])
row = [
im_name, det_out, "TP", class_out,
scores[pred_idx], pred_boxes[pred_idx]
]
self._partial_results += [row]
else:
row = [
im_name, det_out, "FP", "non-eval",
scores[pred_idx], pred_boxes[pred_idx]
]
self._partial_results += [row]
else:
for posible_match in matches.nonzero():
gt_box = gt_boxes.tensor[gt_idx]
gt_x1, gt_y1, gt_x2, gt_y2 = gt_box
pred_box = pred_boxes.tensor[posible_match]
pred_x1, pred_y1, pred_x2, pred_y2 = pred_box.squeeze(
)
if self.eval_mode == 'old':
pred_cx, pred_cy = (
pred_x1 +
(pred_x2 - pred_x1) / 2), (
pred_y1 +
(pred_y2 - pred_y1) / 2)
eval_condition = (
gt_x1 < pred_cx < gt_x2) and (
gt_y1 < pred_cy < gt_y2)
else:
gt_cx, gt_cy = (
gt_x1 + (gt_x2 - gt_x1) / 2), (
gt_y1 + (gt_y2 - gt_y1) / 2)
eval_condition = (
pred_x1 < gt_cx < pred_x2) and (
pred_y1 < gt_cy < pred_y2)
if eval_condition:
if posible_match not in paired_preds:
paired_preds.append(posible_match)
class_out = self._is_polyp_classified(
pred_class[posible_match],
instance_gt_annots[gt_idx]
['category_id'])
row = [
im_name, det_out, "TP",
class_out,
scores[posible_match],
pred_boxes[posible_match]
]
self._partial_results += [row]
else:
row = [
im_name, det_out, "FP",
"non-eval",
scores[posible_match],
pred_boxes[posible_match]
]
self._partial_results += [row]
# for pred_box, pred_score, pred_classif in zip(pred_boxes, scores, pred_class):
# pred_x1, pred_y1, pred_x2, pred_y2 = pred_box
# if instance_gt_annots:
# for annot_dict in instance_gt_annots:
# gt_bbox = annot_dict['bbox'] # xywh
# gt_bbox[2] += gt_bbox[0]
# gt_bbox[3] += gt_bbox[1] # xyxy
#
# gt_x1, gt_y1, gt_x2, gt_y2 = gt_bbox
#
# eval_condition = self._is_localized(gt_bbox, gt_x1, gt_x2, gt_y1, gt_y2, pred_box,
# pred_x1, pred_x2, pred_y1, pred_y2)
#
# if eval_condition:
# class_out = self._is_polyp_classified(pred_classif, annot_dict['category_id'])
#
# row = [im_name, det_out, "TP", class_out, pred_score, pred_box]
# self._partial_results += [row]
# instance_gt_annots.remove(annot_dict)
# break
#
# else:
# row = [im_name, "FP", "FP", "non-eval", pred_score, pred_box]
# self._partial_results += [row]
else:
# No GT but Preds --> FP
if len(pred_boxes) > 0:
for pred_box, pred_score, pred_classif in zip(
pred_boxes, scores, pred_class):
row = [
im_name, "FP", "FP", "non-eval", pred_score,
pred_box
]
self._partial_results += [row]
# No GT and no Preds --> TN
else:
row = [im_name, "TN", "TN", "non-eval", -1, "NA"]
self._partial_results += [row]
def forward(self, batched_inputs):
"""
Args:
batched_inputs: a list, batched outputs of :class:`DatasetMapper` .
Each item in the list contains the inputs for one image.
For now, each item in the list is a dict that contains:
* image: Tensor, image in (C, H, W) format.
* instances (optional): groundtruth :class:`Instances`
* proposals (optional): :class:`Instances`, precomputed proposals.
Other information that's included in the original dicts, such as:
* "height", "width" (int): the output resolution of the model, used in inference.
See :meth:`postprocess` for details.
Returns:
list[dict]:
Each dict is the output for one input image.
The dict contains one key "instances" whose value is a :class:`Instances`.
The :class:`Instances` object has the following keys:
"pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints"
"""
if not self.training:
self.init_model()
return self.inference(batched_inputs)
images, support_images = self.preprocess_image(batched_inputs)
if "instances" in batched_inputs[0]:
for x in batched_inputs:
x['instances'].set(
'gt_classes',
torch.full_like(x['instances'].get('gt_classes'), 0))
gt_instances = [
x["instances"].to(self.device) for x in batched_inputs
]
else:
gt_instances = None
features = self.backbone(images.tensor)
# support branches
support_bboxes_ls = []
for item in batched_inputs:
bboxes = item['support_bboxes']
for box in bboxes:
box = Boxes(box[np.newaxis, :])
support_bboxes_ls.append(box.to(self.device))
B, N, C, H, W = support_images.tensor.shape
assert N == self.support_way * self.support_shot
support_images = support_images.tensor.reshape(B * N, C, H, W)
support_features = self.backbone(support_images)
# support feature roi pooling
feature_pooled = self.roi_heads.roi_pooling(support_features,
support_bboxes_ls)
support_box_features = self.roi_heads._shared_roi_transform(
[support_features[f] for f in self.in_features], support_bboxes_ls)
assert self.support_way == 2 # now only 2 way support
detector_loss_cls = []
detector_loss_box_reg = []
rpn_loss_rpn_cls = []
rpn_loss_rpn_loc = []
for i in range(B): # batch
# query
query_gt_instances = [gt_instances[i]] # one query gt instances
query_images = ImageList.from_tensors([images[i]
]) # one query image
query_feature_res4 = features['res4'][i].unsqueeze(
0) # one query feature for attention rpn
query_features = {
'res4': query_feature_res4
} # one query feature for rcnn
# positive support branch ##################################
pos_begin = i * self.support_shot * self.support_way
pos_end = pos_begin + self.support_shot
pos_support_features = feature_pooled[pos_begin:pos_end].mean(
0, True
) # pos support features from res4, average all supports, for rcnn
pos_support_features_pool = pos_support_features.mean(
dim=[2, 3], keepdim=True
) # average pooling support feature for attention rpn
pos_correlation = F.conv2d(query_feature_res4,
pos_support_features_pool.permute(
1, 0, 2, 3),
groups=1024) # attention map
pos_features = {
'res4': pos_correlation
} # attention map for attention rpn
pos_support_box_features = support_box_features[
pos_begin:pos_end].mean(0, True)
pos_proposals, pos_anchors, pos_pred_objectness_logits, pos_gt_labels, pos_pred_anchor_deltas, pos_gt_boxes = self.proposal_generator(
query_images, pos_features,
query_gt_instances) # attention rpn
pos_pred_class_logits, pos_pred_proposal_deltas, pos_detector_proposals = self.roi_heads(
query_images, query_features, pos_support_box_features,
pos_proposals, query_gt_instances) # pos rcnn
# negative support branch ##################################
neg_begin = pos_end
neg_end = neg_begin + self.support_shot
neg_support_features = feature_pooled[neg_begin:neg_end].mean(
0, True)
neg_support_features_pool = neg_support_features.mean(dim=[2, 3],
keepdim=True)
neg_correlation = F.conv2d(query_feature_res4,
neg_support_features_pool.permute(
1, 0, 2, 3),
groups=1024)
neg_features = {'res4': neg_correlation}
neg_support_box_features = support_box_features[
neg_begin:neg_end].mean(0, True)
neg_proposals, neg_anchors, neg_pred_objectness_logits, neg_gt_labels, neg_pred_anchor_deltas, neg_gt_boxes = self.proposal_generator(
query_images, neg_features, query_gt_instances)
neg_pred_class_logits, neg_pred_proposal_deltas, neg_detector_proposals = self.roi_heads(
query_images, query_features, neg_support_box_features,
neg_proposals, query_gt_instances)
# rpn loss
outputs_images = ImageList.from_tensors([images[i], images[i]])
outputs_pred_objectness_logits = [
torch.cat(pos_pred_objectness_logits +
neg_pred_objectness_logits,
dim=0)
]
outputs_pred_anchor_deltas = [
torch.cat(pos_pred_anchor_deltas + neg_pred_anchor_deltas,
dim=0)
]
outputs_anchors = pos_anchors # + neg_anchors
# convert 1 in neg_gt_labels to 0
for item in neg_gt_labels:
item[item == 1] = 0
outputs_gt_boxes = pos_gt_boxes + neg_gt_boxes #[None]
outputs_gt_labels = pos_gt_labels + neg_gt_labels
if self.training:
proposal_losses = self.proposal_generator.losses(
outputs_anchors, outputs_pred_objectness_logits,
outputs_gt_labels, outputs_pred_anchor_deltas,
outputs_gt_boxes)
proposal_losses = {
k: v * self.proposal_generator.loss_weight
for k, v in proposal_losses.items()
}
else:
proposal_losses = {}
# detector loss
detector_pred_class_logits = torch.cat(
[pos_pred_class_logits, neg_pred_class_logits], dim=0)
detector_pred_proposal_deltas = torch.cat(
[pos_pred_proposal_deltas, neg_pred_proposal_deltas], dim=0)
for item in neg_detector_proposals:
item.gt_classes = torch.full_like(item.gt_classes, 1)
#detector_proposals = pos_detector_proposals + neg_detector_proposals
detector_proposals = [
Instances.cat(pos_detector_proposals + neg_detector_proposals)
]
if self.training:
predictions = detector_pred_class_logits, detector_pred_proposal_deltas
detector_losses = self.roi_heads.box_predictor.losses(
predictions, detector_proposals)
rpn_loss_rpn_cls.append(proposal_losses['loss_rpn_cls'])
rpn_loss_rpn_loc.append(proposal_losses['loss_rpn_loc'])
detector_loss_cls.append(detector_losses['loss_cls'])
detector_loss_box_reg.append(detector_losses['loss_box_reg'])
proposal_losses = {}
detector_losses = {}
proposal_losses['loss_rpn_cls'] = torch.stack(rpn_loss_rpn_cls).mean()
proposal_losses['loss_rpn_loc'] = torch.stack(rpn_loss_rpn_loc).mean()
detector_losses['loss_cls'] = torch.stack(detector_loss_cls).mean()
detector_losses['loss_box_reg'] = torch.stack(
detector_loss_box_reg).mean()
losses = {}
losses.update(detector_losses)
losses.update(proposal_losses)
return losses
def forward(self, batched_inputs):
# También 1 vez por imagen
"""
Args:
batched_inputs: a list, batched outputs of :class:`DatasetMapper` .
Each item in the list contains the inputs for one image.
For now, each item in the list is a dict that contains:
* image: Tensor, image in (C, H, W) format.
* instances (optional): groundtruth :class:`Instances`
* proposals (optional): :class:`Instances`, precomputed proposals.
Other information that's included in the original dicts, such as:
* "height", "width" (int): the output resolution of the model, used in inference.
See :meth:`postprocess` for details.
Returns:
list[dict]:
Each dict is the output for one input image.
The dict contains one key "instances" whose value is a :class:`Instances`.
The :class:`Instances` object has the following keys:
"pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints"
"""
if not self.training:
### Estimar nº clases
support_file_name = './support_dir/support_feature.pkl'
if os.path.exists(support_file_name) and self.n_clases == 1:
device = torch.cuda.current_device()
avaliable = torch.cuda.get_device_properties(
device).total_memory - torch.cuda.memory_reserved(device)
#print("Memoria disponieble MiB: ", avaliable/(1024*1024))
with open(support_file_name, "rb") as hFile:
aux = pickle.load(hFile, encoding="latin1")
size = aux['res5_avg'][0].element_size(
) * aux['res5_avg'][0].nelement()
size += aux['res4_avg'][0].element_size(
) * aux['res4_avg'][0].nelement()
#print("Memoria ocupada por soporte: ", size)
#print("Numero de clases :", math.floor(avaliable/size))
self.n_clases = math.floor(avaliable / (size * 1000))
print("Classes number: ", self.n_clases)
### Fin estimacion
#Cambiar n_clases a valor estático si se quiere.
#n_clases = self.n_clases
n_clases = 101
# Obtener lista de id_clases: [1,2,3....]
metadata = MetadataCatalog.get('fsod_eval')
class_list = list(
metadata.thing_dataset_id_to_contiguous_id.values())
# En cada iteración tomamos n_clases de class_list e iniciamos el modelo con ellas
# Ejemplo con class_list=[1,2,3,4,5] y n_clases=2
# iter1: [1,2]; iter2: [3,4]; iter3: [5]
aux = []
for i in range(math.ceil(len(class_list) / n_clases)):
self.init_model(class_list[i * n_clases:i * n_clases +
n_clases])
aux.append(self.inference(batched_inputs)[0]["instances"])
# aux es una lista de predicciones [pred_n_primeras_clases, pred_siguientes, ....]
# necesitamos unificarlas todas en 1 solo elemento
# -> empleando detectron2.structures.instances.Instances.cat.
_predictions = {"instances": Instances.cat(aux)}
return [_predictions]
images, support_images = self.preprocess_image(batched_inputs)
if "instances" in batched_inputs[0]:
for x in batched_inputs:
x['instances'].set(
'gt_classes',
torch.full_like(x['instances'].get('gt_classes'), 0))
gt_instances = [
x["instances"].to(self.device) for x in batched_inputs
]
else:
gt_instances = None
features = self.backbone(images.tensor)
# support branches
support_bboxes_ls = []
for item in batched_inputs:
bboxes = item['support_bboxes']
for box in bboxes:
box = Boxes(box[np.newaxis, :])
support_bboxes_ls.append(box.to(self.device))
B, N, C, H, W = support_images.tensor.shape
assert N == self.support_way * self.support_shot
support_images = support_images.tensor.reshape(B * N, C, H, W)
support_features = self.backbone(support_images)
# support feature roi pooling
feature_pooled = self.roi_heads.roi_pooling(support_features,
support_bboxes_ls)
support_box_features = self.roi_heads._shared_roi_transform(
[support_features[f] for f in self.in_features], support_bboxes_ls)
#assert self.support_way == 2 # now only 2 way support
detector_loss_cls = []
detector_loss_box_reg = []
rpn_loss_rpn_cls = []
rpn_loss_rpn_loc = []
for i in range(B): # batch
# query
query_gt_instances = [gt_instances[i]] # one query gt instances
query_images = ImageList.from_tensors([images[i]
]) # one query image
query_feature_res4 = features['res4'][i].unsqueeze(
0) # one query feature for attention rpn
query_features = {
'res4': query_feature_res4
} # one query feature for rcnn
# positive support branch ##################################
pos_begin = i * self.support_shot * self.support_way
pos_end = pos_begin + self.support_shot
pos_support_features = feature_pooled[pos_begin:pos_end].mean(
0, True
) # pos support features from res4, average all supports, for rcnn
pos_support_features_pool = pos_support_features.mean(
dim=[2, 3], keepdim=True
) # average pooling support feature for attention rpn
pos_correlation = F.conv2d(query_feature_res4,
pos_support_features_pool.permute(
1, 0, 2, 3),
groups=1024) # attention map
pos_features = {
'res4': pos_correlation
} # attention map for attention rpn
pos_support_box_features = support_box_features[
pos_begin:pos_end].mean(0, True)
pos_proposals, pos_anchors, pos_pred_objectness_logits, pos_gt_labels, pos_pred_anchor_deltas, pos_gt_boxes = self.proposal_generator(
query_images, pos_features,
query_gt_instances) # attention rpn
pos_pred_class_logits, pos_pred_proposal_deltas, pos_detector_proposals = self.roi_heads(
query_images, query_features, pos_support_box_features,
pos_proposals, query_gt_instances) # pos rcnn
# negative support branch ##################################
neg_begin = pos_end
neg_end = neg_begin + self.support_shot
neg_support_features = feature_pooled[neg_begin:neg_end].mean(
0, True)
neg_support_features_pool = neg_support_features.mean(dim=[2, 3],
keepdim=True)
neg_correlation = F.conv2d(query_feature_res4,
neg_support_features_pool.permute(
1, 0, 2, 3),
groups=1024)
neg_features = {'res4': neg_correlation}
neg_support_box_features = support_box_features[
neg_begin:neg_end].mean(0, True)
neg_proposals, neg_anchors, neg_pred_objectness_logits, neg_gt_labels, neg_pred_anchor_deltas, neg_gt_boxes = self.proposal_generator(
query_images, neg_features, query_gt_instances)
neg_pred_class_logits, neg_pred_proposal_deltas, neg_detector_proposals = self.roi_heads(
query_images, query_features, neg_support_box_features,
neg_proposals, query_gt_instances)
# rpn loss
outputs_images = ImageList.from_tensors([images[i], images[i]])
outputs_pred_objectness_logits = [
torch.cat(pos_pred_objectness_logits +
neg_pred_objectness_logits,
dim=0)
]
outputs_pred_anchor_deltas = [
torch.cat(pos_pred_anchor_deltas + neg_pred_anchor_deltas,
dim=0)
]
outputs_anchors = pos_anchors # + neg_anchors
# convert 1 in neg_gt_labels to 0
for item in neg_gt_labels:
item[item == 1] = 0
outputs_gt_boxes = pos_gt_boxes + neg_gt_boxes #[None]
outputs_gt_labels = pos_gt_labels + neg_gt_labels
if self.training:
proposal_losses = self.proposal_generator.losses(
outputs_anchors, outputs_pred_objectness_logits,
outputs_gt_labels, outputs_pred_anchor_deltas,
outputs_gt_boxes)
proposal_losses = {
k: v * self.proposal_generator.loss_weight
for k, v in proposal_losses.items()
}
else:
proposal_losses = {}
# detector loss
detector_pred_class_logits = torch.cat(
[pos_pred_class_logits, neg_pred_class_logits], dim=0)
detector_pred_proposal_deltas = torch.cat(
[pos_pred_proposal_deltas, neg_pred_proposal_deltas], dim=0)
for item in neg_detector_proposals:
item.gt_classes = torch.full_like(item.gt_classes, 1)
#detector_proposals = pos_detector_proposals + neg_detector_proposals
detector_proposals = [
Instances.cat(pos_detector_proposals + neg_detector_proposals)
]
if self.training:
predictions = detector_pred_class_logits, detector_pred_proposal_deltas
detector_losses = self.roi_heads.box_predictor.losses(
predictions, detector_proposals)
rpn_loss_rpn_cls.append(proposal_losses['loss_rpn_cls'])
rpn_loss_rpn_loc.append(proposal_losses['loss_rpn_loc'])
detector_loss_cls.append(detector_losses['loss_cls'])
detector_loss_box_reg.append(detector_losses['loss_box_reg'])
proposal_losses = {}
detector_losses = {}
proposal_losses['loss_rpn_cls'] = torch.stack(rpn_loss_rpn_cls).mean()
proposal_losses['loss_rpn_loc'] = torch.stack(rpn_loss_rpn_loc).mean()
detector_losses['loss_cls'] = torch.stack(detector_loss_cls).mean()
detector_losses['loss_box_reg'] = torch.stack(
detector_loss_box_reg).mean()
losses = {}
losses.update(detector_losses)
losses.update(proposal_losses)
return losses
