def test_to(self): x = Boxes(torch.rand(3, 4)) self.assertEqual(x.to(device="cpu").tensor.device.type, "cpu")
def func(x): boxes = Boxes(x) test = boxes.to(torch.device("cpu")).tensor return boxes.area(), test
def process(self, input, output): previous_len = len(self._partial_results) for instance, output in zip(input, output): input_image_id = instance['image_id'] instance_gt_annots = self._coco_api.loadAnns( self._coco_api.getAnnIds(imgIds=input_image_id)) im_name = os.path.basename(instance['file_name']) fields = output["instances"].get_fields() pred_boxes = fields['pred_boxes'] # xyxy scores = fields['scores'].cpu().numpy() pred_class = fields['pred_classes'] if instance_gt_annots: # GT but not preds --> FN if len(pred_boxes) == 0: for annot_dict in instance_gt_annots: row = [im_name, "FN", "FN", "non-eval", -1, "NA"] self._partial_results += [row] # GT and preds --> TP or FP else: det_out = "TP" from detectron2.structures import Boxes, pairwise_iou, BoxMode gt_boxes = torch.tensor([ annot_dict['bbox'] for annot_dict in instance_gt_annots ]) gt_boxes = BoxMode.convert(gt_boxes, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) gt_boxes = Boxes(gt_boxes.to(pred_boxes.device)) ious = pairwise_iou(gt_boxes, pred_boxes) paired_preds = [] for gt_idx, matches in enumerate(ious): if matches.sum() == 0: row = [im_name, "FN", "FN", "non-eval", -1, "NA"] self._partial_results += [row] else: if self.eval_mode == "iou": pred_idx = matches.argmax() if pred_idx not in paired_preds: paired_preds.append(pred_idx) class_out = self._is_polyp_classified( pred_class[pred_idx], instance_gt_annots[gt_idx] ['category_id']) row = [ im_name, det_out, "TP", class_out, scores[pred_idx], pred_boxes[pred_idx] ] self._partial_results += [row] else: row = [ im_name, det_out, "FP", "non-eval", scores[pred_idx], pred_boxes[pred_idx] ] self._partial_results += [row] else: for posible_match in matches.nonzero(): gt_box = gt_boxes.tensor[gt_idx] gt_x1, gt_y1, gt_x2, gt_y2 = gt_box pred_box = pred_boxes.tensor[posible_match] pred_x1, pred_y1, pred_x2, pred_y2 = pred_box.squeeze( ) if self.eval_mode == 'old': pred_cx, pred_cy = ( pred_x1 + (pred_x2 - pred_x1) / 2), ( pred_y1 + (pred_y2 - pred_y1) / 2) eval_condition = ( gt_x1 < pred_cx < gt_x2) and ( gt_y1 < pred_cy < gt_y2) else: gt_cx, gt_cy = ( gt_x1 + (gt_x2 - gt_x1) / 2), ( gt_y1 + (gt_y2 - gt_y1) / 2) eval_condition = ( pred_x1 < gt_cx < pred_x2) and ( pred_y1 < gt_cy < pred_y2) if eval_condition: if posible_match not in paired_preds: paired_preds.append(posible_match) class_out = self._is_polyp_classified( pred_class[posible_match], instance_gt_annots[gt_idx] ['category_id']) row = [ im_name, det_out, "TP", class_out, scores[posible_match], pred_boxes[posible_match] ] self._partial_results += [row] else: row = [ im_name, det_out, "FP", "non-eval", scores[posible_match], pred_boxes[posible_match] ] self._partial_results += [row] # for pred_box, pred_score, pred_classif in zip(pred_boxes, scores, pred_class): # pred_x1, pred_y1, pred_x2, pred_y2 = pred_box # if instance_gt_annots: # for annot_dict in instance_gt_annots: # gt_bbox = annot_dict['bbox'] # xywh # gt_bbox[2] += gt_bbox[0] # gt_bbox[3] += gt_bbox[1] # xyxy # # gt_x1, gt_y1, gt_x2, gt_y2 = gt_bbox # # eval_condition = self._is_localized(gt_bbox, gt_x1, gt_x2, gt_y1, gt_y2, pred_box, # pred_x1, pred_x2, pred_y1, pred_y2) # # if eval_condition: # class_out = self._is_polyp_classified(pred_classif, annot_dict['category_id']) # # row = [im_name, det_out, "TP", class_out, pred_score, pred_box] # self._partial_results += [row] # instance_gt_annots.remove(annot_dict) # break # # else: # row = [im_name, "FP", "FP", "non-eval", pred_score, pred_box] # self._partial_results += [row] else: # No GT but Preds --> FP if len(pred_boxes) > 0: for pred_box, pred_score, pred_classif in zip( pred_boxes, scores, pred_class): row = [ im_name, "FP", "FP", "non-eval", pred_score, pred_box ] self._partial_results += [row] # No GT and no Preds --> TN else: row = [im_name, "TN", "TN", "non-eval", -1, "NA"] self._partial_results += [row]
def forward(self, batched_inputs): """ Args: batched_inputs: a list, batched outputs of :class:`DatasetMapper` . Each item in the list contains the inputs for one image. For now, each item in the list is a dict that contains: * image: Tensor, image in (C, H, W) format. * instances (optional): groundtruth :class:`Instances` * proposals (optional): :class:`Instances`, precomputed proposals. Other information that's included in the original dicts, such as: * "height", "width" (int): the output resolution of the model, used in inference. See :meth:`postprocess` for details. Returns: list[dict]: Each dict is the output for one input image. The dict contains one key "instances" whose value is a :class:`Instances`. The :class:`Instances` object has the following keys: "pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints" """ if not self.training: self.init_model() return self.inference(batched_inputs) images, support_images = self.preprocess_image(batched_inputs) if "instances" in batched_inputs[0]: for x in batched_inputs: x['instances'].set( 'gt_classes', torch.full_like(x['instances'].get('gt_classes'), 0)) gt_instances = [ x["instances"].to(self.device) for x in batched_inputs ] else: gt_instances = None features = self.backbone(images.tensor) # support branches support_bboxes_ls = [] for item in batched_inputs: bboxes = item['support_bboxes'] for box in bboxes: box = Boxes(box[np.newaxis, :]) support_bboxes_ls.append(box.to(self.device)) B, N, C, H, W = support_images.tensor.shape assert N == self.support_way * self.support_shot support_images = support_images.tensor.reshape(B * N, C, H, W) support_features = self.backbone(support_images) # support feature roi pooling feature_pooled = self.roi_heads.roi_pooling(support_features, support_bboxes_ls) support_box_features = self.roi_heads._shared_roi_transform( [support_features[f] for f in self.in_features], support_bboxes_ls) assert self.support_way == 2 # now only 2 way support detector_loss_cls = [] detector_loss_box_reg = [] rpn_loss_rpn_cls = [] rpn_loss_rpn_loc = [] for i in range(B): # batch # query query_gt_instances = [gt_instances[i]] # one query gt instances query_images = ImageList.from_tensors([images[i] ]) # one query image query_feature_res4 = features['res4'][i].unsqueeze( 0) # one query feature for attention rpn query_features = { 'res4': query_feature_res4 } # one query feature for rcnn # positive support branch ################################## pos_begin = i * self.support_shot * self.support_way pos_end = pos_begin + self.support_shot pos_support_features = feature_pooled[pos_begin:pos_end].mean( 0, True ) # pos support features from res4, average all supports, for rcnn pos_support_features_pool = pos_support_features.mean( dim=[2, 3], keepdim=True ) # average pooling support feature for attention rpn pos_correlation = F.conv2d(query_feature_res4, pos_support_features_pool.permute( 1, 0, 2, 3), groups=1024) # attention map pos_features = { 'res4': pos_correlation } # attention map for attention rpn pos_support_box_features = support_box_features[ pos_begin:pos_end].mean(0, True) pos_proposals, pos_anchors, pos_pred_objectness_logits, pos_gt_labels, pos_pred_anchor_deltas, pos_gt_boxes = self.proposal_generator( query_images, pos_features, query_gt_instances) # attention rpn pos_pred_class_logits, pos_pred_proposal_deltas, pos_detector_proposals = self.roi_heads( query_images, query_features, pos_support_box_features, pos_proposals, query_gt_instances) # pos rcnn # negative support branch ################################## neg_begin = pos_end neg_end = neg_begin + self.support_shot neg_support_features = feature_pooled[neg_begin:neg_end].mean( 0, True) neg_support_features_pool = neg_support_features.mean(dim=[2, 3], keepdim=True) neg_correlation = F.conv2d(query_feature_res4, neg_support_features_pool.permute( 1, 0, 2, 3), groups=1024) neg_features = {'res4': neg_correlation} neg_support_box_features = support_box_features[ neg_begin:neg_end].mean(0, True) neg_proposals, neg_anchors, neg_pred_objectness_logits, neg_gt_labels, neg_pred_anchor_deltas, neg_gt_boxes = self.proposal_generator( query_images, neg_features, query_gt_instances) neg_pred_class_logits, neg_pred_proposal_deltas, neg_detector_proposals = self.roi_heads( query_images, query_features, neg_support_box_features, neg_proposals, query_gt_instances) # rpn loss outputs_images = ImageList.from_tensors([images[i], images[i]]) outputs_pred_objectness_logits = [ torch.cat(pos_pred_objectness_logits + neg_pred_objectness_logits, dim=0) ] outputs_pred_anchor_deltas = [ torch.cat(pos_pred_anchor_deltas + neg_pred_anchor_deltas, dim=0) ] outputs_anchors = pos_anchors # + neg_anchors # convert 1 in neg_gt_labels to 0 for item in neg_gt_labels: item[item == 1] = 0 outputs_gt_boxes = pos_gt_boxes + neg_gt_boxes #[None] outputs_gt_labels = pos_gt_labels + neg_gt_labels if self.training: proposal_losses = self.proposal_generator.losses( outputs_anchors, outputs_pred_objectness_logits, outputs_gt_labels, outputs_pred_anchor_deltas, outputs_gt_boxes) proposal_losses = { k: v * self.proposal_generator.loss_weight for k, v in proposal_losses.items() } else: proposal_losses = {} # detector loss detector_pred_class_logits = torch.cat( [pos_pred_class_logits, neg_pred_class_logits], dim=0) detector_pred_proposal_deltas = torch.cat( [pos_pred_proposal_deltas, neg_pred_proposal_deltas], dim=0) for item in neg_detector_proposals: item.gt_classes = torch.full_like(item.gt_classes, 1) #detector_proposals = pos_detector_proposals + neg_detector_proposals detector_proposals = [ Instances.cat(pos_detector_proposals + neg_detector_proposals) ] if self.training: predictions = detector_pred_class_logits, detector_pred_proposal_deltas detector_losses = self.roi_heads.box_predictor.losses( predictions, detector_proposals) rpn_loss_rpn_cls.append(proposal_losses['loss_rpn_cls']) rpn_loss_rpn_loc.append(proposal_losses['loss_rpn_loc']) detector_loss_cls.append(detector_losses['loss_cls']) detector_loss_box_reg.append(detector_losses['loss_box_reg']) proposal_losses = {} detector_losses = {} proposal_losses['loss_rpn_cls'] = torch.stack(rpn_loss_rpn_cls).mean() proposal_losses['loss_rpn_loc'] = torch.stack(rpn_loss_rpn_loc).mean() detector_losses['loss_cls'] = torch.stack(detector_loss_cls).mean() detector_losses['loss_box_reg'] = torch.stack( detector_loss_box_reg).mean() losses = {} losses.update(detector_losses) losses.update(proposal_losses) return losses
def forward(self, batched_inputs): # También 1 vez por imagen """ Args: batched_inputs: a list, batched outputs of :class:`DatasetMapper` . Each item in the list contains the inputs for one image. For now, each item in the list is a dict that contains: * image: Tensor, image in (C, H, W) format. * instances (optional): groundtruth :class:`Instances` * proposals (optional): :class:`Instances`, precomputed proposals. Other information that's included in the original dicts, such as: * "height", "width" (int): the output resolution of the model, used in inference. See :meth:`postprocess` for details. Returns: list[dict]: Each dict is the output for one input image. The dict contains one key "instances" whose value is a :class:`Instances`. The :class:`Instances` object has the following keys: "pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints" """ if not self.training: ### Estimar nº clases support_file_name = './support_dir/support_feature.pkl' if os.path.exists(support_file_name) and self.n_clases == 1: device = torch.cuda.current_device() avaliable = torch.cuda.get_device_properties( device).total_memory - torch.cuda.memory_reserved(device) #print("Memoria disponieble MiB: ", avaliable/(1024*1024)) with open(support_file_name, "rb") as hFile: aux = pickle.load(hFile, encoding="latin1") size = aux['res5_avg'][0].element_size( ) * aux['res5_avg'][0].nelement() size += aux['res4_avg'][0].element_size( ) * aux['res4_avg'][0].nelement() #print("Memoria ocupada por soporte: ", size) #print("Numero de clases :", math.floor(avaliable/size)) self.n_clases = math.floor(avaliable / (size * 1000)) print("Classes number: ", self.n_clases) ### Fin estimacion #Cambiar n_clases a valor estático si se quiere. #n_clases = self.n_clases n_clases = 101 # Obtener lista de id_clases: [1,2,3....] metadata = MetadataCatalog.get('fsod_eval') class_list = list( metadata.thing_dataset_id_to_contiguous_id.values()) # En cada iteración tomamos n_clases de class_list e iniciamos el modelo con ellas # Ejemplo con class_list=[1,2,3,4,5] y n_clases=2 # iter1: [1,2]; iter2: [3,4]; iter3: [5] aux = [] for i in range(math.ceil(len(class_list) / n_clases)): self.init_model(class_list[i * n_clases:i * n_clases + n_clases]) aux.append(self.inference(batched_inputs)[0]["instances"]) # aux es una lista de predicciones [pred_n_primeras_clases, pred_siguientes, ....] # necesitamos unificarlas todas en 1 solo elemento # -> empleando detectron2.structures.instances.Instances.cat. _predictions = {"instances": Instances.cat(aux)} return [_predictions] images, support_images = self.preprocess_image(batched_inputs) if "instances" in batched_inputs[0]: for x in batched_inputs: x['instances'].set( 'gt_classes', torch.full_like(x['instances'].get('gt_classes'), 0)) gt_instances = [ x["instances"].to(self.device) for x in batched_inputs ] else: gt_instances = None features = self.backbone(images.tensor) # support branches support_bboxes_ls = [] for item in batched_inputs: bboxes = item['support_bboxes'] for box in bboxes: box = Boxes(box[np.newaxis, :]) support_bboxes_ls.append(box.to(self.device)) B, N, C, H, W = support_images.tensor.shape assert N == self.support_way * self.support_shot support_images = support_images.tensor.reshape(B * N, C, H, W) support_features = self.backbone(support_images) # support feature roi pooling feature_pooled = self.roi_heads.roi_pooling(support_features, support_bboxes_ls) support_box_features = self.roi_heads._shared_roi_transform( [support_features[f] for f in self.in_features], support_bboxes_ls) #assert self.support_way == 2 # now only 2 way support detector_loss_cls = [] detector_loss_box_reg = [] rpn_loss_rpn_cls = [] rpn_loss_rpn_loc = [] for i in range(B): # batch # query query_gt_instances = [gt_instances[i]] # one query gt instances query_images = ImageList.from_tensors([images[i] ]) # one query image query_feature_res4 = features['res4'][i].unsqueeze( 0) # one query feature for attention rpn query_features = { 'res4': query_feature_res4 } # one query feature for rcnn # positive support branch ################################## pos_begin = i * self.support_shot * self.support_way pos_end = pos_begin + self.support_shot pos_support_features = feature_pooled[pos_begin:pos_end].mean( 0, True ) # pos support features from res4, average all supports, for rcnn pos_support_features_pool = pos_support_features.mean( dim=[2, 3], keepdim=True ) # average pooling support feature for attention rpn pos_correlation = F.conv2d(query_feature_res4, pos_support_features_pool.permute( 1, 0, 2, 3), groups=1024) # attention map pos_features = { 'res4': pos_correlation } # attention map for attention rpn pos_support_box_features = support_box_features[ pos_begin:pos_end].mean(0, True) pos_proposals, pos_anchors, pos_pred_objectness_logits, pos_gt_labels, pos_pred_anchor_deltas, pos_gt_boxes = self.proposal_generator( query_images, pos_features, query_gt_instances) # attention rpn pos_pred_class_logits, pos_pred_proposal_deltas, pos_detector_proposals = self.roi_heads( query_images, query_features, pos_support_box_features, pos_proposals, query_gt_instances) # pos rcnn # negative support branch ################################## neg_begin = pos_end neg_end = neg_begin + self.support_shot neg_support_features = feature_pooled[neg_begin:neg_end].mean( 0, True) neg_support_features_pool = neg_support_features.mean(dim=[2, 3], keepdim=True) neg_correlation = F.conv2d(query_feature_res4, neg_support_features_pool.permute( 1, 0, 2, 3), groups=1024) neg_features = {'res4': neg_correlation} neg_support_box_features = support_box_features[ neg_begin:neg_end].mean(0, True) neg_proposals, neg_anchors, neg_pred_objectness_logits, neg_gt_labels, neg_pred_anchor_deltas, neg_gt_boxes = self.proposal_generator( query_images, neg_features, query_gt_instances) neg_pred_class_logits, neg_pred_proposal_deltas, neg_detector_proposals = self.roi_heads( query_images, query_features, neg_support_box_features, neg_proposals, query_gt_instances) # rpn loss outputs_images = ImageList.from_tensors([images[i], images[i]]) outputs_pred_objectness_logits = [ torch.cat(pos_pred_objectness_logits + neg_pred_objectness_logits, dim=0) ] outputs_pred_anchor_deltas = [ torch.cat(pos_pred_anchor_deltas + neg_pred_anchor_deltas, dim=0) ] outputs_anchors = pos_anchors # + neg_anchors # convert 1 in neg_gt_labels to 0 for item in neg_gt_labels: item[item == 1] = 0 outputs_gt_boxes = pos_gt_boxes + neg_gt_boxes #[None] outputs_gt_labels = pos_gt_labels + neg_gt_labels if self.training: proposal_losses = self.proposal_generator.losses( outputs_anchors, outputs_pred_objectness_logits, outputs_gt_labels, outputs_pred_anchor_deltas, outputs_gt_boxes) proposal_losses = { k: v * self.proposal_generator.loss_weight for k, v in proposal_losses.items() } else: proposal_losses = {} # detector loss detector_pred_class_logits = torch.cat( [pos_pred_class_logits, neg_pred_class_logits], dim=0) detector_pred_proposal_deltas = torch.cat( [pos_pred_proposal_deltas, neg_pred_proposal_deltas], dim=0) for item in neg_detector_proposals: item.gt_classes = torch.full_like(item.gt_classes, 1) #detector_proposals = pos_detector_proposals + neg_detector_proposals detector_proposals = [ Instances.cat(pos_detector_proposals + neg_detector_proposals) ] if self.training: predictions = detector_pred_class_logits, detector_pred_proposal_deltas detector_losses = self.roi_heads.box_predictor.losses( predictions, detector_proposals) rpn_loss_rpn_cls.append(proposal_losses['loss_rpn_cls']) rpn_loss_rpn_loc.append(proposal_losses['loss_rpn_loc']) detector_loss_cls.append(detector_losses['loss_cls']) detector_loss_box_reg.append(detector_losses['loss_box_reg']) proposal_losses = {} detector_losses = {} proposal_losses['loss_rpn_cls'] = torch.stack(rpn_loss_rpn_cls).mean() proposal_losses['loss_rpn_loc'] = torch.stack(rpn_loss_rpn_loc).mean() detector_losses['loss_cls'] = torch.stack(detector_loss_cls).mean() detector_losses['loss_box_reg'] = torch.stack( detector_loss_box_reg).mean() losses = {} losses.update(detector_losses) losses.update(proposal_losses) return losses