def valid(self, epoch, val_loader): self.G.eval() with torch.no_grad(): # (tn, fp, fn, tp) cm = utils.ConfusionMatrix() for i, (input_, target_, _) in enumerate(val_loader): input_ = input_.to(self.torch_device) output_ = self.G(input_) target_ = target_.to(self.torch_device) ground_truth = target_.int().squeeze(1) prediction = torch.argmax(output_, dim=1).int() cm.update( utils.confusion_matrix(prediction, ground_truth, reduce=False)) metric = 1.5 * cm.f2 + cm.accuracy if metric > self.best_metric: self.best_metric = metric self.save(epoch) self.logger.write( "[Val] epoch: %d accuracy: %f f05: %f f1: %f f2: %f" % (epoch, cm.accuracy, cm.f05, cm.f1, cm.f2))
def train_one_epoch(model, criterion, optimizer, data_loader, lr_scheduler, device, epoch, print_freq, num_classes): epoch_loss = list() model.train() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value}')) header = 'Epoch: [{}]'.format(epoch) for image, target in metric_logger.log_every(data_loader, print_freq, header): image, target = image.to(device), target.to(device) output = model(image) loss = criterion(output, target) epoch_loss.append(loss.item()) confmat.update(target.flatten(), output['out'].argmax(1).flatten()) optimizer.zero_grad() loss.backward() optimizer.step() lr_scheduler.step() metric_logger.update(loss=loss.item(), lr=optimizer.param_groups[0]["lr"]) confmat.reduce_from_all_processes() acc_global, _, _ = confmat.compute() epoch_acc = acc_global.item() return epoch_acc, sum(epoch_loss) / len(epoch_loss) if len( epoch_loss) else 0.0
def valid(self, epoch, val_loader): self.G.eval() with torch.no_grad(): # (tn, fp, fn, tp) cm = utils.ConfusionMatrix() sd = utils.SurfaceDSC() for i, (input_, target_, _) in enumerate(val_loader): _, output_, target_ = self.forward_for_test(input_, target_) sd.update(utils.surface_DSC_batch(target_, output_), n=output_.shape[0]) cm.update(utils.confusion_matrix_2d(output_, target_, 0.5, reduce=False), n=output_.shape[0]) metric = sd.sDSC.avg if metric > self.best_metric: self.best_metric = metric self.save(epoch) self.logger.write( "[Val] epoch: %d f05: %f f1: %f f2: %f jacard: %f dice: %f surf_dice: %f" % (epoch, cm.f05, cm.f1, cm.f2, cm.jcc.avg, cm.dice.avg, sd.sDSC.avg))
def evaluate(model, data_loader, device, num_classes): model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = "Test:" num_processed_samples = 0 with torch.inference_mode(): for image, target in metric_logger.log_every(data_loader, 100, header): image, target = image.to(device), target.to(device) output = model(image) output = output["out"] confmat.update(target.flatten(), output.argmax(1).flatten()) # FIXME need to take into account that the datasets # could have been padded in distributed setup num_processed_samples += image.shape[0] confmat.reduce_from_all_processes() num_processed_samples = utils.reduce_across_processes( num_processed_samples) if (hasattr(data_loader.dataset, "__len__") and len(data_loader.dataset) != num_processed_samples and torch.distributed.get_rank() == 0): # See FIXME above warnings.warn( f"It looks like the dataset has {len(data_loader.dataset)} samples, but {num_processed_samples} " "samples were used for the validation, which might bias the results. " "Try adjusting the batch size and / or the world size. " "Setting the world size to 1 is always a safe bet.") return confmat
def evaluate(model, data_loader, device, num_classes): model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") class_iou_image = list() img_list = list() target_list = list() prediction_list = list() header = "Evaluate:" with torch.inference_mode(): for image, target in metric_logger.log_every(data_loader, 100, header): image, target = image.to(device), target.to(device) confmat_image = utils.ConfusionMatrix(num_classes) output = model(image) output = output["out"] inv_normalize = transforms.Normalize(mean=(-0.485, -0.456, -0.406), std=(1 / 0.229, 1 / 0.224, 1 / 0.225)) img_npy = inv_normalize(image[0], target)[0].cpu().detach().numpy() target_npy = target.cpu().detach().numpy() prediction_npy = output.cpu().detach().numpy() img_list.append(img_npy) target_list.append(target_npy) prediction_list.append(prediction_npy) confmat.update(target.flatten(), output.argmax(1).flatten()) confmat_image.update(target.flatten(), output.argmax(1).flatten()) class_iou_image.append(confmat_image.get_class_iou()) confmat_image.reduce_from_all_processes() confmat.reduce_from_all_processes() return confmat, class_iou_image, img_list, target_list, prediction_list
def test(self, test_loader, val_loader): print("\nStart Test") self.G.eval() with torch.no_grad(): sd = utils.SurfaceDSC() cm = utils.ConfusionMatrix() y_true = np.array([]) y_pred = np.array([]) for i, (input_, target_, f_name) in enumerate(test_loader): input_, output_, target_ = self.forward_for_test( input_, target_) sd.update(utils.surface_DSC_batch(target_, output_), n=output_.shape[0]) cm.update(utils.confusion_matrix_2d(output_, target_, 0.5, reduce=False), n=output_.shape[0]) input_np = input_.type(torch.FloatTensor).numpy() target_np = target_.type(torch.FloatTensor).numpy() output_np = output_.type(torch.FloatTensor).numpy() y_true = np.concatenate([y_true, target_np.flatten()], axis=0) y_pred = np.concatenate([y_pred, output_np.flatten()], axis=0) for batch_idx in range(0, input_.shape[0]): input_b = input_np[batch_idx, 0, :, :] target_b = target_np[batch_idx, 0, :, :] output_b = output_np[batch_idx, 0, :, :] save_path = "%s/fold%s/%s" % (self.save_path, self.fold, f_name[batch_idx][:-4]) utils.image_save(save_path, input_b, target_b, output_b) self.logger.will_write( "[Save] fname:%s dice:%f jss:%f surf_dice:%f" % (f_name[batch_idx][:-4], cm.dice.val[batch_idx], cm.jcc.val[batch_idx], sd.sDSC.val[batch_idx])) pr_values = np.array(precision_recall_curve(y_true, y_pred)) roc_auc = roc_auc_score(y_true, y_pred) pr_auc = auc(pr_values[0], pr_values[1], reorder=True) self.logger.write( "Best dice:%f surf_dice:%f jcc:%f f05:%f f1:%f f2:%f roc:%f pr:%f" % (cm.dice.avg, sd.sDSC.avg, cm.jcc.avg, cm.f05, cm.f1, cm.f2, roc_auc, pr_auc)) print("End Test\n")
def evaluate(model, data_loader, device, num_classes): model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' with torch.no_grad(): for image, target in metric_logger.log_every(data_loader, 100, header): target = target.to(device) output = model(image) output = output['out'] confmat.update(target.flatten(), output.argmax(1).flatten()) confmat.reduce_from_all_processes() return confmat
def test(self, test_loader, val_loader): print("\nStart Test") self.G.eval() with torch.no_grad(): cm = utils.ConfusionMatrix() y_true = np.array([]) y_pred = np.array([]) for i, (input_, target_, clinic_, f_name) in enumerate(test_loader): input_, clinic_, output_, target_ = self.forward_for_test( input_, target_, clinic_) ground_truth = target_.int().squeeze(1) prediction = torch.argmax(output_, dim=1).int() cm.update( utils.confusion_matrix(prediction, ground_truth, reduce=False)) prediction_np = prediction.type(torch.FloatTensor).numpy() ground_truth_np = ground_truth.type(torch.FloatTensor).numpy() y_true = np.concatenate([y_true, ground_truth_np], axis=0) y_pred = np.concatenate([y_pred, prediction_np], axis=0) for batch_idx in range(0, input_.shape[0]): output_b = prediction_np[batch_idx] target_b = ground_truth_np[batch_idx] self.logger.will_write( "[Test] fname:%s true_label:%f prediction:%f" % (f_name[batch_idx][:-4], target_b, output_b)) pr_values = np.array(precision_recall_curve(y_true, y_pred)) roc_auc = roc_auc_score(y_true, y_pred) pr_auc = auc(pr_values[0], pr_values[1], reorder=True) self.logger.write("accuracy:%f f05:%f f1:%f f2:%f roc:%f pr:%f" % (cm.accuracy, cm.f05, cm.f1, cm.f2, roc_auc, pr_auc)) print("End Test\n")
def evaluate(model, criterion, data_loader, device, num_classes): epoch_loss = list() model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' with torch.no_grad(): for image, target in metric_logger.log_every(data_loader, 100, header): image, target = image.to(device), target.to(device) output = model(image) loss = criterion(output, target) epoch_loss.append(loss.item()) output = output['out'] confmat.update(target.flatten(), output.argmax(1).flatten()) confmat.reduce_from_all_processes() print('eval confmat', confmat) acc_global, _, _ = confmat.compute() epoch_acc = acc_global.item() return epoch_acc, sum(epoch_loss) / len(epoch_loss) if len( epoch_loss) else 0.0
def main(args): if args.output_dir: utils.mkdir(args.output_dir) utils.init_distributed_mode(args) print(args) device = torch.device(args.device) # CRF post-processor postprocessor = DenseCRF( iter_max=10, pos_xy_std=1, pos_w=3, bi_xy_std=67, bi_rgb_std=3, bi_w=4, ) folder_name = str(args.model) + str(args.backbone) # Path to prediction images prediction_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "prediction", ) # Path to logits logit_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "logit", ) print("Logit src:", logit_dir) if not os.path.isdir(logit_dir): print("Logit not found, run first: python main.py test [OPTIONS]") quit() # Path to save scores save_dir = os.path.join( args.output_dir, "scores", "voc12", folder_name.lower(), "val", ) save_path = os.path.join(save_dir, "scores_crf.json") print("Score dst:", save_path) dataset_test, num_classes = get_dataset(args.data_path, args.dataset, "val", get_transform(train=False)) confmat = utils.ConfusionMatrix(num_classes) # Process per sample def process(i): image, target = dataset_test.__getitem__(i) filename = os.path.join(str(logit_dir), str(i) + ".npy") logit = np.load(filename)[0] _, H, W = image.shape logit = torch.FloatTensor(logit)[None, ...] logit = F.interpolate(logit, size=(H, W), mode="bilinear", align_corners=False) prob = F.softmax(logit, dim=1)[0].numpy() image = image.numpy().astype(np.uint8).transpose(1, 2, 0) prob = postprocessor(image, prob) label = np.argmax(prob, axis=0) return label, target for i in tqdm(range(len(dataset_test))): image, target = process(i) writer.add_image('Images/image', image, i, dataformats='HW') writer.add_image('Images/target', target, i, dataformats='HW') confmat.update(target.flatten(), image.flatten()) writer.add_scalar("Mean IoU/val", confmat.get_IoU(), i) writer.flush() confmat.reduce_from_all_processes() with open(save_path, "w") as f: print(confmat, file=f)
def __init__(self, learn, n_classes, pred_func, obj, target_func=None): super().__init__(learn, ['P.A.', 'mIoU', 'FP/FN']) self.target_func = target_func if target_func else utils.cm_target_func self.pred_func = pred_func self.cm = utils.ConfusionMatrix(n_classes) self.obj = obj
def evaluate(model, data_loader, device, num_classes, output_dir, save=False): model.eval() metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' confmat = utils.ConfusionMatrix(num_classes) per_mean_iou = list() image_epoch_dir = os.path.join(output_dir, "images/") utils.mkdir(image_epoch_dir) image_dir = os.path.join(output_dir, "images/image/") utils.mkdir(image_dir) target_dir = os.path.join(output_dir, "images/target/") utils.mkdir(target_dir) prediction_dir = os.path.join(output_dir, "images/prediction/") utils.mkdir(prediction_dir) mean_iou_file = os.path.join(output_dir, "mean_iou.csv") per_mean_iou_file = os.path.join(output_dir, "per_image_mean_iou.csv") with torch.no_grad(): start_time = time.time() for idx, (image, target) in enumerate( metric_logger.log_every(data_loader, 10, header)): image, target = image.to(device), target.to(device) inv_normalize = T.Normalize(mean=(-0.485, -0.456, -0.406), std=(1 / 0.229, 1 / 0.224, 1 / 0.225)) #if idx<10: # save_image(inv_normalize(image[0], target)[0], "img{}.png".format(idx)) confmat_image = utils.ConfusionMatrix(num_classes) output = model(image) output = output['out'] if save: image_path = os.path.join(image_dir, '{}.npy'.format(idx)) target_path = os.path.join(target_dir, '{}.npy'.format(idx)) prediction_path = os.path.join(prediction_dir, '{}.npy'.format(idx)) utils.save_on_master( inv_normalize(image[0], target)[0], image_path) utils.save_on_master(target, target_path) utils.save_on_master(output, prediction_path) confmat.update(target.flatten(), output.argmax(1).flatten()) confmat_image.update(target.flatten(), output.argmax(1).flatten()) acc_global, acc, iu = confmat_image.compute() confmat_image.reduce_from_all_processes() image_mean_iou = list((iu * 100).tolist()) per_mean_iou.append(image_mean_iou) confmat.reduce_from_all_processes() total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=int(total_time))) print('Validation time {}'.format(total_time_str)) with open(per_mean_iou_file, 'w', newline='') as f: wr = csv.writer(f, quoting=csv.QUOTE_ALL) wr.writerow(per_mean_iou) with open(mean_iou_file, 'w', newline='') as f: wr = csv.writer(f, quoting=csv.QUOTE_ALL) wr.writerow([confmat.get_mean_iou()]) return confmat
def train(train_dir, val_dir, checkpoint_file=None): train_loader, val_loader = load_data(train_dir, val_dir) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = models.segmentation.deeplabv3_resnet50(pretrained=False, num_classes=2) for param in model.parameters(): param.requires_grad = True model = nn.Sequential(model, CRF(n_spatial_dims=2)) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.002) model.to(device) writer = SummaryWriter('runs/deeplab_experiment_11') epochs = 100 print_freq = 500 save_freq = 25 lr_scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lambda x: (1 - x / (len(train_loader) * epochs))**0.9) if checkpoint_file is not None: checkpoint = torch.load(checkpoint_file) model.load_state_dict(checkpoint['model']) optimizer.load_state_dict(checkpoint['optimizer']) lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) last_epoch = checkpoint['epoch'] acc_global = checkpoint['acc_global'] acc = checkpoint['acc'] iou = checkpoint['iou'] miou = checkpoint['miou'] dice_coef = checkpoint['dice_coef'] mcc = checkpoint['mcc'] else: last_epoch = 0 for epoch in range(last_epoch + 1, last_epoch + epochs + 1): train_mcc = 0.0 train_confmat = utils.ConfusionMatrix(num_classes=2) for inputs, labels in train_loader: model.train() inputs, labels = inputs.to(device), labels.to(device) optimizer.zero_grad() try: logps = model(inputs) #logps = logps['out'].squeeze(1) except: continue loss = criterion(logps, labels) loss.backward() optimizer.step() lr_scheduler.step(loss.cpu().data.numpy()) train_pred = torch.argmax(logps, dim=1) train_confmat.update(labels.flatten().long(), train_pred.flatten()) train_mcc += matthews_corrcoef(labels.cpu().numpy().flatten(), train_pred.cpu().numpy().flatten()) train_acc_global, train_acc, train_iou, train_miou = train_confmat.compute( ) train_mcc = train_mcc / len(train_loader) print("Train loss: ", loss.item(), " ... ", "Train acc: ", train_acc_global, " ... ", "Train mIOU: ", train_miou, " ... ", "Train MCC: ", train_mcc) writer.add_scalar('Train/Loss', loss.item(), epoch) writer.add_scalar('Train/Global Accuracy', train_acc_global, epoch) writer.add_scalar('Train/Accuracy/nontumor', train_acc[0], epoch) writer.add_scalar('Train/Accuracy/tumor', train_acc[1], epoch) writer.add_scalar('Train/IoU/nontumor', train_iou[0], epoch) writer.add_scalar('Train/IoU/tumor', train_iou[1], epoch) writer.add_scalar('Train/mIoU', train_miou, epoch) writer.add_scalar('Train/MCC', train_mcc, epoch) # Evaluate validation loss and confusion matrix after every epoch model.eval() val_loss = 0.0 val_mcc = 0.0 confmat = utils.ConfusionMatrix(num_classes=2) with torch.no_grad(): for val_inputs, val_labels in val_loader: val_inputs, val_labels = val_inputs.to(device), val_labels.to( device) try: val_logps = model(val_inputs) #val_logps = val_logps['out'].squeeze(1) except: continue val_preds = torch.argmax(val_logps, dim=1) probability = torch.sigmoid(val_logps) predicted = (probability > 0.5).int() confmat.update(val_labels.flatten().long(), val_preds.flatten()) val_mcc += matthews_corrcoef( val_labels.cpu().numpy().flatten(), val_preds.cpu().numpy().flatten()) batch_loss = criterion(val_logps, val_labels) val_loss += batch_loss.item() val_loss = val_loss / len(val_loader) acc_global, acc, iou, miou = confmat.compute() dice_coef = dice(val_preds.flatten(), val_labels.flatten()) val_mcc = val_mcc / len(val_loader) print("Val loss: ", val_loss, " ... ", "Val acc: ", acc_global, " ... ", "Val mIOU: ", miou, " ... ", "Val Dice coeff: ", dice_coef, "Val MCC: ", val_mcc) writer.add_scalar('Val/Loss', val_loss, epoch) writer.add_scalar('Val/Global Accuracy', acc_global, epoch) writer.add_scalar('Val/Accuracy/nontumor', acc[0], epoch) writer.add_scalar('Val/Accuracy/tumor', acc[1], epoch) writer.add_scalar('Val/IoU/nontumor', iou[0], epoch) writer.add_scalar('Val/IoU/tumor', iou[1], epoch) writer.add_scalar('Val/mIoU', miou, epoch) writer.add_scalar('Val/Dice coeff', dice_coef, epoch) writer.add_scalar('Val/MCC', val_mcc, epoch) writer.close() # Save checkpoint after every save_freq epochs if epoch % save_freq == 0: utils.save_on_master( { 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'lr_scheduler': lr_scheduler.state_dict(), 'epoch': epoch, 'acc_global': acc_global, 'acc': acc, 'iou': iou, 'miou': miou, 'dice_coef': dice_coef, 'mcc': val_mcc }, os.path.join( './checkpoints', 'deeplab_resnet101_experiment_11_{}.pth'.format(epoch)))
def evaluate(args, model, data_loader, ref_ids, refer, bert_model, device, num_classes, display=False, baseline_model=None, objs_ids=None, num_objs_list=None): model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") refs_ids_list = [] outputs = [] # dict to save results for DAVIS total_outputs = {} # evaluation variables cum_I, cum_U = 0, 0 eval_seg_iou_list = [.5, .6, .7, .8, .9] seg_correct = np.zeros(len(eval_seg_iou_list), dtype=np.int32) seg_total = 0 mean_IoU = [] header = 'Test:' with torch.no_grad(): k = 0 l = 0 for image, target, sentences, attentions in metric_logger.log_every( data_loader, 100, header): image, target, sentences, attentions = image.to(device), target.to( device), sentences.to(device), attentions.to(device) sentences = sentences.squeeze(1) attentions = attentions.squeeze(1) if args.dataset == 'davis' or args.dataset == 'a2d': sentences = sentences.unsqueeze(-1) attentions = attentions.unsqueeze(-1) target = target.cpu().data.numpy() for j in range(sentences.size(-1)): refs_ids_list.append(k) if args.baseline_bilstm: sent = sentences[:, :, :, j] att = attentions[:, :, j] num_tokens = torch.sum(att, dim=-1) processed_seqs = sent[:num_tokens, :] hidden_states, cell_states = baseline_model[0]( processed_seqs) hidden_states = hidden_states[0] processed_hidden_states = hidden_states[:num_tokens, :] last_hidden_states = torch.mean(processed_hidden_states, dim=0) last_hidden_states = baseline_model[1](last_hidden_states) embedding = last_hidden_states.unsqueeze(1) else: last_hidden_states = bert_model( sentences[:, :, j], attention_mask=attentions[:, :, j])[0] embedding = last_hidden_states[:, 0, :] output, _, _ = model(image, embedding.squeeze(1)) output = output['out'].cpu() output_mask = output.argmax(1).data.numpy() outputs.append(output_mask) I, U = computeIoU(output_mask, target) if U == 0: this_iou = 0.0 else: this_iou = I * 1.0 / U mean_IoU.append(this_iou) cum_I += I cum_U += U for n_eval_iou in range(len(eval_seg_iou_list)): eval_seg_iou = eval_seg_iou_list[n_eval_iou] seg_correct[n_eval_iou] += (this_iou >= eval_seg_iou) seg_total += 1 del image, target, attentions if display: plt.figure() plt.axis('off') if args.dataset == 'refcoco' or args.dataset == 'refcoco+': ref = refer.loadRefs(ref_ids[k]) image_info = refer.Imgs[ref[0]['image_id']] for p in range(len(ref[0]['sentences'])): l += 1 if args.dataset == 'refcoco' or args.dataset == 'refcoco+': sentence = ref[0]['sentences'][p]['raw'] im_path = os.path.join(refer.IMAGE_DIR, image_info['file_name']) elif args.dataset == 'davis': idx = ref_ids[k] sentence = refer[idx] im_path = os.path.join(args.davis_data_root, img_list[k]) elif args.dataset == 'a2d': sentence = refer[k] image_name = ref_ids[k] im_path = os.path.join(args.a2d_root_dir, image_name) im = imread(im_path) plt.imshow(im) if args.dataset == 'davis': if img_list[k] not in total_outputs: total_outputs[img_list[k]] = {} o_mask = output_mask.copy() o_mask = o_mask.astype(int) * int( idx.split('_')[-1]) total_outputs[img_list[k]] = o_mask.squeeze(0) else: total_outputs[img_list[k]][output_mask.squeeze( 0) == True] = int(idx.split('_')[-1]) plt.text(0, 0, sentence, fontsize=12) ax = plt.gca() ax.set_autoscale_on(False) # mask definition img = np.ones((im.shape[0], im.shape[1], 3)) color_mask = np.array([0, 255, 0]) / 255.0 for i in range(3): img[:, :, i] = color_mask[i] if args.dataset == 'refcoco' or args.dataset == 'refcoco+': output_mask = outputs[-len(ref[0]['sentences']) + p].transpose(1, 2, 0) ax.imshow(np.dstack((img, output_mask * 0.5))) if not os.path.isdir(results_folder): os.makedirs(results_folder) figname = os.path.join(args.results_folder, str(l) + '.png') plt.close() k += 1 if args.dataset == 'davis': for r in total_outputs.keys(): new_im = Image.fromarray(total_outputs[r].astype(np.uint8)) file_name = r.split('/')[-1].split('.')[0] folder_name = r.split('/')[-2] if not os.path.isdir(os.path.join(submission_path, folder_name)): os.makedirs(os.path.join(submission_path, folder_name)) new_im.save( os.path.join(args.submission_path, folder_name, file_name + '.png')) mean_IoU = np.array(mean_IoU) mIoU = np.mean(mean_IoU) print('Final results:') print('Mean IoU is %.2f\n' % (mIoU * 100.)) results_str = '' for n_eval_iou in range(len(eval_seg_iou_list)): results_str += ' precision@%s = %.2f\n' % \ (str(eval_seg_iou_list[n_eval_iou]), seg_correct[n_eval_iou] * 100. / seg_total) results_str += ' overall IoU = %.2f\n' % (cum_I * 100. / cum_U) print(results_str) return refs_ids_list, outputs
def main(args): folder_name = str(args.model) + str(args.backbone) # Path to save logits logit_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "logit", ) utils.mkdir(logit_dir) print("Logit dst:", logit_dir) # Path to save images image_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "image", ) utils.mkdir(image_dir) print("Image dst:", image_dir) # Path to save processed images processed_image_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "processed_image", ) utils.mkdir(processed_image_dir) print("Processed Image dst:", processed_image_dir) # Path to ground truth images ground_truth_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "ground_truth", ) utils.mkdir(ground_truth_dir) print("Ground truth dst:", ground_truth_dir) # Path to prediction images prediction_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "prediction", ) utils.mkdir(prediction_dir) print("Prediction dst:", prediction_dir) # Path to target images target_dir = os.path.join( args.output_dir, "features", "voc12", folder_name.lower(), "val", "target", ) utils.mkdir(target_dir) print("Target dst:", target_dir) # Path to save scores save_dir = os.path.join( args.output_dir, "scores", "voc12", folder_name.lower(), "val", ) utils.mkdir(save_dir) save_path = os.path.join(save_dir, "scores.json") print("Score dst:", save_path) if args.output_dir: utils.mkdir(args.output_dir) utils.init_distributed_mode(args) print(args) iterator = utils.Iterator() device = torch.device(args.device) dataset_test, num_classes = get_dataset(args.data_path, args.dataset, "val", get_transform(train=False)) if args.distributed: test_sampler = torch.utils.data.distributed.DistributedSampler( dataset_test) else: test_sampler = torch.utils.data.SequentialSampler(dataset_test) data_loader_test = torch.utils.data.DataLoader(dataset_test, batch_size=1, shuffle=False, sampler=test_sampler, num_workers=args.workers, collate_fn=utils.collate_fn) model = _load_model(arch_type=args.model, backbone=args.backbone, pretrained=False, progress=True, num_classes=num_classes, aux_loss=args.aux_loss) model.to(torch.device('cuda')) if args.distributed: model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model) model_without_ddp = model if args.distributed: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) model_without_ddp = model.module params_to_optimize = [ { "params": [ p for p in model_without_ddp.backbone.parameters() if p.requires_grad ] }, { "params": [ p for p in model_without_ddp.classifier.parameters() if p.requires_grad ] }, ] if args.aux_loss: params = [ p for p in model_without_ddp.aux_classifier.parameters() if p.requires_grad ] params_to_optimize.append({"params": params, "lr": args.lr * 10}) optimizer = torch.optim.SGD(params_to_optimize, lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) checkpoint = torch.load( "/home/AD/rraina/segmentation_benchmark/semseg/output_models/checkpoint.pth", map_location='cpu') model_without_ddp.load_state_dict(checkpoint['model']) optimizer.load_state_dict(checkpoint['optimizer']) model.eval() start_time = time.time() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' with torch.no_grad(): for idx, (image, target) in tqdm(enumerate(dataset_test)): image, target = image.to(device), target.to(device) image = image.unsqueeze(0) target = target.unsqueeze(0) output = model(image) output = output['out'] # Saving Logits filename = os.path.join(logit_dir, str(idx) + ".npy") np.save(filename, output.cpu().numpy()) # Saving Ground Truths ground_truth = Image.open(str(data_loader_test.dataset.masks[idx])) filename = os.path.join(ground_truth_dir, str(idx) + ".png") ground_truth.save(str(filename)) # Saving Images images = Image.open(str(data_loader_test.dataset.images[idx])) filename = os.path.join(image_dir, str(idx) + ".png") images.save(str(filename)) # Saving Processed Image processed_image_in = image[0].cpu().numpy().transpose(1, 2, 0) processed_image = Image.fromarray( (processed_image_in * 255).astype(np.uint8)) filename = os.path.join(processed_image_dir, str(idx) + ".png") processed_image.save(str(filename)) # Saving Target Image target_in = target[0].cpu().numpy() target_image = Image.fromarray(np.uint8(target_in * 255), 'L') filename = os.path.join(target_dir, str(idx) + ".png") target_image.save(str(filename)) # Saving Prediction Image prediction_in = get_mask(output) prediction_image = Image.fromarray( (prediction_in * 255).astype(np.uint8)) filename = os.path.join(prediction_dir, str(idx) + ".png") prediction_image.save(str(filename)) confmat.update(target.flatten(), output.argmax(1).flatten()) confmat.reduce_from_all_processes() with open(save_path, "w") as f: print(confmat, file=f) total_time = time.time() - start_time total_time_str = str(datetime.timedelta(seconds=int(total_time))) print('Evaluation time {}'.format(total_time_str))
def evaluate(model, data_loader, args, bert_model, device, num_classes, epoch, logger, baseline_model): model.eval() confmat = utils.ConfusionMatrix(num_classes) metric_logger = utils.MetricLogger(delimiter=" ") header = 'Test:' val_loss = 0 seg_loss = 0 cos_loss = 0 total_its = 0 acc_ious = 0 with torch.no_grad(): for data in metric_logger.log_every(data_loader, 100, header): total_its += 1 image, target, sentences, attentions = data image, target, sentences, attentions = image.to(device), target.to( device), sentences.to(device), attentions.to(device) sentences = sentences.squeeze(1) attentions = attentions.squeeze(1) if args.baseline_bilstm: num_tokens = torch.sum(attentions, dim=-1) unbinded_sequences = list(torch.unbind(sentences, dim=0)) processed_seqs = [ seq[:num_tokens[i], :] for i, seq in enumerate(unbinded_sequences) ] packed_sentences = torch.nn.utils.rnn.pack_sequence( processed_seqs, enforce_sorted=False) hidden_states, cell_states = baseline_model[0]( packed_sentences) hidden_states = torch.nn.utils.rnn.pad_packed_sequence( hidden_states, batch_first=True, total_length=20) hidden_states = hidden_states[0] unbinded_hidden_states = list( torch.unbind(hidden_states, dim=0)) processed_hidden_states = [ seq[:num_tokens[i], :] for i, seq in enumerate(unbinded_hidden_states) ] mean_hidden_states = [ torch.mean(seq, dim=0).unsqueeze(0) for seq in processed_hidden_states ] last_hidden_states = torch.cat(mean_hidden_states, dim=0) last_hidden_states = baseline_model[1](last_hidden_states) last_hidden_states = last_hidden_states.unsqueeze(1) else: last_hidden_states = bert_model(sentences, attention_mask=attentions)[0] embedding = last_hidden_states[:, 0, :] output, vis_emb, lan_emb = model(image, embedding.squeeze(1)) iou = IoU(output['out'], target) acc_ious += iou loss = criterion(output, target, args) output = output['out'] confmat.update(target.flatten(), output.argmax(1).flatten()) confmat.reduce_from_all_processes() val_loss = val_loss / total_its iou = acc_ious / total_its logger.scalar_summary('loss', val_loss, epoch) logger.scalar_summary('iou', iou, epoch) return confmat, iou