Exemple #1
0
def validate_model(val_loader, model, criterion, epoch, print_freq):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    with torch.no_grad():
        end = time.time()
        target_index_output = []
        target_index_target = []
        for i, (input, target) in enumerate(val_loader):
            target = target.cuda(non_blocking=True)

            # compute output
            output = model(input)
            loss = criterion(output, target)

            # --------------------------------------
            # for auroc get value from target index
            output_cpu = output.squeeze().cpu().data.numpy()
            target_cpu = target.cpu().data.numpy()
            output_cpu = np.array([
                softmax(out)[args.target_index] for out in output_cpu
            ])  # convert to probability
            target_index_output.extend(output_cpu.astype(np.float))
            target_index_target.extend(
                np.equal(target_cpu, args.target_index).astype(np.int))
            # --------------------------------------

            # measure accuracy and record loss
            prec1 = accuracy(output, target, topk=(1, ))
            losses.update(loss.item(), input.size(0))
            top1.update(prec1[0].cpu().data.numpy()[0], input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            if i % print_freq == 0:
                print('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                          i,
                          len(val_loader),
                          batch_time=batch_time,
                          loss=losses,
                          top1=top1))

    auc, roc = compute_auroc(target_index_output, target_index_target)
    save_auroc(auc, roc, os.path.join(args.result, str(epoch) + '.png'))

    print(' * Prec@1 {top1.avg:.3f} at Epoch {epoch:0}'.format(top1=top1,
                                                               epoch=epoch))
    print(' * auc@1 {auc:.3f}'.format(auc=auc))

    return top1.avg
Exemple #2
0
def validate_model(val_loader, model, criterion, epoch, print_freq):
    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    cnt_cnt_label = [0] * args.num_classes
    cnt_exact_pred = [0] * args.num_classes

    # switch to evaluate mode
    model.eval()

    if args.evaluate:
        evaluate_csv_file = os.path.join(args.result, 'evaluate.csv')
        feval = open(evaluate_csv_file, 'wt')

    with torch.no_grad():
        end = time.time()
        target_index_output, target_index_target = list(), list()
        for i, (input, target, input_path) in enumerate(val_loader):
            target = target.cuda(non_blocking=True)

            # compute output
            output = model(input)
            loss = criterion(output, target)

            # --------------------------------------
            # for auroc get value from target index
            output_cpu = output.squeeze().cpu().data.numpy()
            output_cpu = np.array([softmax(out)[args.target_index] for out in output_cpu])
            target_index_output.extend(output_cpu.astype(np.float))
            target_index_target.extend(np.equal(target.cpu().data.numpy(), args.target_index).astype(np.int))
            # --------------------------------------

            if args.evaluate:
                output_softmax = np.array([softmax(out) for out in output.cpu().numpy()])
                for file_path, pred_values in zip(input_path, output_softmax):
                    _, file = os.path.split(file_path)
                    name, _ = os.path.splitext(file)
                    line = ','.join([name] + [str(v) for v in pred_values])
                    feval.write(line + '\n')

            # measure accuracy and record loss
            prec1 = accuracy(output, target, topk=(1,))
            losses.update(loss.item(), input.size(0))
            top1.update(prec1[0].cpu().data.numpy()[0], input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            # put together for acc per label
            pred_list = pred(output).cpu().numpy().squeeze()
            target_list = target.cpu().numpy().squeeze()
            for (p, t) in zip(pred_list, target_list):
                cnt_cnt_label[t] += 1
                if p == t:
                    cnt_exact_pred[t] += 1

                pred_list = pred(output).cpu().numpy().squeeze()
                for pred_idx, pred_item in enumerate(pred_list):
                    dst = os.path.join(args.classification_result, 'kidney' if pred_item == 1 else 'no_kidney')

                    if not os.path.exists(dst):
                        os.makedirs(dst)

                    seg_img = input_path[pred_idx]
                    shutil.copy(seg_img, dst)

            if i % print_freq == 0:
                log(('Test: [{0}/{1}]\t' +
                     'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' +
                     'Loss {loss.val:.4f} ({loss.avg:.4f})\t' +
                     'Prec@1 {top1.val:.3f} ({top1.avg:.3f})').format(i,
                                                                      len(val_loader),
                                                                      batch_time=batch_time,
                                                                      loss=losses,
                                                                      top1=top1))

        auc, roc = compute_auroc(target_index_output, target_index_target)
        # save_auroc(auc, roc, os.path.join(args.result, str(epoch) + '.png'))

        log(' * Prec@1 {top1.avg:.3f} at Epoch {epoch:0}'.format(top1=top1, epoch=epoch))
        log(' * auc@1 {auc:.3f}'.format(auc=auc))

        for (i, (n_label, n_exact)) in enumerate(zip(cnt_cnt_label, cnt_exact_pred)):
            acc_label = (n_exact / n_label * 100) if n_label > 0 else 0
            log('acc of label {:0d}: {:0.3f}%'.format(i, acc_label))

    return auc
Exemple #3
0
def validate_model(val_loader, model, criterion, epoch, print_freq):
    if args.evaluate:
        print('eval check')
        val_loader = load_dataset(find_csv(args.data, ''))
    else:
        val_loader = load_dataset(find_csv(args.data, 'val'))

    batch_time = AverageMeter()
    losses = AverageMeter()
    top1 = AverageMeter()
    cnt_cnt_label = [0] * args.num_classes
    cnt_exact_pred = [0] * args.num_classes

    # switch to evaluate mode
    model.eval()

    if args.evaluate:
        evaluate_csv_file = os.path.join(args.result, 'evaluate.csv')
        feval = open(evaluate_csv_file, 'wt')

    with torch.no_grad():
        end = time.time()
        target_index_output, target_index_target = list(), list()

        for i, (target, cont_input, cat_input,
                identification) in enumerate(val_loader):

            target = target.cuda(non_blocking=True)
            target = target.squeeze()

            # compute output
            output = model(cont_input, cat_input)

            # loss = criterion(output, target)
            loss = criterion_smoothing(output, target)

            # --------------------------------------
            # for auroc get value from target index
            output_cpu = output.cpu().data.numpy()

            output_cpu = np.array(
                [softmax(out)[args.target_index] for out in output_cpu])
            target_index_output.extend(output_cpu.astype(np.float))
            target_index_target.extend(
                np.equal(target.cpu().data.numpy(),
                         args.target_index).astype(np.int))
            # --------------------------------------

            if args.evaluate:
                output_softmax = np.array(
                    [softmax(out) for out in output.cpu().numpy()])
                for ident, pred_values in zip(identification, output_softmax):
                    ident = str(np.squeeze(ident.data.numpy()))
                    idx_biger = np.argmax(pred_values)
                    diag = 'CKD' if idx_biger == 1 else 'AKI or NOR'

                    line = ','.join([ident, diag])
                    feval.write(line + '\n')

            # measure accuracy and record loss
            prec1 = accuracy(output, target, topk=(1, ))

            losses.update(loss.item(), target.size(0))
            top1.update(prec1[0].cpu().data.numpy()[0], target.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            # put together for acc per label
            pred_list = pred(output).cpu().numpy()[0]
            target_list = target.cpu().numpy()

            for (p, t) in zip(pred_list, target_list):
                cnt_cnt_label[t] += 1
                if p == t:
                    cnt_exact_pred[t] += 1

            if i % print_freq == 0 and False:
                log(('Test: [{0}/{1}]\t' +
                     'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' +
                     'Loss {loss.val:.4f} ({loss.avg:.4f})\t' +
                     'Prec@1 {top1.val:.3f} ({top1.avg:.3f})').format(
                         i,
                         len(val_loader),
                         batch_time=batch_time,
                         loss=losses,
                         top1=top1))

        auc, roc = compute_auroc(target_index_output, target_index_target)

        global best_acc
        global best_auc
        # if auc > best_auc:
        if top1.avg > best_acc:
            log(' * Prec@1 {top1.avg:.3f} at Epoch {epoch:0}'.format(
                top1=top1, epoch=epoch))
            log(' * auc@1 {auc:.3f}'.format(auc=auc))
            best_acc = top1.avg
            best_auc = auc

            acc_label_list = []
            for (i, (n_label,
                     n_exact)) in enumerate(zip(cnt_cnt_label,
                                                cnt_exact_pred)):
                acc_label = (n_exact / n_label * 100) if n_label > 0 else 0
                acc_label_list.append(acc_label)
                log('acc of label {:0d}: {:0.3f}%'.format(i, acc_label))

            save_values(epoch, losses.avg, top1.avg, auc, acc_label_list)
            print("=" * 50)
    return auc
def validate_model(val_loader, model, criterion, epoch, print_freq):
    batch_time = AverageMeter()
    top1 = AverageMeter()

    # switch to evaluate mode
    model.eval()

    with torch.no_grad():
        end = time.time()
        target_index_output = []
        target_index_target = []
        for i, (input, target, filename) in enumerate(val_loader):
            target = target.cuda(non_blocking=True)

            # compute output
            output, output_seg = model(input)

            if args.acc_classify:
                prec1 = accuracy(output, target, topk=(1, ))
                prec1 = prec1[0].cpu().data.numpy()[0]

                # for auroc
                output_cpu = output.squeeze().cpu().data.numpy()
                output_cpu = np.array([
                    softmax(out)[args.target_index] for out in output_cpu
                ])  # convert to probability
            else:
                output_max = F.max_pool2d(output_seg, global_pooling_size)
                output_max_cpu = output_max.cpu().data.numpy()
                target_cpu = target.cpu().data.numpy()

                output_max_cpu = [
                    1.0 if o > 0.5 else 0.0 for o in output_max_cpu
                ]
                prec1 = np.average(
                    np.equal(output_max_cpu, target_cpu).astype(
                        np.float)) * 100

                # for auroc
                output_cpu = output_max.squeeze().cpu().data.numpy()

            # --------------------------------------
            # for auroc get value from target index
            target_index_output.extend(output_cpu.astype(np.float))
            target_index_target.extend(
                np.equal(target.cpu().data.numpy(),
                         args.target_index).astype(np.int))
            # --------------------------------------

            # measure accuracy and record loss
            top1.update(prec1, input.size(0))

            # measure elapsed time
            batch_time.update(time.time() - end)
            end = time.time()

            # save segmentation result
            if args.seg_result != '':
                name = [os.path.split(f)[1] for f in filename]
                save_tensor_image(output_seg, name, args.seg_result)

            if i % print_freq == 0:
                print('Test: [{0}/{1}]\t'
                      'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                      'Prec@1 {top1.val:.3f} ({top1.avg:.3f})'.format(
                          i, len(val_loader), batch_time=batch_time,
                          top1=top1))

    auc, roc = compute_auroc(target_index_output, target_index_target)
    save_auroc(auc, roc, os.path.join(args.result, str(epoch) + '.png'))

    print(' * Prec@1 {top1.avg:.3f} at Epoch {epoch:0}'.format(top1=top1,
                                                               epoch=epoch))
    print(' * auc@1 {auc:.3f}'.format(auc=auc))

    return auc