def save_blobs(method, args): images, blobs = lidc.load() masks = np.load('data/aam-lidc-pred-masks.npy') print 'masks shape {}'.format(masks.shape) pred_blobs, probs = detect_blobs(images, masks, args.threshold, real_blobs=blobs, method=method) write_blobs( pred_blobs, 'data/{}-{}-aam-lidc-pred-blobs.pkl'.format(method, args.threshold)) froc = eval.froc(blobs, pred_blobs, probs, distance='rad') froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE) abpi = average_bppi(pred_blobs) util.save_froc([froc], 'data/{}-{}-lidc-blobs-froc'.format(method, args.threshold), ['{} FROC on LIDC dataset'.format(method)], fppi_max=100) np.savetxt('data/{}-{}-lidc-blobs-abpi.txt'.format(method, args.threshold), np.array([abpi])) print('Average blobs per image LIDC {}'.format(abpi)) images, blobs = jsrt.load(set_name='jsrt140p') masks = np.load('data/aam-jsrt140p-pred-masks.npy') pred_blobs, probs = detect_blobs(images, masks, args.threshold, real_blobs=blobs, method=method) write_blobs( pred_blobs, 'data/{}-{}-aam-jsrt140p-pred-blobs.pkl'.format( method, args.threshold)) froc = eval.froc(blobs, pred_blobs, probs, distance='rad') froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE) abpi = average_bppi(pred_blobs) util.save_froc([froc], 'data/{}-{}-jsrt140p-blobs-froc'.format( method, args.threshold), ['{} FROC on JSRT140 positives dataset'.format(method)], fppi_max=100) np.savetxt( 'data/{}-{}-jsrt140p-blobs-abpi.txt'.format(method, args.threshold), np.array([abpi])) print('Average blobs per image JSRT positives {}'.format(abpi))
def evaluate_model(model, real_blobs_tr, pred_blobs_tr, rois_tr, real_blobs_te, pred_blobs_te, rois_te, load_model=False): X_tr, Y_tr, X_te, Y_te = neural.create_train_test_sets( real_blobs_tr, pred_blobs_tr, rois_tr, real_blobs_te, pred_blobs_te, rois_te) if load_model == True: print 'load weights {}'.format(model.name) model.network.load_weights('data/{}_weights.h5'.format(model.name)) # FIX: remove and add zmuv mean and zmuv std no Preprocessor augment.py if not hasattr(model.preprocessor, 'zmuv_mean'): model.preprocessor.fit(X_tr, Y_tr) else: _ = model.fit(X_tr, Y_tr, X_te, Y_te) model.save('data/' + model.name) pred_blobs_te, probs_te, _ = neural.predict_proba(model, pred_blobs_te, rois_te) return eval.froc(real_blobs_te, pred_blobs_te, probs_te)
def save_performance_history(model_name, args, rois, blobs, pred_blobs, folds): model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model_name = model.name epochs = model.training_params['nb_epoch'] frocs = [] legends = [] fold_idx = 0 for tr, te in folds: model.load('data/' + model_name + '.fold-{}'.format(fold_idx + 1)) frocs.append([]) epochs_set = list(range(1, epochs + 1, 2)) for epoch in epochs_set: weights_file_name = 'data/{}.weights.{:02d}.hdf5'.format( model.name, epoch) model.network.load_weights(weights_file_name) pred_blobs_te, probs_te = neural.predict_proba( model, pred_blobs[te], rois[te]) frocs[fold_idx].append( eval.froc(blobs[te], pred_blobs_te, probs_te)) fold_idx += 1 frocs = np.array(frocs) froc_history = [] aucs_history = [] legends = [] i = 0 print "check -> frocs.shape {}".format(frocs.shape) for epoch in range(1, epochs + 1, 2): frocs_by_epoch = frocs[:, i] froc_history.append( eval.average_froc(np.array(frocs_by_epoch), np.linspace(0.0, 10.0, 101))) aucs_history.append([]) aucs_history[-1].append( util.auc(froc_history[-1], np.linspace(0.2, 4.0, 101))**2) aucs_history[-1].append( util.auc(froc_history[-1], np.linspace(0.0, 5.0, 101))**2) aucs_history[-1].append( util.auc(froc_history[-1], np.linspace(0.0, 10.0, 101))**2) legends.append('Val FROC (LIDC-IDRI), epoch {}'.format(epoch)) i += 1 util.save_froc(froc_history, 'data/{}-val-froc-by-epoch'.format(model_name), legends, with_std=False) util.save_aucs(list(range(1, epochs + 1, 2)), aucs_history, 'data/{}-val-aucs'.format(model_name), ['AUC between 2-4', 'AUC between 0-5', 'AUC between 0-10'])
def detection_vs_distance(method, args): images, blobs = lidc.load() masks = np.load('data/aam-lidc-pred-masks.npy') pred_blobs, probs = detect_blobs(images, masks, args.threshold, real_blobs=blobs, method=method) write_blobs( pred_blobs, 'data/{}-{}-aam-lidc-pred-blobs.pkl'.format(method, args.threshold)) froc = eval.froc(blobs, pred_blobs, probs, distance='rad') froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
def evaluate_classifier(clf, feats_tr, y_tr, real_blobs_te, pred_blobs_te, feats_te): y_tr = y_tr.T[1] print 'feats {}, {}'.format(feats_tr.shape, y_tr.shape) print "raw values ", np.mean(feats_tr, axis=0), np.std( feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr) ''' preproc = augment.Preprocessor() feats_tr = preproc.fit_transform(feats_tr, y_tr) print "pre values ", np.mean(feats_tr, axis=0), np.std(feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr) ''' scaler = StandardScaler() #scaler = MinMaxScaler(feature_range=(-1, 1)) feats_tr = scaler.fit_transform(feats_tr) print "z-scaled ", np.mean(feats_tr, axis=0), np.std( feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr) print "Train SVM" #clf.fit(feats_tr, y_tr) #clf_prob = CalibratedClassifierCV(base_estimator=clf, cv='prefit') clf_prob = CalibratedClassifierCV(base_estimator=clf) clf_prob.fit(feats_tr, y_tr) minv, maxv = 1e10, -1e10 probs_te = [] for feats in feats_te: print "feats te i {}".format(feats.shape) #feats = preproc.transform(feats) feats = scaler.transform(feats) minv = min(minv, feats.min()) maxv = max(maxv, feats.max()) probs_te.append(clf_prob.predict_proba(feats).T[1]) print 'probs ->', probs_te[-1] probs_te = np.array(probs_te) print "min, max feats fed on svm {}, {}".format(minv, maxv) return eval.froc(real_blobs_te, pred_blobs_te, probs_te)
def froc_by_epochs(data, blobs, augmented_blobs, rois, folds, network_model, nb_epochs=30, epoch_interval=2): network_init = None roi_size = 32 streams = 'none' imgs = [] masks = [] for i in range(len(data)): img, lung_mask = data.get(i, downsample=True) sampled, lce, norm = preprocess.preprocess_hardie(img, lung_mask, downsample=True) imgs.append([lce]) masks.append(lung_mask) imgs = np.array(imgs) masks = np.array(masks) # Hardcoding blob set shapes blobs2 = blobs blobs = blobs.reshape((len(blobs), 3)) nb_checkpoints = int(nb_epochs / epoch_interval) epochs = np.linspace(epoch_interval, nb_checkpoints * epoch_interval, nb_checkpoints).astype(np.int) av_frocs = [] names = [] aucs1 = [] aucs2 = [] for epoch in epochs: frocs = [] fold = 1 for tr_idx, te_idx in folds: print "Fold {} ...".format(fold) X_train, Y_train, X_test, Y_test = neural.create_train_test_sets( rois[tr_idx], augmented_blobs[tr_idx], blobs[tr_idx], rois[te_idx], augmented_blobs[te_idx], blobs[te_idx], streams=streams, detector=True) # load network network = neural.create_network(network_model, X_train.shape, fold, streams, detector=False) name = 'data/{}_fold_{}.epoch_{}'.format(network_model, fold, epoch) network.network.load_weights('{}_weights.h5'.format(name)) # open network on detector mode detector_network = neural.create_network(network_model, X_train.shape, fold, streams, detector=True) copy_weights(network, detector_network) # evaluate network on test blobs_te_pred, probs_te_pred = detect_with_network( detector_network, imgs[te_idx], masks[te_idx], fold=fold) froc = eval.froc(blobs2[te_idx], blobs_te_pred, probs_te_pred) frocs.append(froc) fold += 1 names.append('{}, epoch {}'.format(network_model, epoch)) ops = eval.average_froc(frocs, fppi_range) av_frocs.append(ops) aucs1.append(util.auc(ops, range(0, 60))) aucs2.append(util.auc(ops, range(0, 40))) util.save_auc( np.array(range(1, len(aucs1) + 1)) * epoch_interval, aucs1, 'data/{}-auc-0-60'.format(network_model)) util.save_auc( np.array(range(1, len(aucs2) + 1)) * epoch_interval, aucs2, 'data/{}-auc-0-40'.format(network_model)) return av_frocs, names
def eval_cnn_detector(data, blobs, augmented_blobs, rois, folds, model): fold = 1 network_init = None roi_size = 32 streams = 'none' imgs = [] masks = [] for i in range(len(data)): img, lung_mask = data.get(i, downsample=True) sampled, lce, norm = preprocess.preprocess_hardie(img, lung_mask, downsample=True) imgs.append([lce]) masks.append(lung_mask) imgs = np.array(imgs) masks = np.array(masks) # Hardcoding blob set shapes blobs2 = blobs blobs = blobs.reshape((len(blobs), 3)) frocs = [] for tr_idx, te_idx in folds: print "Fold {} ...".format(fold) X_train, Y_train, X_test, Y_test = neural.create_train_test_sets( rois[tr_idx], augmented_blobs[tr_idx], blobs[tr_idx], rois[te_idx], augmented_blobs[te_idx], blobs[te_idx], streams=streams, detector=True) network = neural.create_network(model, X_train.shape, fold, streams, detector=False) if network_init is not None: network.network.load_weights('data/{}_fold_{}_weights.h5'.format( network_init, fold)) # save network name = 'data/{}_fold_{}'.format(model, fold) history = network.fit(X_train, Y_train, X_test, Y_test, streams=(streams != 'none'), cropped_shape=(roi_size, roi_size), checkpoint_prefix=name, checkpoint_interval=2, loss='mse') network.save(name) # open network on detector mode network.network.summary() detector_network = neural.create_network(model, X_train.shape, fold, streams, detector=True) detector_network.network.summary() copy_weights(network, detector_network) #network.network.load_weights('{}_weights.h5'.format(name)) #network.load(name) # evaluate network on test blobs_te_pred, probs_te_pred = detect_with_network(detector_network, imgs[te_idx], masks[te_idx], fold=fold) froc = eval.froc(blobs2[te_idx], blobs_te_pred, probs_te_pred) frocs.append(froc) fold += 1 av_froc = eval.average_froc(frocs, fppi_range) return av_froc