def save_blobs(method, args):
images, blobs = lidc.load()
masks = np.load('data/aam-lidc-pred-masks.npy')
print 'masks shape {}'.format(masks.shape)
pred_blobs, probs = detect_blobs(images,
masks,
args.threshold,
real_blobs=blobs,
method=method)
write_blobs(
pred_blobs,
'data/{}-{}-aam-lidc-pred-blobs.pkl'.format(method, args.threshold))
froc = eval.froc(blobs, pred_blobs, probs, distance='rad')
froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
abpi = average_bppi(pred_blobs)
util.save_froc([froc],
'data/{}-{}-lidc-blobs-froc'.format(method, args.threshold),
['{} FROC on LIDC dataset'.format(method)],
fppi_max=100)
np.savetxt('data/{}-{}-lidc-blobs-abpi.txt'.format(method, args.threshold),
np.array([abpi]))
print('Average blobs per image LIDC {}'.format(abpi))
images, blobs = jsrt.load(set_name='jsrt140p')
masks = np.load('data/aam-jsrt140p-pred-masks.npy')
pred_blobs, probs = detect_blobs(images,
masks,
args.threshold,
real_blobs=blobs,
method=method)
write_blobs(
pred_blobs, 'data/{}-{}-aam-jsrt140p-pred-blobs.pkl'.format(
method, args.threshold))
froc = eval.froc(blobs, pred_blobs, probs, distance='rad')
froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
abpi = average_bppi(pred_blobs)
util.save_froc([froc],
'data/{}-{}-jsrt140p-blobs-froc'.format(
method, args.threshold),
['{} FROC on JSRT140 positives dataset'.format(method)],
fppi_max=100)
np.savetxt(
'data/{}-{}-jsrt140p-blobs-abpi.txt'.format(method, args.threshold),
np.array([abpi]))
print('Average blobs per image JSRT positives {}'.format(abpi))
def evaluate_model(model,
real_blobs_tr,
pred_blobs_tr,
rois_tr,
real_blobs_te,
pred_blobs_te,
rois_te,
load_model=False):
X_tr, Y_tr, X_te, Y_te = neural.create_train_test_sets(
real_blobs_tr, pred_blobs_tr, rois_tr, real_blobs_te, pred_blobs_te,
rois_te)
if load_model == True:
print 'load weights {}'.format(model.name)
model.network.load_weights('data/{}_weights.h5'.format(model.name))
# FIX: remove and add zmuv mean and zmuv std no Preprocessor augment.py
if not hasattr(model.preprocessor, 'zmuv_mean'):
model.preprocessor.fit(X_tr, Y_tr)
else:
_ = model.fit(X_tr, Y_tr, X_te, Y_te)
model.save('data/' + model.name)
pred_blobs_te, probs_te, _ = neural.predict_proba(model, pred_blobs_te,
rois_te)
return eval.froc(real_blobs_te, pred_blobs_te, probs_te)
def save_performance_history(model_name, args, rois, blobs, pred_blobs, folds):
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model_name = model.name
epochs = model.training_params['nb_epoch']
frocs = []
legends = []
fold_idx = 0
for tr, te in folds:
model.load('data/' + model_name + '.fold-{}'.format(fold_idx + 1))
frocs.append([])
epochs_set = list(range(1, epochs + 1, 2))
for epoch in epochs_set:
weights_file_name = 'data/{}.weights.{:02d}.hdf5'.format(
model.name, epoch)
model.network.load_weights(weights_file_name)
pred_blobs_te, probs_te = neural.predict_proba(
model, pred_blobs[te], rois[te])
frocs[fold_idx].append(
eval.froc(blobs[te], pred_blobs_te, probs_te))
fold_idx += 1
frocs = np.array(frocs)
froc_history = []
aucs_history = []
legends = []
i = 0
print "check -> frocs.shape {}".format(frocs.shape)
for epoch in range(1, epochs + 1, 2):
frocs_by_epoch = frocs[:, i]
froc_history.append(
eval.average_froc(np.array(frocs_by_epoch),
np.linspace(0.0, 10.0, 101)))
aucs_history.append([])
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.2, 4.0, 101))**2)
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.0, 5.0, 101))**2)
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.0, 10.0, 101))**2)
legends.append('Val FROC (LIDC-IDRI), epoch {}'.format(epoch))
i += 1
util.save_froc(froc_history,
'data/{}-val-froc-by-epoch'.format(model_name),
legends,
with_std=False)
util.save_aucs(list(range(1, epochs + 1, 2)), aucs_history,
'data/{}-val-aucs'.format(model_name),
['AUC between 2-4', 'AUC between 0-5', 'AUC between 0-10'])
def detection_vs_distance(method, args):
images, blobs = lidc.load()
masks = np.load('data/aam-lidc-pred-masks.npy')
pred_blobs, probs = detect_blobs(images,
masks,
args.threshold,
real_blobs=blobs,
method=method)
write_blobs(
pred_blobs,
'data/{}-{}-aam-lidc-pred-blobs.pkl'.format(method, args.threshold))
froc = eval.froc(blobs, pred_blobs, probs, distance='rad')
froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
def evaluate_classifier(clf, feats_tr, y_tr, real_blobs_te, pred_blobs_te,
feats_te):
y_tr = y_tr.T[1]
print 'feats {}, {}'.format(feats_tr.shape, y_tr.shape)
print "raw values ", np.mean(feats_tr, axis=0), np.std(
feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr)
'''
preproc = augment.Preprocessor()
feats_tr = preproc.fit_transform(feats_tr, y_tr)
print "pre values ", np.mean(feats_tr, axis=0), np.std(feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr)
'''
scaler = StandardScaler()
#scaler = MinMaxScaler(feature_range=(-1, 1))
feats_tr = scaler.fit_transform(feats_tr)
print "z-scaled ", np.mean(feats_tr, axis=0), np.std(
feats_tr, axis=0), np.min(feats_tr), np.max(feats_tr)
print "Train SVM"
#clf.fit(feats_tr, y_tr)
#clf_prob = CalibratedClassifierCV(base_estimator=clf, cv='prefit')
clf_prob = CalibratedClassifierCV(base_estimator=clf)
clf_prob.fit(feats_tr, y_tr)
minv, maxv = 1e10, -1e10
probs_te = []
for feats in feats_te:
print "feats te i {}".format(feats.shape)
#feats = preproc.transform(feats)
feats = scaler.transform(feats)
minv = min(minv, feats.min())
maxv = max(maxv, feats.max())
probs_te.append(clf_prob.predict_proba(feats).T[1])
print 'probs ->', probs_te[-1]
probs_te = np.array(probs_te)
print "min, max feats fed on svm {}, {}".format(minv, maxv)
return eval.froc(real_blobs_te, pred_blobs_te, probs_te)
def froc_by_epochs(data,
blobs,
augmented_blobs,
rois,
folds,
network_model,
nb_epochs=30,
epoch_interval=2):
network_init = None
roi_size = 32
streams = 'none'
imgs = []
masks = []
for i in range(len(data)):
img, lung_mask = data.get(i, downsample=True)
sampled, lce, norm = preprocess.preprocess_hardie(img,
lung_mask,
downsample=True)
imgs.append([lce])
masks.append(lung_mask)
imgs = np.array(imgs)
masks = np.array(masks)
# Hardcoding blob set shapes
blobs2 = blobs
blobs = blobs.reshape((len(blobs), 3))
nb_checkpoints = int(nb_epochs / epoch_interval)
epochs = np.linspace(epoch_interval, nb_checkpoints * epoch_interval,
nb_checkpoints).astype(np.int)
av_frocs = []
names = []
aucs1 = []
aucs2 = []
for epoch in epochs:
frocs = []
fold = 1
for tr_idx, te_idx in folds:
print "Fold {} ...".format(fold)
X_train, Y_train, X_test, Y_test = neural.create_train_test_sets(
rois[tr_idx],
augmented_blobs[tr_idx],
blobs[tr_idx],
rois[te_idx],
augmented_blobs[te_idx],
blobs[te_idx],
streams=streams,
detector=True)
# load network
network = neural.create_network(network_model,
X_train.shape,
fold,
streams,
detector=False)
name = 'data/{}_fold_{}.epoch_{}'.format(network_model, fold,
epoch)
network.network.load_weights('{}_weights.h5'.format(name))
# open network on detector mode
detector_network = neural.create_network(network_model,
X_train.shape,
fold,
streams,
detector=True)
copy_weights(network, detector_network)
# evaluate network on test
blobs_te_pred, probs_te_pred = detect_with_network(
detector_network, imgs[te_idx], masks[te_idx], fold=fold)
froc = eval.froc(blobs2[te_idx], blobs_te_pred, probs_te_pred)
frocs.append(froc)
fold += 1
names.append('{}, epoch {}'.format(network_model, epoch))
ops = eval.average_froc(frocs, fppi_range)
av_frocs.append(ops)
aucs1.append(util.auc(ops, range(0, 60)))
aucs2.append(util.auc(ops, range(0, 40)))
util.save_auc(
np.array(range(1,
len(aucs1) + 1)) * epoch_interval, aucs1,
'data/{}-auc-0-60'.format(network_model))
util.save_auc(
np.array(range(1,
len(aucs2) + 1)) * epoch_interval, aucs2,
'data/{}-auc-0-40'.format(network_model))
return av_frocs, names
def eval_cnn_detector(data, blobs, augmented_blobs, rois, folds, model):
fold = 1
network_init = None
roi_size = 32
streams = 'none'
imgs = []
masks = []
for i in range(len(data)):
img, lung_mask = data.get(i, downsample=True)
sampled, lce, norm = preprocess.preprocess_hardie(img,
lung_mask,
downsample=True)
imgs.append([lce])
masks.append(lung_mask)
imgs = np.array(imgs)
masks = np.array(masks)
# Hardcoding blob set shapes
blobs2 = blobs
blobs = blobs.reshape((len(blobs), 3))
frocs = []
for tr_idx, te_idx in folds:
print "Fold {} ...".format(fold)
X_train, Y_train, X_test, Y_test = neural.create_train_test_sets(
rois[tr_idx],
augmented_blobs[tr_idx],
blobs[tr_idx],
rois[te_idx],
augmented_blobs[te_idx],
blobs[te_idx],
streams=streams,
detector=True)
network = neural.create_network(model,
X_train.shape,
fold,
streams,
detector=False)
if network_init is not None:
network.network.load_weights('data/{}_fold_{}_weights.h5'.format(
network_init, fold))
# save network
name = 'data/{}_fold_{}'.format(model, fold)
history = network.fit(X_train,
Y_train,
X_test,
Y_test,
streams=(streams != 'none'),
cropped_shape=(roi_size, roi_size),
checkpoint_prefix=name,
checkpoint_interval=2,
loss='mse')
network.save(name)
# open network on detector mode
network.network.summary()
detector_network = neural.create_network(model,
X_train.shape,
fold,
streams,
detector=True)
detector_network.network.summary()
copy_weights(network, detector_network)
#network.network.load_weights('{}_weights.h5'.format(name))
#network.load(name)
# evaluate network on test
blobs_te_pred, probs_te_pred = detect_with_network(detector_network,
imgs[te_idx],
masks[te_idx],
fold=fold)
froc = eval.froc(blobs2[te_idx], blobs_te_pred, probs_te_pred)
frocs.append(froc)
fold += 1
av_froc = eval.average_froc(frocs, fppi_range)
return av_froc