def model_selection_with_convfeats(args):
# Load img, blobs and masks
imgs, blobs, paths = lidc.load(pts=True, set_name=args.ds_tr)
pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format(
args.detector))
masks = np.load('data/aam-lidc-pred-masks.npy')
assert len(imgs) == len(masks) and len(pred_blobs) == len(masks)
# Load folds
folds = util.model_selection_folds(imgs)
# Create rois
#rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs)
# Set up CV
frocs = []
legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)]
fold_idx = 0
import time
from sklearn.neighbors import KNeighborsClassifier
for tr, te in folds:
print "Load features fold {}".format(fold_idx)
start = time.time()
feats_tr, Y_tr = load_features('data/{}-f{}-lidc-feats'.format(
args.detector, fold_idx))
print 'tr time {}'.format(time.time() - start)
start = time.time()
feats_te = np.load('data/{}-f{}-te-lidc-feats.npy'.format(
args.detector, fold_idx))
print 'te time {}'.format(time.time() - start)
print "-> tr {}, {}, te {}".format(feats_tr.shape, Y_tr.shape,
feats_te.shape)
# Train/test model
print "Evaluate clf"
#clf = KNeighborsClassifier(n_neighbors=3)
clf = LinearSVC(C=args.svm_C)
froc = evaluate_classifier(clf, feats_tr, Y_tr, blobs[te],
pred_blobs[te], feats_te)
frocs.append(froc)
# Record model results
current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs]
util.save_froc(current_frocs,
'data/lsvm-z-C{}-{}-folds-froc'.format(
args.svm_C, args.detector),
legends[:len(frocs)],
with_std=False)
fold_idx += 1
legends = ['Val FROC (LIDC-IDRI)']
average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101))
util.save_froc([average_froc],
'data/lsvm-z-C{}-{}-val-froc'.format(
args.svm_C, args.detector),
legends,
with_std=True)
def model_evaluation_tr_lidc_te_jsrt(model_name, args):
imgs_tr, blobs_tr = lidc.load()
pred_blobs_tr = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format(
args.detector))
masks_tr = np.load('data/aam-lidc-pred-masks.npy')
imgs_te, blobs_te = jsrt.load(set_name='jsrt140p')
pred_blobs_te = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format(
args.detector))
masks_te = np.load('data/aam-jsrt140p-pred-masks.npy')
rois_tr = create_rois(imgs_tr, masks_tr, pred_blobs_tr, args)
rois_te = create_rois(imgs_te, masks_te, pred_blobs_te, args)
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model.name += '-{}-lidc'.format(args.detector)
froc = evaluate_model(model, blobs_tr, pred_blobs_tr, rois_tr, blobs_te,
pred_blobs_te, rois_te)
froc = eval.average_froc([froc])
legends = ['Test FROC (JSRT positives)']
util.save_froc([froc],
'data/{}-{}-lidc-jsrt-froc'.format(model.name,
args.detector),
legends,
with_std=False)
def model_selection(model_name, args):
# Load img, blobs and masks
imgs, blobs, paths = lidc.load(pts=True, set_name=args.ds_tr)
if args.ds_tr != args.ds_val:
_, blobs_val, _ = lidc.load(pts=True, set_name=args.ds_val)
else:
blobs_val = blobs
pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format(
args.detector))
masks = np.load('data/aam-lidc-pred-masks.npy')
assert len(imgs) == len(masks) and len(pred_blobs) == len(masks)
# Load folds
folds = util.model_selection_folds(imgs)
# Create rois
rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs)
rois_val = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs_val)
# Set up CV
frocs = []
legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)]
fold_idx = 0
for tr, te in folds:
# Load and setup model
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model.network.summary()
model.name = model.name + '.fold-{}'.format(fold_idx + 1)
if args.load_model:
print "Loading model: data/{}".format(model.name)
model.load('data/' + model.name)
# Train/test model
froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr],
blobs_val[te], pred_blobs[te], rois_val[te],
args.load_model)
frocs.append(froc)
# Record model results
current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs]
util.save_froc(current_frocs,
'data/{}-{}-folds-froc'.format(model.name[:-7],
args.detector),
legends[:len(frocs)],
with_std=False)
model.save('data/' + model.name)
fold_idx += 1
legends = ['Val FROC (LIDC-IDRI)']
average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101))
util.save_froc([average_froc],
'data/{}-{}-val-froc'.format(model.name[:-7],
args.detector),
legends,
with_std=True)
def save_performance_history(model_name, args, rois, blobs, pred_blobs, folds):
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model_name = model.name
epochs = model.training_params['nb_epoch']
frocs = []
legends = []
fold_idx = 0
for tr, te in folds:
model.load('data/' + model_name + '.fold-{}'.format(fold_idx + 1))
frocs.append([])
epochs_set = list(range(1, epochs + 1, 2))
for epoch in epochs_set:
weights_file_name = 'data/{}.weights.{:02d}.hdf5'.format(
model.name, epoch)
model.network.load_weights(weights_file_name)
pred_blobs_te, probs_te = neural.predict_proba(
model, pred_blobs[te], rois[te])
frocs[fold_idx].append(
eval.froc(blobs[te], pred_blobs_te, probs_te))
fold_idx += 1
frocs = np.array(frocs)
froc_history = []
aucs_history = []
legends = []
i = 0
print "check -> frocs.shape {}".format(frocs.shape)
for epoch in range(1, epochs + 1, 2):
frocs_by_epoch = frocs[:, i]
froc_history.append(
eval.average_froc(np.array(frocs_by_epoch),
np.linspace(0.0, 10.0, 101)))
aucs_history.append([])
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.2, 4.0, 101))**2)
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.0, 5.0, 101))**2)
aucs_history[-1].append(
util.auc(froc_history[-1], np.linspace(0.0, 10.0, 101))**2)
legends.append('Val FROC (LIDC-IDRI), epoch {}'.format(epoch))
i += 1
util.save_froc(froc_history,
'data/{}-val-froc-by-epoch'.format(model_name),
legends,
with_std=False)
util.save_aucs(list(range(1, epochs + 1, 2)), aucs_history,
'data/{}-val-aucs'.format(model_name),
['AUC between 2-4', 'AUC between 0-5', 'AUC between 0-10'])
def save_blobs(method, args):
images, blobs = lidc.load()
masks = np.load('data/aam-lidc-pred-masks.npy')
print 'masks shape {}'.format(masks.shape)
pred_blobs, probs = detect_blobs(images,
masks,
args.threshold,
real_blobs=blobs,
method=method)
write_blobs(
pred_blobs,
'data/{}-{}-aam-lidc-pred-blobs.pkl'.format(method, args.threshold))
froc = eval.froc(blobs, pred_blobs, probs, distance='rad')
froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
abpi = average_bppi(pred_blobs)
util.save_froc([froc],
'data/{}-{}-lidc-blobs-froc'.format(method, args.threshold),
['{} FROC on LIDC dataset'.format(method)],
fppi_max=100)
np.savetxt('data/{}-{}-lidc-blobs-abpi.txt'.format(method, args.threshold),
np.array([abpi]))
print('Average blobs per image LIDC {}'.format(abpi))
images, blobs = jsrt.load(set_name='jsrt140p')
masks = np.load('data/aam-jsrt140p-pred-masks.npy')
pred_blobs, probs = detect_blobs(images,
masks,
args.threshold,
real_blobs=blobs,
method=method)
write_blobs(
pred_blobs, 'data/{}-{}-aam-jsrt140p-pred-blobs.pkl'.format(
method, args.threshold))
froc = eval.froc(blobs, pred_blobs, probs, distance='rad')
froc = eval.average_froc([froc], DETECTOR_FPPI_RANGE)
abpi = average_bppi(pred_blobs)
util.save_froc([froc],
'data/{}-{}-jsrt140p-blobs-froc'.format(
method, args.threshold),
['{} FROC on JSRT140 positives dataset'.format(method)],
fppi_max=100)
np.savetxt(
'data/{}-{}-jsrt140p-blobs-abpi.txt'.format(method, args.threshold),
np.array([abpi]))
print('Average blobs per image JSRT positives {}'.format(abpi))
def eval_models(model_instance, network_set, save_history=True):
paths, locs, rads, subs, sizes, kinds = jsrt.jsrt(set='jsrt140')
left_masks = jsrt.left_lung(set='jsrt140')
right_masks = jsrt.right_lung(set='jsrt140')
data = DataProvider(paths, left_masks, right_masks)
size = len(paths)
blobs = []
for i in range(size):
blobs.append([[locs[i][0], locs[i][1], rads[i]]])
blobs = np.array(blobs)
methods = [] #['wmci']
thresholds = [0.5]
frocs = []
legend = []
folds = StratifiedKFold(subs,
n_folds=10,
shuffle=True,
random_state=FOLDS_SEED)
for i in range(len(methods)):
pred_blobs, proba = detect_blobs_with_dataprovider(
data, methods[i], thresholds[i])
frocs.append(util.froc_folds(blobs, pred_blobs, proba, folds))
legend.append('{}, t={}'.format(methods[i], thresholds[i]))
augmented_blobs = add_random_blobs(data,
blobs,
blobs_by_image=512,
rng=np.random)
rois = model_instance.create_rois(data, augmented_blobs,
model_instance.downsample)
#train_cnn_detector(data, blobs, augmented_blobs, rois, folds)
for network_name in network_set:
frocs.append(
eval_cnn_detector(data,
blobs,
augmented_blobs,
rois,
folds,
model=network_name))
legend.append(network_name)
util.save_froc(frocs,
'data/cmp-detectors-10-epochs',
legend,
with_std=False,
fppi_max=fppi_range[-1])
def exp_eval_ots_jsrt_only(args):
# load LIDC & JSRT-positives data
network = VGG16(mode='ots-feat', pool_layer=args.pool_layer)
print "Model Evaluation Protocol 2"
imgs, blobs = jsrt.load(set_name='jsrt140p')
pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format(
args.detector))
masks = np.load('data/aam-jsrt140p-pred-masks.npy')
rois = lnd.create_rois(imgs, masks, pred_blobs, args)
folds = KFold(n_splits=5, shuffle=True,
random_state=util.FOLDS_SEED).split(imgs)
fold_idx = 0
frocs = []
legends = ['Fold {}'.format(i + 1) for i in range(5)]
for tr, te in folds:
# Eval classifier
rois_tr, Y_tr, _, _ = neural.create_train_test_sets(
blobs_tr, pred_blobs_tr, rois[tr], None, None, None)
generator = augment.get_default_generator(
(args.roi_size, args.roi_size))
rois_tr, Y_tr = augment.balance_and_perturb(rois_tr, Y_tr, generator)
clf = LinearSVC(C=args.svm_c)
froc = evaluate_classifier(clf, rois_tr, Y_tr, blobs[te],
pred_blobs[te], feats[te])
frocs.append(froc)
current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs]
util.save_froc(current_frocs,
'data/lsvm-{}-jsrtonly-folds'.format(args.detector),
legends[:len(frocs)],
with_std=False)
fold_idx += 1
froc = eval.average_froc(frocs)
legends = ['Test FROC (JSRT positives)']
util.save_froc([froc],
'data/lsvm-{}-jsrtonly'.format(args.detector),
legends,
with_std=True)
def model_evaluation_jsrt_only(model_name, args):
print "Model Evaluation Protocol 2"
imgs, blobs = jsrt.load(set_name='jsrt140p')
pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format(
args.detector))
masks = np.load('data/aam-jsrt140p-pred-masks.npy')
rois = create_rois(imgs, masks, pred_blobs, args)
folds = KFold(n_splits=5, shuffle=True,
random_state=util.FOLDS_SEED).split(imgs)
fold_idx = 0
frocs = []
legends = ['Fold {}'.format(i + 1) for i in range(5)]
for tr, te in folds:
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model.name = model.name + '-{}-lidc.fold-{}'.format(
args.detector, fold_idx + 1)
froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr],
blobs[te], pred_blobs[te], rois[te])
frocs.append(froc)
current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs]
util.save_froc(current_frocs,
'data/{}-{}-only-jsrt-folds'.format(
model.name[:-7], args.detector),
legends[:len(frocs)],
with_std=False)
model.save('data/' + model.name)
fold_idx += 1
froc = eval.average_froc(frocs)
legends = ['Test FROC (JSRT positives)']
util.save_froc([froc],
'data/{}-{}-only-jsrt'.format(model.name[:-7],
args.detector),
legends,
with_std=True)
def model_selection_unsup(model_name, args):
imgs, blobs, paths = lidc.load(pts=True)
pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format(
args.detector))
masks = np.load('data/aam-lidc-pred-masks.npy')
assert len(imgs) == len(masks) and len(pred_blobs) == len(masks)
folds = util.model_selection_folds(imgs)
rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs)
frocs = []
legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)]
fold_idx = 0
for tr, te in folds:
model = neural.create_network(model_name, args,
(1, args.roi_size, args.roi_size))
model.name = model.name + '.fold-{}'.format(fold_idx + 1)
froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr],
blobs[te], pred_blobs[te], rois[te])
frocs.append(froc)
current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs]
util.save_froc(current_frocs,
'data/{}-{}-folds-froc'.format(model_name,
args.detector),
legends[:len(frocs)],
with_std=False)
model.save('data/' + model.name)
fold_idx += 1
legends = ['Val FROC (LIDC-IDRI)']
average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101))
util.save_froc([average_froc],
'data/{}-{}-val-froc'.format(model_name, args.detector),
legends,
with_std=True)