def exp_eval_ots_lidc_jsrt(args): # load LIDC & JSRT-positives data imgs_tr, blobs_tr = lidc.load() pred_blobs_tr = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks_tr = np.load('data/aam-lidc-pred-masks.npy') imgs_te, blobs_te = jsrt.load(set_name='jsrt140p') pred_blobs_te = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks_te = np.load('data/aam-jsrt140p-pred-masks.npy') rois_tr = lnd.create_rois(imgs_tr, masks_tr, pred_blobs_tr, args) rois_te = lnd.create_rois(imgs_te, masks_te, pred_blobs_te, args) # Create rois dataset rois_tr, Y_tr, _, _ = neural.create_train_test_sets( blobs_tr, pred_blobs_tr, rois_tr, None, None, None) generator = augment.get_default_generator((args.roi_size, args.roi_size)) rois_tr, Y_tr = augment.balance_and_perturb(rois_tr, Y_tr, generator) range_tr = np.max(rois_tr), np.min(rois_tr) print "range {}".format(range_tr) # Extract features network = VGG16(mode='ots-feat', pool_layer=args.pool_layer) feats_tr = extract_convfeats(network, rois_tr, range_tr) feats_te = extract_convfeats(network, rois_te, range_tr) np.save('data/{}-lidc-feats.npy'.format(args.detector, fold_idx), feats_tr) np.save('data/{}-jsrt140p-feats.npy'.format(args.detector, fold_idx), feats_te) # Eval classifier clf = LinearSVC(C=args.svm_c) froc = evaluate_classifier(clf, feats_tr, Y_tr, blobs_te, pred_blobs_te, feats_te)
def model_evaluation_tr_lidc_te_jsrt(model_name, args): imgs_tr, blobs_tr = lidc.load() pred_blobs_tr = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks_tr = np.load('data/aam-lidc-pred-masks.npy') imgs_te, blobs_te = jsrt.load(set_name='jsrt140p') pred_blobs_te = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks_te = np.load('data/aam-jsrt140p-pred-masks.npy') rois_tr = create_rois(imgs_tr, masks_tr, pred_blobs_tr, args) rois_te = create_rois(imgs_te, masks_te, pred_blobs_te, args) model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model.name += '-{}-lidc'.format(args.detector) froc = evaluate_model(model, blobs_tr, pred_blobs_tr, rois_tr, blobs_te, pred_blobs_te, rois_te) froc = eval.average_froc([froc]) legends = ['Test FROC (JSRT positives)'] util.save_froc([froc], 'data/{}-{}-lidc-jsrt-froc'.format(model.name, args.detector), legends, with_std=False)
def model_selection_with_convfeats(args): # Load img, blobs and masks imgs, blobs, paths = lidc.load(pts=True, set_name=args.ds_tr) pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-lidc-pred-masks.npy') assert len(imgs) == len(masks) and len(pred_blobs) == len(masks) # Load folds folds = util.model_selection_folds(imgs) # Create rois #rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs) # Set up CV frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)] fold_idx = 0 import time from sklearn.neighbors import KNeighborsClassifier for tr, te in folds: print "Load features fold {}".format(fold_idx) start = time.time() feats_tr, Y_tr = load_features('data/{}-f{}-lidc-feats'.format( args.detector, fold_idx)) print 'tr time {}'.format(time.time() - start) start = time.time() feats_te = np.load('data/{}-f{}-te-lidc-feats.npy'.format( args.detector, fold_idx)) print 'te time {}'.format(time.time() - start) print "-> tr {}, {}, te {}".format(feats_tr.shape, Y_tr.shape, feats_te.shape) # Train/test model print "Evaluate clf" #clf = KNeighborsClassifier(n_neighbors=3) clf = LinearSVC(C=args.svm_C) froc = evaluate_classifier(clf, feats_tr, Y_tr, blobs[te], pred_blobs[te], feats_te) frocs.append(froc) # Record model results current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs] util.save_froc(current_frocs, 'data/lsvm-z-C{}-{}-folds-froc'.format( args.svm_C, args.detector), legends[:len(frocs)], with_std=False) fold_idx += 1 legends = ['Val FROC (LIDC-IDRI)'] average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101)) util.save_froc([average_froc], 'data/lsvm-z-C{}-{}-val-froc'.format( args.svm_C, args.detector), legends, with_std=True)
def visual_results_jsrt_only(model_name, args): print "Visual results for model {} JSRT only".format(model_name) imgs, blobs = jsrt.load(set_name='jsrt140p') pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-jsrt140p-pred-masks.npy') rois = create_rois(imgs, masks, pred_blobs, args) folds = KFold(n_splits=5, shuffle=True, random_state=util.FOLDS_SEED).split(imgs) fold_idx = 0 for tr, te in folds: model.load('data/' + model.name + '.fold-{}'.format(fold_idx + 1)) model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) X_tr, Y_tr, X_te, Y_te = neural.create_train_test_sets( real_blobs_tr, pred_blobs_tr, rois_tr, real_blobs_te, pred_blobs_te, rois_te) print 'load weights {}'.format(model.name) model.network.load_weights('data/{}_weights.h5'.format(model.name)) # FIX: remove and add zmuv mean and zmuv std no Preprocessor augment.py if not hasattr(model.preprocessor, 'zmuv_mean'): model.preprocessor.fit(X_tr, Y_tr) model.save('data/' + model.name) pred_blobs_te, probs_te = neural.predict_proba(model, pred_blobs_te, rois_te) util.save_rois_with_probs(rois_te, probs_te) fold_idx += 1
def model_output(model_name, args): print "Model Outputs" imgs, blobs = jsrt.load(set_name='jsrt140p') pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-jsrt140p-pred-masks.npy') rois = create_rois(imgs, masks, pred_blobs, args) folds = KFold(n_splits=5, shuffle=True, random_state=util.FOLDS_SEED).split(imgs) fold_idx = 0 frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(5)] index = np.array(range(len(imgs))) for tr, te in folds: X_tr, Y_tr, _, _ = neural.create_train_test_sets( blobs[tr], pred_blobs[tr], rois[tr], blobs[te], pred_blobs[te], rois[te]) model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model.name = model.name + '-{}-lidc.fold-{}'.format( args.detector, fold_idx + 1) model.network.load_weights('data/{}_weights.h5'.format(model.name)) if not hasattr(model.preprocessor, 'zmuv_mean'): model.preprocessor.fit(X_tr, Y_tr) print "Predict ..." pred_blobs_te, probs_te, rois_te = neural.predict_proba( model, pred_blobs[te], rois[te]) print "Save ..." eval.save_outputs(imgs[te], blobs[te], pred_blobs_te, probs_te, rois_te, index[te])
def model_selection(model_name, args): # Load img, blobs and masks imgs, blobs, paths = lidc.load(pts=True, set_name=args.ds_tr) if args.ds_tr != args.ds_val: _, blobs_val, _ = lidc.load(pts=True, set_name=args.ds_val) else: blobs_val = blobs pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-lidc-pred-masks.npy') assert len(imgs) == len(masks) and len(pred_blobs) == len(masks) # Load folds folds = util.model_selection_folds(imgs) # Create rois rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs) rois_val = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs_val) # Set up CV frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)] fold_idx = 0 for tr, te in folds: # Load and setup model model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model.network.summary() model.name = model.name + '.fold-{}'.format(fold_idx + 1) if args.load_model: print "Loading model: data/{}".format(model.name) model.load('data/' + model.name) # Train/test model froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr], blobs_val[te], pred_blobs[te], rois_val[te], args.load_model) frocs.append(froc) # Record model results current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs] util.save_froc(current_frocs, 'data/{}-{}-folds-froc'.format(model.name[:-7], args.detector), legends[:len(frocs)], with_std=False) model.save('data/' + model.name) fold_idx += 1 legends = ['Val FROC (LIDC-IDRI)'] average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101)) util.save_froc([average_froc], 'data/{}-{}-val-froc'.format(model.name[:-7], args.detector), legends, with_std=True)
def save_rois(args): imgs_tr, blobs_tr = lidc.load(pts=False) pred_blobs_tr = detect.read_blobs('data/sbf-aam-lidc-pred-blobs.pkl') masks_tr = np.load('data/aam-lidc-pred-masks.npy') imgs_te, blobs_te = jsrt.load(set_name='jsrt140p') pred_blobs_te = detect.read_blobs('data/sbf-aam-jsrt140p-pred-blobs.pkl') masks_te = np.load('data/aam-jsrt140p-pred-masks.npy') rois_tr = create_rois(imgs_tr, masks_tr, pred_blobs_tr, args, real_blobs=blobs_tr) rois_te = create_rois(imgs_te, masks_te, pred_blobs_te, args, real_blobs=blobs_te) X_tr, Y_tr, X_te, Y_te = neural.create_train_test_sets( blobs_tr, pred_blobs_tr, rois_tr, blobs_te, pred_blobs_te, rois_te) X_tr, Y_tr = util.split_data_pos_neg(X_tr, Y_tr) X_te, Y_te = util.split_data_pos_neg(X_te, Y_te) X_pos = X_tr[0] idx = np.random.randint(0, len(X_tr[1]), len(X_pos)) X_neg = X_tr[1][idx] print len(X_pos), len(X_neg) for i in range(len(X_pos)): util.imwrite('data/lidc/roi{}p.jpg'.format(i), X_pos[i][0]) np.save('data/lidc/roi{}p.npy'.format(i), X_pos[i]) util.imwrite('data/lidc/roi{}n.jpg'.format(i), X_neg[i][0]) np.save('data/lidc/roi{}n.npy'.format(i), X_neg[i]) X_pos = X_te[0] idx = np.random.randint(0, len(X_te[1]), len(X_pos)) X_neg = X_te[1][idx] print len(X_pos), len(X_neg) for i in range(len(X_pos)): util.imwrite('data/jsrt140/roi{}p.jpg'.format(i), X_pos[i][0]) np.save('data/jsrt140/roi{}p.npy'.format(i), X_pos[i]) util.imwrite('data/jsrt140/roi{}n.jpg'.format(i), X_neg[i][0]) np.save('data/jsrt140/roi{}n.npy'.format(i), X_neg[i])
def eval_trained_model(model_name, args): imgs, blobs = lidc.load() pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-lidc-pred-masks.npy') folds = util.model_selection_folds(imgs) rois = create_rois(imgs, masks, pred_blobs, args) save_performance_history(model_name, args, rois, blobs, pred_blobs, folds)
def test_show_blobs(): import util import detect import preprocess import matplotlib.pyplot as plt set_name = 'jsrt140' imgs, blobs = load(set_name=set_name) pred_blobs = detect.read_blobs('data/wmci-aam-jsrt140-blobs-gt.pkl') for i in range(len(imgs)): max_intensity = np.max(imgs[i]) img = util.label_blob(imgs[i].astype(np.float32), list(blobs[i][0]), color=(max_intensity, 0, 0)) plt.hist(np.array(img).ravel(), 256, range=(0, 4096)) plt.show()
def exp_eval_ots_jsrt_only(args): # load LIDC & JSRT-positives data network = VGG16(mode='ots-feat', pool_layer=args.pool_layer) print "Model Evaluation Protocol 2" imgs, blobs = jsrt.load(set_name='jsrt140p') pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-jsrt140p-pred-masks.npy') rois = lnd.create_rois(imgs, masks, pred_blobs, args) folds = KFold(n_splits=5, shuffle=True, random_state=util.FOLDS_SEED).split(imgs) fold_idx = 0 frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(5)] for tr, te in folds: # Eval classifier rois_tr, Y_tr, _, _ = neural.create_train_test_sets( blobs_tr, pred_blobs_tr, rois[tr], None, None, None) generator = augment.get_default_generator( (args.roi_size, args.roi_size)) rois_tr, Y_tr = augment.balance_and_perturb(rois_tr, Y_tr, generator) clf = LinearSVC(C=args.svm_c) froc = evaluate_classifier(clf, rois_tr, Y_tr, blobs[te], pred_blobs[te], feats[te]) frocs.append(froc) current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs] util.save_froc(current_frocs, 'data/lsvm-{}-jsrtonly-folds'.format(args.detector), legends[:len(frocs)], with_std=False) fold_idx += 1 froc = eval.average_froc(frocs) legends = ['Test FROC (JSRT positives)'] util.save_froc([froc], 'data/lsvm-{}-jsrtonly'.format(args.detector), legends, with_std=True)
def model_evaluation_jsrt_only(model_name, args): print "Model Evaluation Protocol 2" imgs, blobs = jsrt.load(set_name='jsrt140p') pred_blobs = detect.read_blobs('data/{}-jsrt140p-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-jsrt140p-pred-masks.npy') rois = create_rois(imgs, masks, pred_blobs, args) folds = KFold(n_splits=5, shuffle=True, random_state=util.FOLDS_SEED).split(imgs) fold_idx = 0 frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(5)] for tr, te in folds: model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model.name = model.name + '-{}-lidc.fold-{}'.format( args.detector, fold_idx + 1) froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr], blobs[te], pred_blobs[te], rois[te]) frocs.append(froc) current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs] util.save_froc(current_frocs, 'data/{}-{}-only-jsrt-folds'.format( model.name[:-7], args.detector), legends[:len(frocs)], with_std=False) model.save('data/' + model.name) fold_idx += 1 froc = eval.average_froc(frocs) legends = ['Test FROC (JSRT positives)'] util.save_froc([froc], 'data/{}-{}-only-jsrt'.format(model.name[:-7], args.detector), legends, with_std=True)
def model_selection_unsup(model_name, args): imgs, blobs, paths = lidc.load(pts=True) pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-lidc-pred-masks.npy') assert len(imgs) == len(masks) and len(pred_blobs) == len(masks) folds = util.model_selection_folds(imgs) rois = create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs) frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)] fold_idx = 0 for tr, te in folds: model = neural.create_network(model_name, args, (1, args.roi_size, args.roi_size)) model.name = model.name + '.fold-{}'.format(fold_idx + 1) froc = evaluate_model(model, blobs[tr], pred_blobs[tr], rois[tr], blobs[te], pred_blobs[te], rois[te]) frocs.append(froc) current_frocs = [eval.average_froc([froc_i]) for froc_i in frocs] util.save_froc(current_frocs, 'data/{}-{}-folds-froc'.format(model_name, args.detector), legends[:len(frocs)], with_std=False) model.save('data/' + model.name) fold_idx += 1 legends = ['Val FROC (LIDC-IDRI)'] average_froc = eval.average_froc(frocs, np.linspace(0.0, 10.0, 101)) util.save_froc([average_froc], 'data/{}-{}-val-froc'.format(model_name, args.detector), legends, with_std=True)
def extract_features_from_convnet(args): # Load img, blobs and masks imgs, blobs, paths = lidc.load(pts=True, set_name=args.ds_tr) pred_blobs = detect.read_blobs('data/{}-lidc-pred-blobs.pkl'.format( args.detector)) masks = np.load('data/aam-lidc-pred-masks.npy') assert len(imgs) == len(masks) and len(pred_blobs) == len(masks) # Load folds folds = util.model_selection_folds(imgs) # Create rois rois = lnd.create_rois(imgs, masks, pred_blobs, args, real_blobs=blobs) # Load model network = VGG16(mode='ots-feat', pool_layer=args.pool_layer) network.summary() # Set up CV frocs = [] legends = ['Fold {}'.format(i + 1) for i in range(util.NUM_VAL_FOLDS)] fold_idx = 0 for tr, te in folds: # TODO: apply extract convfeats funcs for tr and te sets print "Fold {}".format(fold_idx + 1) X_tr, Y_tr, _, _ = neural.create_train_test_sets( blobs[tr], pred_blobs[tr], rois[tr], blobs[te], pred_blobs[te], rois[te]) gc.collect() generator = augment.get_default_generator( (args.roi_size, args.roi_size)) X_tr, Y_tr = augment.balance_and_perturb(X_tr, Y_tr, generator) gc.collect() '''' counta = 0 countb = 0 count = 0 while counta < 10 and countb < 10: if Y_tr[count][1] > 0 and counta < 10: util.imshow("positives", X_tr[count][0], display_shape=(256, 256)) counta += 1 elif Y_tr[count][1] == 0.0 and countb < 10: util.imshow("negatives", X_tr[count][0], display_shape=(256, 256)) countb += 1 count += 1 ''' range_tr = (X_tr.min(), X_tr.max()) print "Range {}".format(range_tr) print "Extract feats on balanced tr set" feats_tr = extract_convfeats(network, X_tr, range_tr) save_features( "data/vgg16-{}-{}-f{}-lidc-feats".format(args.pool_layer, args.detector, fold_idx), feats_tr, Y_tr) gc.collect() print "Extract feats on te set" feats_te = extract_convfeats_from_rois(network, rois[te], range_tr) print "Test feats to save shape {}".format(feats_te.shape) np.save( "data/vgg16-{}-{}-f{}-te-lidc-feats.npy".format( args.pool_layer, args.detector, fold_idx), feats_te) gc.collect() fold_idx += 1