def prediction_func(CLASS, envifile, maskfile=''): print("Validating Stain...") plt.ion() E = envi.envi(envifile) if not maskfile == "": E.close() # close the ENVI file mask = scipy.misc.imread(maskfile, flatten=True) print(numpy.count_nonzero(mask)) E = envi.envi(envifile, mask=mask) batch_size = 10000 Fv = E.loadbatch(batch_size) # load the first batch n = 0 while not Fv == []: # loop until an empty batch is returned if n == 0: Tv = CLASS.predict(Fv.transpose()).transpose() else: Tv = numpy.append( Tv, CLASS.predict(Fv.transpose()).transpose(), 1 ) # append the predicted labels from this batch to those of previous batches COLORS = hyperspectral.unsift2( Tv, E.batchmask()) # convert the matrix of class labels to a 2D array RGB = numpy.rollaxis(COLORS, 0, 3).astype(numpy.ubyte) plt.imshow(RGB) # display it plt.pause(0.05) Fv = E.loadbatch(batch_size) # load the next batch n = n + 1 return RGB
def load_test(data_file_names): # load test data masks_path = '/media/stim-scratch/berisha/00-annotations/hd/right/com-right/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for testing C = C.astype(np.uint32) bool_mask = np.sum(C.astype(np.uint32), 0) # get number of classes num_classes = C.shape[0] for i in range(1, num_classes): C[i, :, :] *= i + 1 total_mask = np.sum(C.astype(np.uint32), 0) # validation mask Etest = envi.envi(data_file_names[0], mask=total_mask) N = np.count_nonzero(total_mask) # set the batch size Tv = [] # initialize the target array to empty x_test = Etest.loadbatch(N) y_test = total_mask.flat[np.flatnonzero( total_mask)] # get the indices of valid pixels Etest.close() return x_test, y_test
def load_data(envifile, stainfile, trainmask="", normalize = True): if trainmask == "": E = envi.envi(filename=envifile) else: mask = scipy.misc.imread(trainmask, flatten=True) E = envi.envi(envifile, mask=mask) F= E.loadall() # return a matrix from envi file #bands * w * h F = np.rollaxis(F, 0, 3) # read the stained file stain = scipy.misc.imread(stainfile) if normalize: F, stain = norm(F, stain, mode='standard') else: stain = stain/255. return F , stain
def subsample(envi_file, mask_file, n): E = envi.envi(envi_file) D = E.loadall() Ds = D[:, ::n, ::n] Ds_sift = hyperspectral.sift2(Ds) Ds_flatten = Ds_sift.flatten() #mask = scipy.misc.imread(mask_file, flatten=True) #D_subsample = E.loadmask(mask).transpose() #D_flatten = D_subsample.flatten() out_name = envi_file + '_sub' outfile = open(out_name, 'wb') Ds_flatten.tofile(outfile) outfile.close() return [Ds.shape[0], Ds.shape[1], Ds.shape[2]]
def feed_forward_net(envifile, stainfile, trainmask="", n_epoch=50, N=5000): if trainmask == "": E = envi.envi(envifile) else: mask = scipy.misc.imread(trainmask, flatten=True) E = envi.envi(envifile, mask=mask) mask = classify.random_mask(E.mask, N) scipy.misc.imsave("random.bmp", mask) Ft = E.loadmask(mask).transpose() stain = numpy.rollaxis(scipy.misc.imread(stainfile), 2) #plt.figure() #plt.imshow(stain) Tt = hyperspectral.sift2(stain, mask).transpose( ) # sift a 2D hyperspectral image into a PxB matrix where P is the number of pixels and B is the number of bands print("Training MLPRegressor...") net = tflearn.input_data(shape=[None, Ft.shape[1]]) net = tflearn.fully_connected(net, 7, activation='relu') #, net = tflearn.fully_connected(net, 3, activation='relu') #, net = tflearn.regression(net, batch_size=128, loss='mean_square', metric='R2', learning_rate=0.01) #learning_rate=0.01 CLASS = tflearn.DNN(net, tensorboard_dir='.\log', tensorboard_verbose=2) CLASS.fit(Ft, Tt, n_epoch=n_epoch, show_metric=True, snapshot_step=500, validation_set=0.2) #if validate == False: return CLASS
def load_train(data_file_names): # load train data masks_path = '/media/stim-scratch/berisha/00-annotations/hd/left/com-left/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for training # open the ENVI file for reading, use the validation image for batch access Etrain = envi.envi(data_file_names[0]) # x_train, y_train = Etrain.loadtrain(C) x_train, y_train = Etrain.loadtrain_balance(C, num_samples=100000) Etrain.close() return x_train, y_train
def box_filter(envi_file, n): E = envi.envi(envi_file) D = E.loadall() if E.header.interleave is not 'bsq': print("ERROR: filtering is just implemented for BSQ!") # D_filter = scipy.ndimage.uniform_filter(D, size=[5, 5, 1], mode='constant', cval=0.0) k = numpy.ones([1, n, n], dtype='float32') * 1.0 / (float(n) * float(n) ) # definne box filter D_filter = scipy.ndimage.filters.convolve( D, k, mode='reflect') # convolve the hyperspectral image with box filter D_filter_sift = hyperspectral.sift2( D_filter) # sift a 2D hyperspectral image into a PxB matrix D_filter_1D = D_filter_sift.flatten() # flattenn the matrix to an array envi_file_blur = envi_file + "_blured_bsq" # name the output outfile = open(envi_file_blur, 'wb') # open a binary file to write the array D_filter_1D.tofile(outfile) outfile.close() # create a header file- shutil.copy2(envi_file + '.hdr', envi_file_blur + '.hdr') return envi_file_blur
import classify import matplotlib.pyplot as plt from envi import envi import scipy.misc prob_file = 'cnn-load-batch-test-response' output_file = 'cnn-load-batch-test.png' prob_envi = envi(prob_file) prob_image = prob_envi.loadall() class_image = classify.prob2class(prob_image) rgb = classify.class2color(class_image) scipy.misc.imsave(output_file, rgb)
A comparison of several classifiers using ('optimal' params) from scikit-learn on FTIR data. Code source: Sebastian Berisha """ # load train data data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for training # open the ENVI file for reading, use the validation image for batch access Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16') #x_train, y_train = Etrain.loadtrain(C) x_train, y_train = Etrain.loadtrain_balance(C, num_samples=60000) Etrain.close() # load test data data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/brc961-proj/brc961-proj/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/brc961/masks/no-mnf-bcemn/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for testing C = C.astype(np.uint32) bool_mask = np.sum(C.astype(np.uint32), 0)
from sklearn.ensemble import RandomForestClassifier import classify import glob import envi # load train data data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for training # open the ENVI file for reading, use the validation image for batch access Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16') x_train, y_train = Etrain.loadtrain(C) #x_train, y_train = Etrain.loadtrain_balance(C, num_samples=60000) Etrain.close() # build a classifier clf = RandomForestClassifier(n_estimators=20) # Utility function to report best scores def report(results, n_top=3): for i in range(1, n_top + 1): candidates = np.flatnonzero(results['rank_test_score'] == i) for candidate in candidates: print("Model with rank: {0}".format(i)) print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
# Load the envi file names. data_file_names = [] compiled = re.compile("\.") for f in os.listdir(data_path): if not f.startswith('.'): abs_path = path.join(data_path, f) file_name = re.split(compiled, abs_path)[0] if path.isfile(abs_path) and not (file_name in data_file_names): data_file_names.append(file_name) if len(data_file_names) < 1: raise ValueError('Missing data files in folder %s!' % data_path) elif len(data_file_names) > 1: warnings.warn("More than one ENVI file in data folder") Etrain = envi.envi(data_file_names[0]) #x_train, y_train = Etrain.loadtrain(C) x_train, y_train = Etrain.loadtrain_balance(C, num_samples=100000) Etrain.close() # load test data data_path = '/media/stim-processed/berisha/breast-processing/hd/brc961/pca/pca16/' masks_path = '/media/stim-scratch/berisha/00-annotations/hd/right/com-right/' classimages = sorted(glob.glob(masks_path + '*.png')) # load the class file names C = classify.filenames2class( classimages) # generate the class images for testing C = C.astype(np.uint32) bool_mask = np.sum(C.astype(np.uint32), 0)
import sklearn.neural_network import glob from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score from sklearn.metrics import confusion_matrix import classify data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/' classimages = sorted(glob.glob(masks_path + '*.png')) #load the class file names C = classify.filenames2class(classimages) #generate the class images for training Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16') #open the ENVI file for reading, use the validation image for batch access Ftrain, Ttrain = Etrain.loadtrain(C) #Ftrain, Ttrain = Etrain.load_train_balance(C, 60000) #load the training set rf_model = RandomForestClassifier(n_estimators=100, max_depth=10) rf_model.fit(Ftrain, Ttrain) #CLASS1 = sklearn.naive_bayes.GaussianNB() #train a Bayesian classifier #CLASS1.fit(Ft1, Tt1) Etrain.close() ###########################################testing############################################### data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/brc961-proj/brc961-proj/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/brc961/masks/no-mnf-bcemn/'
# Load the envi file names. data_file_names = [] compiled = re.compile("\.") for f in os.listdir(data_path): if not f.startswith('.'): abs_path = path.join(data_path, f) file_name = re.split(compiled, abs_path)[0] if path.isfile(abs_path) and not (file_name in data_file_names): data_file_names.append(file_name) if len(data_file_names) < 1: raise ValueError('Missing data files in folder %s!' % data_path) elif len(data_file_names) > 1: warnings.warn("More than one ENVI file in data folder") Etrain = envi.envi(data_file_names[0]) x_train, y_train = Etrain.loadtrain_balance(C, num_samples=10) Etrain.close() # ############################################################################# # Train classifiers # # For an initial search, a logarithmic grid with basis # 10 is often helpful. Using a basis of 2, a finer # tuning can be achieved but at a much higher cost. C_range = np.logspace(-3, 3, 7) gamma_range = np.logspace(-3, 3, 7) param_grid = dict(gamma=gamma_range, C=C_range) cv = StratifiedShuffleSplit(n_splits=5, test_size=0.25, random_state=42) grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv, n_jobs=16)