def prediction_func(CLASS, envifile, maskfile=''):
print("Validating Stain...")
plt.ion()
E = envi.envi(envifile)
if not maskfile == "":
E.close() # close the ENVI file
mask = scipy.misc.imread(maskfile, flatten=True)
print(numpy.count_nonzero(mask))
E = envi.envi(envifile, mask=mask)
batch_size = 10000
Fv = E.loadbatch(batch_size) # load the first batch
n = 0
while not Fv == []: # loop until an empty batch is returned
if n == 0:
Tv = CLASS.predict(Fv.transpose()).transpose()
else:
Tv = numpy.append(
Tv,
CLASS.predict(Fv.transpose()).transpose(), 1
) # append the predicted labels from this batch to those of previous batches
COLORS = hyperspectral.unsift2(
Tv,
E.batchmask()) # convert the matrix of class labels to a 2D array
RGB = numpy.rollaxis(COLORS, 0, 3).astype(numpy.ubyte)
plt.imshow(RGB) # display it
plt.pause(0.05)
Fv = E.loadbatch(batch_size) # load the next batch
n = n + 1
return RGB
def load_test(data_file_names):
# load test data
masks_path = '/media/stim-scratch/berisha/00-annotations/hd/right/com-right/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for testing
C = C.astype(np.uint32)
bool_mask = np.sum(C.astype(np.uint32), 0)
# get number of classes
num_classes = C.shape[0]
for i in range(1, num_classes):
C[i, :, :] *= i + 1
total_mask = np.sum(C.astype(np.uint32), 0) # validation mask
Etest = envi.envi(data_file_names[0], mask=total_mask)
N = np.count_nonzero(total_mask) # set the batch size
Tv = [] # initialize the target array to empty
x_test = Etest.loadbatch(N)
y_test = total_mask.flat[np.flatnonzero(
total_mask)] # get the indices of valid pixels
Etest.close()
return x_test, y_test
def load_data(envifile, stainfile, trainmask="", normalize = True):
if trainmask == "":
E = envi.envi(filename=envifile)
else:
mask = scipy.misc.imread(trainmask, flatten=True)
E = envi.envi(envifile, mask=mask)
F= E.loadall() # return a matrix from envi file #bands * w * h
F = np.rollaxis(F, 0, 3)
# read the stained file
stain = scipy.misc.imread(stainfile)
if normalize:
F, stain = norm(F, stain, mode='standard')
else:
stain = stain/255.
return F , stain
def subsample(envi_file, mask_file, n):
E = envi.envi(envi_file)
D = E.loadall()
Ds = D[:, ::n, ::n]
Ds_sift = hyperspectral.sift2(Ds)
Ds_flatten = Ds_sift.flatten()
#mask = scipy.misc.imread(mask_file, flatten=True)
#D_subsample = E.loadmask(mask).transpose()
#D_flatten = D_subsample.flatten()
out_name = envi_file + '_sub'
outfile = open(out_name, 'wb')
Ds_flatten.tofile(outfile)
outfile.close()
return [Ds.shape[0], Ds.shape[1], Ds.shape[2]]
def feed_forward_net(envifile, stainfile, trainmask="", n_epoch=50, N=5000):
if trainmask == "":
E = envi.envi(envifile)
else:
mask = scipy.misc.imread(trainmask, flatten=True)
E = envi.envi(envifile, mask=mask)
mask = classify.random_mask(E.mask, N)
scipy.misc.imsave("random.bmp", mask)
Ft = E.loadmask(mask).transpose()
stain = numpy.rollaxis(scipy.misc.imread(stainfile), 2)
#plt.figure()
#plt.imshow(stain)
Tt = hyperspectral.sift2(stain, mask).transpose(
) # sift a 2D hyperspectral image into a PxB matrix where P is the number of pixels and B is the number of bands
print("Training MLPRegressor...")
net = tflearn.input_data(shape=[None, Ft.shape[1]])
net = tflearn.fully_connected(net, 7, activation='relu') #,
net = tflearn.fully_connected(net, 3, activation='relu') #,
net = tflearn.regression(net,
batch_size=128,
loss='mean_square',
metric='R2',
learning_rate=0.01) #learning_rate=0.01
CLASS = tflearn.DNN(net, tensorboard_dir='.\log', tensorboard_verbose=2)
CLASS.fit(Ft,
Tt,
n_epoch=n_epoch,
show_metric=True,
snapshot_step=500,
validation_set=0.2)
#if validate == False:
return CLASS
def load_train(data_file_names):
# load train data
masks_path = '/media/stim-scratch/berisha/00-annotations/hd/left/com-left/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for training
# open the ENVI file for reading, use the validation image for batch access
Etrain = envi.envi(data_file_names[0])
# x_train, y_train = Etrain.loadtrain(C)
x_train, y_train = Etrain.loadtrain_balance(C, num_samples=100000)
Etrain.close()
return x_train, y_train
def box_filter(envi_file, n):
E = envi.envi(envi_file)
D = E.loadall()
if E.header.interleave is not 'bsq':
print("ERROR: filtering is just implemented for BSQ!")
# D_filter = scipy.ndimage.uniform_filter(D, size=[5, 5, 1], mode='constant', cval=0.0)
k = numpy.ones([1, n, n], dtype='float32') * 1.0 / (float(n) * float(n)
) # definne box filter
D_filter = scipy.ndimage.filters.convolve(
D, k,
mode='reflect') # convolve the hyperspectral image with box filter
D_filter_sift = hyperspectral.sift2(
D_filter) # sift a 2D hyperspectral image into a PxB matrix
D_filter_1D = D_filter_sift.flatten() # flattenn the matrix to an array
envi_file_blur = envi_file + "_blured_bsq" # name the output
outfile = open(envi_file_blur,
'wb') # open a binary file to write the array
D_filter_1D.tofile(outfile)
outfile.close()
# create a header file-
shutil.copy2(envi_file + '.hdr', envi_file_blur + '.hdr')
return envi_file_blur
import classify import matplotlib.pyplot as plt from envi import envi import scipy.misc prob_file = 'cnn-load-batch-test-response' output_file = 'cnn-load-batch-test.png' prob_envi = envi(prob_file) prob_image = prob_envi.loadall() class_image = classify.prob2class(prob_image) rgb = classify.class2color(class_image) scipy.misc.imsave(output_file, rgb)
A comparison of several classifiers using ('optimal' params) from scikit-learn on FTIR data.
Code source: Sebastian Berisha
"""
# load train data
data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/'
masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for training
# open the ENVI file for reading, use the validation image for batch access
Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16')
#x_train, y_train = Etrain.loadtrain(C)
x_train, y_train = Etrain.loadtrain_balance(C, num_samples=60000)
Etrain.close()
# load test data
data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/brc961-proj/brc961-proj/new-cnn/'
masks_path = '/media/stim-processed/berisha/breast-processing/lm/brc961/masks/no-mnf-bcemn/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for testing
C = C.astype(np.uint32)
bool_mask = np.sum(C.astype(np.uint32), 0)
from sklearn.ensemble import RandomForestClassifier
import classify
import glob
import envi
# load train data
data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/'
masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for training
# open the ENVI file for reading, use the validation image for batch access
Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16')
x_train, y_train = Etrain.loadtrain(C)
#x_train, y_train = Etrain.loadtrain_balance(C, num_samples=60000)
Etrain.close()
# build a classifier
clf = RandomForestClassifier(n_estimators=20)
# Utility function to report best scores
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f} (std: {1:.3f})".format(
# Load the envi file names.
data_file_names = []
compiled = re.compile("\.")
for f in os.listdir(data_path):
if not f.startswith('.'):
abs_path = path.join(data_path, f)
file_name = re.split(compiled, abs_path)[0]
if path.isfile(abs_path) and not (file_name in data_file_names):
data_file_names.append(file_name)
if len(data_file_names) < 1:
raise ValueError('Missing data files in folder %s!' % data_path)
elif len(data_file_names) > 1:
warnings.warn("More than one ENVI file in data folder")
Etrain = envi.envi(data_file_names[0])
#x_train, y_train = Etrain.loadtrain(C)
x_train, y_train = Etrain.loadtrain_balance(C, num_samples=100000)
Etrain.close()
# load test data
data_path = '/media/stim-processed/berisha/breast-processing/hd/brc961/pca/pca16/'
masks_path = '/media/stim-scratch/berisha/00-annotations/hd/right/com-right/'
classimages = sorted(glob.glob(masks_path +
'*.png')) # load the class file names
C = classify.filenames2class(
classimages) # generate the class images for testing
C = C.astype(np.uint32)
bool_mask = np.sum(C.astype(np.uint32), 0)
import sklearn.neural_network import glob from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score from sklearn.metrics import confusion_matrix import classify data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/masks/no-mnf-bcemn/' classimages = sorted(glob.glob(masks_path + '*.png')) #load the class file names C = classify.filenames2class(classimages) #generate the class images for training Etrain = envi.envi(data_path + 'br1003-br2085b-bas-nor-fin-bip-pca16') #open the ENVI file for reading, use the validation image for batch access Ftrain, Ttrain = Etrain.loadtrain(C) #Ftrain, Ttrain = Etrain.load_train_balance(C, 60000) #load the training set rf_model = RandomForestClassifier(n_estimators=100, max_depth=10) rf_model.fit(Ftrain, Ttrain) #CLASS1 = sklearn.naive_bayes.GaussianNB() #train a Bayesian classifier #CLASS1.fit(Ft1, Tt1) Etrain.close() ###########################################testing############################################### data_path = '/media/stim-processed/berisha/breast-processing/lm/br1003/no-mnf/brc961-proj/brc961-proj/new-cnn/' masks_path = '/media/stim-processed/berisha/breast-processing/lm/brc961/masks/no-mnf-bcemn/'
# Load the envi file names.
data_file_names = []
compiled = re.compile("\.")
for f in os.listdir(data_path):
if not f.startswith('.'):
abs_path = path.join(data_path, f)
file_name = re.split(compiled, abs_path)[0]
if path.isfile(abs_path) and not (file_name in data_file_names):
data_file_names.append(file_name)
if len(data_file_names) < 1:
raise ValueError('Missing data files in folder %s!' % data_path)
elif len(data_file_names) > 1:
warnings.warn("More than one ENVI file in data folder")
Etrain = envi.envi(data_file_names[0])
x_train, y_train = Etrain.loadtrain_balance(C, num_samples=10)
Etrain.close()
# #############################################################################
# Train classifiers
#
# For an initial search, a logarithmic grid with basis
# 10 is often helpful. Using a basis of 2, a finer
# tuning can be achieved but at a much higher cost.
C_range = np.logspace(-3, 3, 7)
gamma_range = np.logspace(-3, 3, 7)
param_grid = dict(gamma=gamma_range, C=C_range)
cv = StratifiedShuffleSplit(n_splits=5, test_size=0.25, random_state=42)
grid = GridSearchCV(SVC(), param_grid=param_grid, cv=cv, n_jobs=16)