def read_data(self, path, crop_spectra=True, X_min=800, X_max=2000):
data = reshape_data(path, X_min=X_min, X_max=X_max)[0]
if crop_spectra == True:
data = data.iloc[:, 400:900].values
data = area_normalization(data)
X_test_pca = self.pca.transform(data)
return X_test_pca
'model_path': '../result/SVM/SVM_MODEL/',
'print_raw': True,
'show_interactive_plot': True,
'show_pca_loadings': False,
'show_tsne': False
}
plt.ioff()
# Writing paras into .json file
with open(paras['model_path'] + "paras_record.json", "w") as f:
json.dump(paras, f)
print("File successfully written... ...")
nor_Fat = area_normalization(
reshape_data('../Data/Trainging Data/20%Fat/',
X_max=paras['X_max'],
X_min=paras['X_min'])[0])
nor_Tumor = area_normalization(
reshape_data('../Data/Trainging Data/Tumor/',
X_max=paras['X_max'],
X_min=paras['X_min'])[0])
# Splitting data into training set and testing set
X_train, X_test, y_train, y_test, X_train_path, X_test_path = load_data_preprocess(
paras['train_test_percent'],
crop_spectra=paras['crop_spectra'],
aug=paras['aug'],
reshape=paras['reshape'],
one_hot=paras['one_hot'],
X_min=paras['X_min'],
X_max=paras['X_max'],
if __name__ == '__main__':
# Importing data and model
predictor = Predictor('../result/SVM/SVM_MODEL/model.pkl')
# Printing info about model and data
print("classifier model info: ", predictor.model)
with open("../result/SVM/SVM_MODEL/paras_record.json", 'r') as paras_f:
paras = json.load(paras_f)
print(paras)
data = predictor.read_data('../Data/Testing Data/Yale Data/',
crop_spectra=paras['crop_spectra'],
X_min=paras['X_min'],
X_max=paras['X_max'])
Fat, Fat_file_path = reshape_data(
'../Data/Testing Data/Yale Data/Healthy/',
X_min=paras['X_min'],
X_max=paras['X_max'])
Tumor, Tumor_file_path = reshape_data(
'../Data/Testing Data/Yale Data/Tumor/',
X_min=paras['X_min'],
X_max=paras['X_max'])
path = pd.concat(
[pd.DataFrame(Fat_file_path),
pd.DataFrame(Tumor_file_path)],
ignore_index=True)
path.columns = ['file_path']
# Getting true label
true_label = make_labels(Fat, Tumor)
label = predictor.predict(data)
import numpy as np
from keras.engine.saving import load_model
from pandas._libs import json
from load_data_preprocess.load_data import reshape_data
if __name__ == "__main__":
print("Loading paras... ... ")
with open(
"../result/CNN/CNN_MODEL/weights/CNN_Noise_DataAug/paras_record.json",
'r') as paras_f:
paras = json.load(paras_f)
print(paras)
X_test = reshape_data("../Data/Testing Data/S19-19384",
X_min=paras['X_min'],
X_max=paras['X_max'])[0]
X_test = np.array(X_test).astype('float32')
X_test = X_test.reshape(X_test.shape + (1, ))
X_test /= np.max(X_test)
my_model = load_model(
'../result/CNN/CNN_MODEL/weights/CNN_Noise_DataAug/highest_val_acc_weights_epoch05-val_acc1.000_base_cnn.h5'
)
print(my_model.summary())
print('X_test.shape: ', X_test.shape)
result = np.around(my_model.predict(X_test), decimals=2)
print(result)
# pd.DataFrame(result).to_csv('../result/CNN/CNN_TESTING/S19-19384/fully_cnn/fully_cnn1200_noAug.csv')
def load_data_preprocess(train_test_percent,
crop_spectra=False,
aug=False,
reshape=False,
one_hot=False,
X_min=800,
X_max=2000,
fat_path='../Data/Trainging Data/20%Fat/',
tumor_path='../Data/Trainging Data/Tumor/',
print_raw=False):
'''
:param train_test_percent:
:param crop_spectra:
:param aug:
:param reshape:
:param one_hot:
:param X_min:
:param X_max:
:param fat_path:
:param tumor_path:
:param print_raw:
:param print_file_path:
:return:
'''
# load data
Fat, Fat_file_path = reshape_data(fat_path, X_min=X_min, X_max=X_max)
Tumor, Tumor_file_path = reshape_data(tumor_path, X_min=X_min, X_max=X_max)
path = pd.concat([pd.DataFrame(Fat_file_path), pd.DataFrame(Tumor_file_path)], ignore_index=True)
data_All = pd.concat([pd.DataFrame(Fat.append(Tumor, ignore_index=True)), path], axis=1)
data_All = np.array(data_All)
print('Fat.shape: ', Fat.shape)
print('Tumor.shape: ', Tumor.shape)
print('Data_all.shape: ', data_All.shape)
labels = make_labels(Fat, Tumor)
print('labels: ', labels)
X_train, X_test, y_train, y_test = train_test_split(data_All, labels, test_size=train_test_percent,
random_state=42)
X_train_path = X_train
X_test_path = X_test
if aug == True:
from sklearn.decomposition import PCA
noise_aug = []
noise = np.copy(X_train)
mu = np.mean(noise, axis=0)
pca = PCA()
noise_model = pca.fit(noise)
nComp = 10
Xhat = np.dot(pca.transform(noise)[:, :nComp], pca.components_[:nComp, :])
noise_level = np.dot(pca.transform(noise)[:, nComp:], pca.components_[nComp:, :])
Xhat += mu
SNR = np.linspace(1, 5, 50)
for i in range(len(SNR)):
noise_aug.append(SNR[i] * noise_level + Xhat)
j = 0
for spectra in noise_aug[i]:
noise_aug[i][j] = spectra / np.max(spectra)
j += 1
X_train = np.array(noise_aug).reshape(50 * 177, 1200)
y_train = [item for i in range(50) for item in y_train]
if crop_spectra == True:
X_train, X_test = crop(X_train, X_test, min=400, max=900)
X_train, X_test, y_train, y_test = preprocess(X_train[:, :-1], X_test[:, :-1], y_train, y_test, reshape=reshape,
mean_center=False,
norm=True,
one_hot=one_hot)
print('X_train.shape: ', X_train.shape)
print('X_test.shape: ', X_test.shape)
print('y_train.shape: ', y_train.shape)
print('y_test.shape: ', y_test.shape)
if print_raw == True:
return X_train, X_test, y_train, y_test, X_train_path, X_test_path
return X_train, X_test, y_train, y_test