def evaluate_cnn(self, model, pixels, batch_size=64, merge=True):
predictions = model.predict_generator(generator=self.tile_generator(
pixels, batch_size=batch_size, merge=merge),
steps=len(pixels) // batch_size,
verbose=1)
eval_generator = self.tile_generator(pixels, batch_size=1)
labels = np.empty(predictions.shape)
count = 0
while count < len(labels):
image_b, label_b = next(eval_generator)
labels[count] = label_b
count += 1
label_index = np.argmax(labels, axis=1)
pred_index = np.argmax(predictions, axis=1)
np.set_printoptions(precision=2)
# Plot non-normalized confusion matrix
util.plot_confusion_matrix(label_index,
pred_index,
classes=np.array(
list(util.indexed_dictionary)),
class_dict=util.indexed_dictionary)
# Plot normalized confusion matrix
util.plot_confusion_matrix(label_index,
pred_index,
classes=np.array(
list(util.indexed_dictionary)),
class_dict=util.indexed_dictionary,
normalize=True)
count = 0
for i in range(len(label_index)):
if (label_index[i] == pred_index[i]):
count += 1
print("Accuracy is {}".format(count / len(label_index)))
def predict():
file_list_predict = get_data_paths('predict')
line_parser = MbtiParser()
predict_generator = BatchGenerator(file_list_predict,
line_parser,
batch_size=batch_size_predict,
build_voc=False,
max_sentence_length=max_sentence_length,
voc_path=voc_path)
cnn = CnnTextClassifier(**cnn_hyperparameters)
pred_dic = cnn.predict(predict_generator)
plt.figure()
plot_confusion_matrix(pred_dic['ground_truth'],
pred_dic['predictions'],
list(range(num_classes)),
title='Confusion matrix, without normalization')
plt.figure()
plot_confusion_matrix(pred_dic['ground_truth'],
pred_dic['predictions'],
list(range(num_classes)),
title='Confusion matrix, with normalization',
normalize=True)
plt.show()
def visualize_evaluation(self,
X,
Y,
labels,
title,
normalize=None,
save=True,
path=None):
"""
Parameters
----------
X : TYPE
DESCRIPTION.
Y : TYPE
DESCRIPTION.
labels : TYPE
DESCRIPTION.
normalize : {‘true’, ‘pred’, ‘all’}, optional
Normalizes confusion matrix over the true (rows), predicted (columns) conditions or
all the population. If None, confusion matrix will not be normalized. The default
is None.
save : TYPE, optional
DESCRIPTION. The default is True.
path : TYPE, optional
DESCRIPTION. The default is None.
Raises
------
ValueError
DESCRIPTION.
Returns
-------
None.
"""
if save and path == None:
raise ValueError(
'Invalid value given to `path`, `path` can not be `None` when `save` is True'
)
#Evaluate and print report of classification
Y_pred, acc, values = self.__get_acc(X, Y, labels)
#Get and plot Confusion matrix
cm_n = confusion_matrix(Y, Y_pred, normalize=normalize)
cm = confusion_matrix(Y, Y_pred)
if '.png' not in path:
path += '.png'
path_n = path.split('.png')[0] + '_Normalised.png'
plot_confusion_matrix(cm, values, path, title)
plot_confusion_matrix(cm_n, values, path_n, title)
return acc
def evaluate_discriminator(X_test,
labels,
gans_path,
path_cm,
title,
normalize='all'):
print(
'\n___________________________________________________________________________'
)
print('Evaluating Discriminator...\n')
#loading Discriminator model
if gans_path[-1] != '/':
gans_path += '/'
discriminator = load_model(gans_path + 'Discriminator.h5')
generator = load_model(gans_path + 'Generator.h5')
#Preparing data
X_benignware = X_test[0].todense()
X_malware = X_test[1].todense()
#Getting predictions from Generator
X_generated = generator.predict(X_malware)
len_gen = X_generated.shape[0]
#Preparing X_real, Y_real & Y_generated
X_real = X_benignware
len_real = X_real.shape[0]
Y_real = np.zeros(len_real)
Y_generated = np.ones(len_gen)
#Making predictions on Discriminator
Y_pred_real = discriminator.predict(X_real)
Y_pred_gen = discriminator.predict(X_generated)
Y_pred = np.concatenate((Y_pred_real, Y_pred_gen))
Y_pred = np.where(Y_pred > 0.5, 1, 0)
Y = np.array(np.concatenate((Y_real, Y_generated)))
acc, values = get_acc(Y, Y_pred, labels)
#Get and plot Confusion matrix
cm_n = confusion_matrix(Y, Y_pred, normalize=normalize)
cm = confusion_matrix(Y, Y_pred)
if '.png' not in path_cm:
path_cm += '.png'
path_n = path_cm.split('.png')[0] + '_Normalised.png'
plot_confusion_matrix(cm, values, path_cm, title)
plot_confusion_matrix(cm_n, values, path_n, title)
print('\nEvaluation of Discriminator is Completed!')
print(
'___________________________________________________________________________\n'
)
return acc
def predict():
X_train, X_test, y_train, y_test = create_data_set()
fd = FaceDetector()
pred_dic = fd.predict(X_test,y_test)
plt.figure()
plot_confusion_matrix(pred_dic['ground_truth'],pred_dic['predictions'], classes=range(np.max(pred_dic['predictions'])),
title='Confusion matrix, without normalization')
plt.show()
return pred_dic
pydot.find_graphviz = lambda: True
x, y = make_classification(n_samples=100,
n_informative=2,
n_features=2,
n_redundant=0,
n_clusters_per_class=1,
random_state=7)
y
model = Sequential()
model.add(Dense(units=1, input_shape=(2, ), activation='sigmoid'))
#sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model.fit(x=x, y=y, verbose=0, epochs=200)
print(model.get_weights())
print(model.summary())
plot_loss_accuracy(history)
plot_decision_boundary(lambda x: model.predict(x), x, y)
y_pred = model.predict_classes(x, verbose=0)
plot_confusion_matrix(model, x, y)
import pandas as pd
import numpy as np
from keras.utils import np_utils
pydot.find_graphviz = lambda: True
input = pd.read_csv("sample1.csv")
X_train = input.iloc[:,1:3].as_matrix()
y_train = input["Output"].as_matrix()
model1 = Sequential()
model1.add(Dense(units =1,input_shape=(2,),activation='sigmoid'))
#sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
model1.compile(optimizer='sgd', loss='binary_crossentropy', metrics=['accuracy'])
history1 = model1.fit(x = X_train,y = y_train,verbose=0,epochs=1)
print(model1.get_weights())
plot_loss_accuracy(history1)
plot_decision_boundary(lambda X_train: model1.predict(X_train),X_train,y_train)
y_pred = model1.predict_classes(X_train, verbose=0)
plot_confusion_matrix(model1,X_train,y_train)
def test_casme(batch_size, spatial_epochs, temporal_epochs, train_id, dB,
spatial_size, flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ice/OS/Datasets/"
workplace = root_db_path + dB + "/"
inputDir = root_db_path + dB + "/" + dB + "/"
######################################################
classes = 5
if dB == 'CASME2_TIM':
table = loading_casme_table(workplace + 'CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 2
samples = 246
n_exp = 5
# VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
listOfIgnoredSamples = []
VidPerSubject = [2, 1]
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
############################################
os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'CASME2_Optical':
table = loading_casme_table(workplace + 'CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index, _ = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 26
samples = 246
n_exp = 5
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 9
data_dim = r * w
pad_sequence = 9
channel = 3
############################################
# os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'CASME2_RGB':
# print(inputDir)
table = loading_casme_table(workplace +
'CASME2_RGB/CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casmergb_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 26
samples = 245 # not used, delete it later
n_exp = 5
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
############################################
elif dB == 'SMIC_TIM10':
table = loading_smic_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 16
samples = 164
n_exp = 3
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 1
classes = 3
#########################################################
elif dB == 'SAMM_Optical':
table, table_objective = loading_samm_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 29
samples = 159
n_exp = 8
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 9
data_dim = r * w
pad_sequence = 10
channel = 3
classes = 8
#########################################################
elif dB == 'SAMM_TIM10':
table, table_objective = loading_samm_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 29
samples = 159
n_exp = 8
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
classes = 8
#########################################################
# print(VidPerSubject)
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
elif flag == 'st4vis':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 3
#########################################
############ Reading Images and Labels ################
SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB,
resizedFlag, table, workplace, spatial_size,
channel)
print("Loaded Images into the tray...")
labelperSub = label_matching(workplace, dB, subjects, VidPerSubject)
print("Loaded Labels into the tray...")
if channel_flag == 1:
inputDir = root_db_path + dB + "/" + dB + "/"
SubperdB_strain = Read_Input_Images(
root_db_path + 'CASME2_Strain_TIM10' + '/' +
'CASME2_Strain_TIM10' + '/', listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size,
3)
SubperdB_gray = Read_Input_Images(
root_db_path + 'CASME2_TIM' + '/' + 'CASME2_TIM' + '/',
listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace,
spatial_size, 3)
elif channel_flag == 3:
inputDir_strain = '/media/ice/OS/Datasets/CASME2_Strain_TIM10/CASME2_Strain_TIM10/'
SubperdB_strain = Read_Input_Images(inputDir_strain,
listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 3)
inputDir_gray = '/media/ice/OS/Datasets/CASME2_TIM/CASME2_TIM/'
SubperdB_gray = Read_Input_Images(inputDir_gray, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
elif channel_flag == 2:
SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 1)
SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
#######################################################
########### Model Configurations #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
else:
data_dim = 8192
########################################################
########### Image Data Generator ##############
image_generator = ImageDataGenerator(zca_whitening=True,
rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
rescale=1.5)
###############################################
########### Training Process ############
# Todo:
# 1) LOSO (done)
# 2) call model (done)
# 3) saving model architecture
# 4) Saving Checkpoint (done)
# 5) make prediction (done)
if tensorboard_flag == 1:
tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/"
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
weights_dir = '/media/ice/OS/Datasets/Weights/53/'
image_path = '/home/ice/Documents/Micro-Expression/image/'
table_count = 0
for sub in range(subjects):
############### Reinitialization & weights reset of models ########################
temporal_model_weights = weights_dir + 'temporal_enrichment_ID_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
vgg_model_weights = weights_dir + 'vgg_spatial_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
vgg_model_strain_weights = weights_dir + 'vgg_spatial_strain_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
conv_ae_weights = weights_dir + 'autoencoder_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
conv_ae_strain_weights = weights_dir + 'autoencoder_strain_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
weights_path=temporal_model_weights)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
conv_ae = convolutional_autoencoder(spatial_size=spatial_size,
weights_path=conv_ae_weights)
conv_ae.compile(loss='binary_crossentropy', optimizer=adam)
conv_ae_strain = convolutional_autoencoder(
spatial_size=spatial_size, weights_path=conv_ae_strain_weights)
conv_ae_strain.compile(loss='binary_crossentropy', optimizer=adam)
vgg_model = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path=vgg_model_weights)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
vgg_model_strain = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path=vgg_model_strain_weights)
vgg_model_strain.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO(
sub, SubperdB, labelperSub, subjects, classes)
# Rearrange Training labels into a vector of images, breaking sequence
Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r,
w, channel)
Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w,
channel)
# Special Loading for 4-Channel
if channel_flag == 1 or channel_flag == 3:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects, classes)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 3)
Test_X_Strain = Test_X_Strain.reshape(
Test_X.shape[0] * timesteps_TIM, r, w, 3)
Train_X_Gray, _, Test_X_Gray, _, _ = data_loader_with_LOSO(
sub, SubperdB_gray, labelperSub, subjects, classes)
Test_X_Gray = Test_X_Gray.reshape(Test_X_Gray.shape[0] * 10, r, w,
3)
# print(Train_X_Strain.shape)
# Train_X_Strain = Train_X_Strain[0]
# Train_X_Strain = Train_X_Strain.reshape((224, 224, 3, 1))
# Train_X_Strain = Train_X_Strain.reshape((224, 224, 3))
# cv2.imwrite('steveharvey.png', Train_X_Strain)
# Concatenate Train X & Train_X_Strain
# Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3)
# Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3)
total_channel = 4
# Extend Y labels 10 fold, so that all images have labels
Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0)
Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0)
##################### Training & Testing #########################
# print(Train_X_spatial.shape)
test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w)
test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X = test_X.astype('float32') / 255.
Test_X_Strain = Test_X_Strain.reshape(Test_X_Strain.shape[0], channel,
r, w)
Test_X_Gray = Test_X_Gray.reshape(Test_X_Gray.shape[0], channel, r, w)
# test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X_strain = test_X.astype('float32') / 255.
# print(X.shape)
###### conv weights must be freezed for transfer learning ######
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# vgg
model = Model(inputs=vgg_model.input,
outputs=vgg_model.layers[35].output)
plot_model(model,
to_file="spatial_module_FULL_TRAINING.png",
show_shapes=True)
output = model.predict(test_X)
# vgg strain
model_strain = Model(inputs=vgg_model_strain.input,
outputs=vgg_model_strain.layers[35].output)
plot_model(model_strain,
to_file="spatial_module_FULL_TRAINING_strain.png",
show_shapes=True)
output_strain = model_strain.predict(Test_X_Strain)
# ae
# model_ae = Model(inputs=conv_ae.input, outputs=conv_ae.output)
# plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# output_ae = model_ae.predict(normalized_test_X)
# output_ae = model.predict(output_ae)
# ae strain
# model_ae_strain = Model(inputs=conv_ae_strain.input, outputs=conv_ae_strain.output)
# plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# output_ae_strain = model_ae_strain.predict(normalized_test_X_strain)
# output_ae_strain = model_ae_strain.predict(output_ae_strain)
# concatenate features
output = np.concatenate((output, output_strain), axis=1)
features = output.reshape(int(Test_X.shape[0]), timesteps_TIM,
output.shape[1])
# temporal
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
# visualize cam
countcam = 0
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/log_hde' +
str(train_id) + '.txt', 'a')
file.write(str(sub + 1) + "\n")
for item_idx in range(len(predict)):
test_strain = Test_X_Gray[item_idx + countcam]
test_strain = test_strain.reshape((224, 224, 3))
item = test_strain
cam_output = visualize_cam(model, 29, 0, item)
cam_output2 = visualize_cam(model, 29, 1, item)
cam_output3 = visualize_cam(model, 29, 2, item)
cam_output4 = visualize_cam(model, 29, 3, item)
cam_output5 = visualize_cam(model, 29, 4, item)
overlaying_cam = overlay(item, cam_output)
overlaying_cam2 = overlay(item, cam_output2)
overlaying_cam3 = overlay(item, cam_output3)
overlaying_cam4 = overlay(item, cam_output4)
overlaying_cam5 = overlay(item, cam_output5)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam0.png', overlaying_cam)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam1.png', overlaying_cam2)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam2.png', overlaying_cam3)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam3.png', overlaying_cam4)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam4.png', overlaying_cam5)
countcam += 9
######## write the log file for megc 2018 ############
result_string = table[table_count, 1] + ' ' + str(
int(Test_Y_gt[item_idx])) + ' ' + str(
predict[item_idx]) + '\n'
file.write(result_string)
######################################################
table_count += 1
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# print(type(ct))a
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects)
###############################################################################
tot_mat_cm = np.asarray(tot_mat, dtype=int)
plt.figure()
classes_test = [0, 1, 2, 3, 4]
plot_confusion_matrix(tot_mat_cm,
classes_test,
normalize=True,
title='Confusion matrix_single_db')
plt.show()
#X, y = make_moons(n_samples=1000, noise=0.05, random_state=0)
plot_data(X, y)
#single perceptron model for binary classifcation
model1 = Sequential()
model1.add(Dense(1, input_shape=(2, ), activation='sigmoid'))
model1.compile('adam', 'binary_crossentropy', metrics=['accuracy'])
plot_model(model1, show_shapes=True, to_file='model1.png')
history1 = model1.fit(X, y, verbose=0, epochs=100)
plot_loss_accuracy(history1)
plot_decision_boundary(lambda x: model1.predict(x), X, y)
y_pred = model1.predict_classes(X, verbose=0)
plot_confusion_matrix(model1, X, y)
#mlp model for binary classification
model2 = Sequential()
model2.add(Dense(4, input_shape=(2, ), activation='tanh'))
model2.add(Dense(2, activation='tanh'))
model2.add(Dense(1, activation='sigmoid'))
model2.compile(Adam(lr=0.01), 'binary_crossentropy', metrics=['accuracy'])
plot_model(model2, show_shapes=True, to_file='model2.png')
history2 = model2.fit(X, y, verbose=0, epochs=50)
plot_loss_accuracy(history2)
plot_decision_boundary(lambda x: model2.predict(x), X, y)
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from utilities import plot_confusion_matrix
true_labels = [2, 0, 0, 2, 4, 4, 1, 0, 3, 3, 3]
pred_labels = [2, 1, 0, 2, 4, 3, 1, 0, 1, 3, 3]
confusion_mat = confusion_matrix(true_labels, pred_labels)
print(confusion_mat)
targets = ['Class-0', 'Class-1', 'Class-2', 'Class-3', 'Class-4']
print('\n',
classification_report(true_labels, pred_labels, target_names=targets))
plot_confusion_matrix(confusion_mat, true_labels, normalize=True)
np.save('no_fs_time', no_fs_time)
np.save('hs_time', hs_time)
np.save('cs_time', cs_time)
np.save('dfa_time', dfa_time)
np.save('no_fs_score', no_fs_score)
np.save('hs_score', hs_score)
np.save('cs_score', cs_score)
np.save('dfa_score', dfa_score)
np.save('no_fs_f1', no_fs_f1)
np.save('hs_f1', hs_f1)
np.save('cs_f1', cs_f1)
np.save('dfa_f1', dfa_f1)
time_msg = "Times: %f(no_fs) %f(hs) %f(cs) %f(dfa)" % (
no_fs_time.mean(), hs_time.mean(), cs_time.mean(), dfa_time.mean())
print(time_msg)
scores_msg = "Scores: %f(no_fs) %f(hs) %f(cs) %f(dfa)" % (
no_fs_score.mean(), hs_score.mean(), cs_score.mean(), dfa_score.mean())
print(scores_msg)
f1_msg = "f1: %f(no_fs) %f(hs) %f(cs) %f(dfa)" % (
no_fs_f1.mean(), hs_f1.mean(), cs_f1.mean(), dfa_f1.mean())
print(f1_msg)
plot_confusion_matrix(correct_y, no_fs_y, genres)
plot_confusion_matrix(correct_y, hs_y, genres)
plot_confusion_matrix(correct_y, cs_y, genres)
plot_confusion_matrix(correct_y, dfa_y, genres)
fprs[label][median],
label='%s vs rest' % genre_list[label])
all_pr_scores = np.asarray(pr_scores.values()).flatten()
summary = (np.mean(scores), np.std(scores), np.mean(all_pr_scores),
np.std(all_pr_scores))
#print("%.3f\t%.3f\t%.3f\t%.3f\t" % summary)
#saving the trained model to disk
joblib.dump(clf, 'saved_model/model_ceps.pkl')
return np.mean(train_errors), np.mean(test_errors), np.asarray(cms)
if __name__ == "__main__":
start = timeit.default_timer()
print
print " Starting of classification model \n"
print " Classification is running ... \n"
X, y = read_ceps(genre_list)
train_avg, test_avg, cms = train_model(X, y, "ceps", plot=True)
cm_avg = np.mean(cms, axis=0)
cm_norm = cm_avg / np.sum(cm_avg, axis=0)
print " Classification is done \n"
stop = timeit.default_timer()
print " Total time taken (s) = ", (stop - start)
print "\n Plotting of confusion matrix ... \n"
plot_confusion_matrix(cm_norm, genre_list, "ceps",
"CEPS classifier - Confusion matrix")
print " Please check the plots in 'graphs' directory \n"
# Relative importance of each feature within the whole feature set
importance_ratio = high_rank_fre / sum(high_rank_fre)
trace = go.Bar(x=feature_names, y=importance_ratio, name=classifier)
importance_traces.append(trace)
# print accuracy of each classifier
print("Accuracy of", classifier, "classifier is:",
"%.4f" % accuracy_score(label_true, label_predicted))
# plot confusion matrix
cm = confusion_matrix(label_true, label_predicted)
print("Confusion matrix:")
print(cm)
print()
plot_confusion_matrix(cm=cm,
label_names=label_names,
title="Confusion matrix of " + classifier +
" classifier",
normalize=True)
layout = dict(
title=dict(
text='Relative importance of features by different classifier',
x=0.5,
),
yaxis=dict(title="ratio"),
xaxis=dict(title='candy characteristic', ),
)
fig = dict(data=importance_traces, layout=layout)
py.plot(fig, filename=importance_path,
auto_open=False) # show importance of features
loss_acc_plot.semilogy(epoch_vector, data, label=label)
loss_acc_plot.legend(loc="upper left")
# Plot confusion matrix
# Compute confusion matrix
y_pred = model.predict(X_test)
cnf_matrix = confusion_matrix(y_test.argmax(1), y_pred.argmax(1))
# Plot normalized confusion matrix
classes = [0]
classes.extend(df.digit.unique()[:-1])
classes = list(map(str, classes))
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=classes,
normalize=True,
title='Normalized confusion matrix',
cmap=plt.cm.Greys)
plt.show()
# Calculate the F1 score
print("F1 score per label: " + ', '.join(
map(
str,
np.around(f1_score(y_test.argmax(1), y_pred.argmax(1), average=None),
2))))
print("Global F1 score: %.2f" %
f1_score(y_test.argmax(1), y_pred.argmax(1), average="micro"))
def inference_validation_data(args):
# Arugments & parameters
dataset_dir = args.dataset_dir
subdir = args.subdir
workspace = args.workspace
holdout_fold = args.holdout_fold
iteration = args.iteration
filename = args.filename
cuda = args.cuda
labels = config.labels
if 'mobile' in subdir:
devices = ['a', 'b', 'c']
else:
devices = ['a']
validation = True
classes_num = len(labels)
# Paths
hdf5_path = os.path.join(workspace, 'features', 'logmel', subdir,
'development.h5')
dev_train_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup',
'fold1_train.txt')
dev_validate_csv = os.path.join(dataset_dir, subdir, 'evaluation_setup',
'fold{}_evaluate.txt'.format(holdout_fold))
model_path = os.path.join(workspace, 'models', subdir, filename,
'holdout_fold={}'.format(holdout_fold),
'md_{}_iters.tar'.format(iteration))
# Load model
model = Model(classes_num)
checkpoint = torch.load(model_path)
model.load_state_dict(checkpoint['state_dict'])
if cuda:
model.cuda()
# Predict & evaluate
for device in devices:
print('Device: {}'.format(device))
# Data generator
generator = DataGenerator(hdf5_path=hdf5_path,
batch_size=batch_size,
dev_train_csv=dev_train_csv,
dev_validate_csv=dev_validate_csv)
generate_func = generator.generate_validate(data_type='validate',
devices=device,
shuffle=False)
# Inference
dict = forward(model=model,
generate_func=generate_func,
cuda=cuda,
return_target=True)
outputs = dict['output'] # (audios_num, classes_num)
targets = dict['target'] # (audios_num, classes_num)
predictions = np.argmax(outputs, axis=-1)
classes_num = outputs.shape[-1]
# Evaluate
confusion_matrix = calculate_confusion_matrix(
targets, predictions, classes_num)
class_wise_accuracy = calculate_accuracy(targets, predictions,
classes_num)
# Print
print_accuracy(class_wise_accuracy, labels)
print('confusion_matrix: \n', confusion_matrix)
# Plot confusion matrix
plot_confusion_matrix(
confusion_matrix,
title='Device {}'.format(device.upper()),
labels=labels,
values=class_wise_accuracy)
def train(lr, batchSize, epochs, tvweightdetection, tvweightsegmentation,
saveImages):
avDev = getDev()
print(avDev)
seed = 1
setSeed(seed)
from utilities import showImagesDetection, showImagesSegmentation, showDetectedImages, visualiseSegmented
from dataloader import CudaVisionDataLoader
listTransforms_train = [
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
]
listTransforms_test = [
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
]
data = CudaVisionDataLoader()
parentDir = './small_data'
dirDetectionDataset = os.path.join(parentDir, 'detection')
dirSegmentationDataset = os.path.join(parentDir, 'segmentation')
train_loader_detection = data.__call__(
os.path.join(dirDetectionDataset, 'train'), "detection",
listTransforms_train, batchSize)
validate_loader_detection = data.__call__(
os.path.join(dirDetectionDataset, 'validate'), "detection",
listTransforms_test, batchSize)
test_loader_detection = data.__call__(
os.path.join(dirDetectionDataset, 'test'), "detection",
listTransforms_test, batchSize)
train_loader_segmentation = data.__call__(
os.path.join(dirSegmentationDataset, 'train'), "segmentation",
listTransforms_train, batchSize)
validate_loader_segmentation = data.__call__(
os.path.join(dirSegmentationDataset, 'validate'), "segmentation",
listTransforms_test, batchSize)
test_loader_segmentation = data.__call__(
os.path.join(dirSegmentationDataset, 'test'), "segmentation",
listTransforms_test, batchSize)
from model import soccerSegment
from metrics import det_metrics, segmentationAccuracy, seg_iou, det_confusion_matrix, seg_confusion_matrix
from utilities import get_colored_image, get_predected_centers, plot_confusion_matrix
import torchvision.models as models
print("Loading pretrained ResNet18")
resnet18 = models.resnet18(pretrained=True)
model = soccerSegment(resnet18, [5, 6, 7, 8], [64, 128, 256, 256, 0],
[512, 256, 256, 128], [512, 512, 256], 256)
model.to(avDev)
criterionDetected = nn.MSELoss()
criterionSegmented = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
checkpoint_path = './checkpoints/checkpoint'
val_patience = 10
val_counter = 0
best_acc = np.NINF
epoch = 0
color_classes = [[255, 0, 0], [0, 0, 255], [0, 255, 0]]
detlosses = []
seglosses = []
valdetlosses = []
valseglosses = []
print("Load saved model if any")
if os.path.exists(checkpoint_path):
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
epoch = checkpoint['epoch'] + 1
seglosses = checkpoint['seglosses']
detlosses = checkpoint['detlosses']
valseglosses = checkpoint['valseglosses']
valdetlosses = checkpoint['valdetlosses']
print("Checkpoint Loaded")
print("Specified epochs for training: ", epoch,
" The patience value for early stopping: ", val_patience,
"Batch Size: ", batchSize, " Learning Rate: ", lr)
print("Starting training")
while epoch < epochs:
detlosstrain = 0
seglosstrain = 0
print("Epoch: ", epoch)
det_train_acc = 0.0
seg_train_acc = 0.0
model.train()
steps = 0
fullDataCovered = 0
dataiter_detection = train_loader_detection.__iter__()
dataiter_segmentation = train_loader_segmentation.__iter__()
while (fullDataCovered != 1):
for i in range(4):
try:
images, targets, target_centers = dataiter_detection.next()
except:
fullDataCovered = 1
dataiter_detection = train_loader_detection.__iter__()
images, targets, target_centers = dataiter_detection.next()
images = images.to(avDev)
targets = targets.to(avDev)
optimizer.zero_grad()
segmented, detected = model(images)
tvLoss = TVLossDetect(tvweightdetection)
total_variation_loss = tvLoss.forward(detected)
mseloss = criterionDetected(detected, targets)
loss = mseloss + total_variation_loss
detlosstrain += loss.item()
steps += 1
# Getting gradients w.r.t. parameters
loss.backward()
# Updating parameters
optimizer.step()
ground_truth_centers = get_predected_centers(target_centers)
colored_images = get_colored_image(detected)
det_train_acc += np.average(
det_metrics(ground_truth_centers, colored_images,
color_classes)[0])
i += 1
try:
images, targets = dataiter_segmentation.next()
except:
fullDataCovered = 1
dataiter_segmentation = train_loader_segmentation.__iter__()
images, targets = dataiter_segmentation.next()
images = images.to(avDev)
targets = targets.to(avDev)
optimizer.zero_grad()
segmented, detected = model(images)
segmentedLabels = torch.argmax(segmented, dim=1)
accuracies = segmentationAccuracy(segmentedLabels.long(), targets,
[0, 1, 2])
tvLoss = TVLossSegment(tvweightsegmentation)
total_variation_loss = tvLoss.forward(segmented)
cross_entropy_loss = criterionSegmented(segmented, targets)
loss = cross_entropy_loss + total_variation_loss
seglosstrain += loss.item()
steps += 1
# Getting gradients w.r.t. parameters
loss.backward()
# Updating parameters
optimizer.step()
seg_train_acc += accuracies[3]
det_train_acc /= len(train_loader_detection)
seg_train_acc /= len(train_loader_segmentation)
detlosstrain /= len(train_loader_detection)
seglosstrain /= len(train_loader_segmentation)
print("Training finished , starting with validation")
det_val_acc = 0.0
seg_val_acc = 0.0
model.eval()
steps = 0
detlosses.append(detlosstrain)
seglosses.append(seglosstrain)
detlossval = 0
seglossval = 0
for images, targets, target_centers in validate_loader_detection:
with torch.no_grad():
images = images.to(avDev)
targets = targets.to(avDev)
segmented, detected = model(images)
tvLoss = TVLossDetect(tvweightdetection)
total_variation_loss = tvLoss.forward(detected)
mseloss = criterionDetected(detected, targets)
detloss = mseloss + total_variation_loss
detlossval += detloss.item()
steps += 1
ground_truth_centers = get_predected_centers(target_centers)
colored_images = get_colored_image(detected)
det_val_acc += np.average(
det_metrics(ground_truth_centers, colored_images,
color_classes)[0])
for images, targets in validate_loader_segmentation:
model.eval()
with torch.no_grad():
images = images.to(avDev)
targets = targets.to(avDev)
segmented, detected = model(images)
segmentedLabels = torch.argmax(segmented, dim=1)
tvLoss = TVLossSegment(tvweightsegmentation)
total_variation_loss = tvLoss.forward(segmented)
entropy_loss = criterionSegmented(segmented, targets.long())
segloss = entropy_loss + total_variation_loss
seglossval += segloss.item()
accuracies = segmentationAccuracy(segmentedLabels.long(), targets,
[0, 1, 2])
seg_val_acc += accuracies[3]
det_val_acc /= len(validate_loader_detection)
seg_val_acc /= len(validate_loader_segmentation)
detlossval /= len(validate_loader_detection)
seglossval /= len(validate_loader_segmentation)
valdetlosses.append(detlossval)
valseglosses.append(seglossval)
print("Epoch: ", epoch, " Finished")
print("Epoch: ", epoch, " Detection Train Accuracy: ", det_train_acc,
" Detection Validation Accuracy: ", det_val_acc,
"Detection Loss Train:", detlosstrain)
print("Epoch: ", epoch, " Segmentation Train Accuracy: ",
det_train_acc, " Segmentation Validation Accuracy: ",
seg_val_acc, "Segmentation Train Accuracy:", seglosstrain)
acc = (det_val_acc + seg_val_acc) / 2
if acc > best_acc:
val_counter = 0
best_acc = acc
print("New Best Validation Accuracy Detection: ", det_val_acc,
" Segmentation: ", seg_val_acc)
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'detlosses': detlosses,
'seglosses': seglosses,
'valdetlosses': valdetlosses,
'valseglosses': valseglosses
}, checkpoint_path)
elif val_counter < val_patience:
val_counter += 1
print("Best Validation Accuracy Not Updated ")
else:
print("Full Patience Reached ")
break
epoch += 1
count = 0
num = 0
plot_learning_curve(detlosses, valdetlosses, "detection")
plot_learning_curve(seglosses, valseglosses, "segmentation")
det_test_metric = np.zeros((5, len(color_classes)))
confusiondet = np.zeros((4, 4))
confusionseg = np.zeros((3, 3))
print("Training finished starting with testdatatset")
print("Starting with detection")
for images, targets, target_center in test_loader_detection:
count += 1
model.eval()
with torch.no_grad():
images = images.to(avDev)
targets = targets.to(avDev)
segmented, detected = model(images)
tvLoss = TVLossDetect(tvweightdetection)
total_variation_loss = tvLoss.forward(detected)
mseloss = criterionDetected(detected, targets)
loss = mseloss + total_variation_loss
print("Detection Test loss: ", loss.item())
ground_truth_centers = get_predected_centers(target_center)
colored_images = get_colored_image(detected)
det_test_metric += det_metrics(ground_truth_centers,
colored_images, color_classes)
confusiondet += det_confusion_matrix(ground_truth_centers,
colored_images, color_classes)
if (saveImages):
for j in range(batchSize):
num += 1
showImagesDetection(images[j], num)
showDetectedImages(detected[j], num, "output")
showDetectedImages(targets[j], num, "truth")
det_test_metric /= len(test_loader_detection)
plot_confusion_matrix(confusiondet, "detection")
plot_confusion_matrix(confusiondet, "detection")
print('Test Detection Overall Accuracy: {}.',
np.average(det_test_metric[0]))
print('Ball Accuracy:', det_test_metric[0][0])
print('Goal Pillar Accuracy:', det_test_metric[0][1])
print('Robot Accuracy:', det_test_metric[0][2])
print('Test Detection Overall Recall: {}.', np.average(det_test_metric[1]))
print('Ball Recall:', det_test_metric[1][0])
print('Goal Pillar Recall:', det_test_metric[1][1])
print('Robot Recall:', det_test_metric[1][2])
print('Test Detection Overall Precision: {}.',
np.average(det_test_metric[2]))
print('Ball Precision:', det_test_metric[2][0])
print('Goal Pillar Precision:', det_test_metric[2][1])
print('Robot Precision:', det_test_metric[2][2])
print('Test Detection Overall F1score: {}.',
np.average(det_test_metric[3]))
print('Ball F1score:', det_test_metric[3][0])
print('Goal Pillar F1score:', det_test_metric[3][1])
print('Robot F1score:', det_test_metric[3][2])
print('Test Detection Overall False Rate: {}.',
np.average(det_test_metric[4]))
print('Ball False Rate:', det_test_metric[4][0])
print('Goal Pillar False Rate:', det_test_metric[4][1])
print('Robot False Rate:', det_test_metric[4][2])
accuracies = [0, 0, 0, 0]
iou = [0, 0, 0, 0]
num = 0
count = 0
for images, targets in test_loader_segmentation:
count += 1
model.eval()
with torch.no_grad():
images = images.to(avDev)
targets = targets.to(avDev)
segmented, detected = model(images)
segmentedLabels = torch.argmax(segmented, dim=1)
tvLoss = TVLossSegment(tvweightsegmentation)
total_variation_loss = tvLoss.forward(segmented)
entropy_loss = criterionSegmented(segmented, targets.long())
loss = entropy_loss + total_variation_loss
print("Segmentation Test loss: ", loss.item())
confusionseg = seg_confusion_matrix(targets, segmentedLabels)
accuracies_returned = segmentationAccuracy(segmentedLabels.long(),
targets, [0, 1, 2])
iou_returned = seg_iou(targets, segmentedLabels.long(), [0, 1, 2])
accuracies = list(map(add, accuracies, accuracies_returned))
iou = list(map(add, iou, iou_returned))
if (saveImages):
for j in range(batchSize):
num += 1
showImagesSegmentation(images[j], num)
visualiseSegmented(segmentedLabels[j], num, "output")
visualiseSegmented(targets[j], num, "truth")
plot_confusion_matrix(confusionseg, "segmentation")
print('Test Segmentation Accuracy: {}.', accuracies[3] / count)
print('Field Accuracy: ', accuracies[2] / count)
print('Line Accuracy: ', accuracies[1] / count)
print('Background Accuracy: ', accuracies[0] / count)
print('Iou: ', iou[3] / count)
print('Iou Field: ', iou[2] / count)
print('Iou Line: ', iou[1] / count)
print('Iou Background: ', iou[0] / count)
loss = mseloss + total_variation_loss
print("Detection Test loss: ", loss.item())
ground_truth_centers = get_predected_centers(target_center)
colored_images = get_colored_image(detected)
det_test_metric += det_metrics(ground_truth_centers, colored_images,
color_classes)
confusiondet += det_confusion_matrix(ground_truth_centers,
colored_images, color_classes)
for j in range(batchSize):
num += 1
showImagesDetection(images[j], num)
showDetectedImages(detected[j], num, "output")
showDetectedImages(targets[j], num, "truth")
det_test_metric /= len(test_loader_detection)
plot_confusion_matrix(confusiondet, "detection")
print('Test Detection Overall Accuracy: {}.', np.average(det_test_metric[0]))
print('Ball Accuracy:', det_test_metric[0][0])
print('Goal Pillar Accuracy:', det_test_metric[0][1])
print('Robot Accuracy:', det_test_metric[0][2])
print('Test Detection Overall Recall: {}.', np.average(det_test_metric[1]))
print('Ball Recall:', det_test_metric[1][0])
print('Goal Pillar Recall:', det_test_metric[1][1])
print('Robot Recall:', det_test_metric[1][2])
print('Test Detection Overall Precision: {}.', np.average(det_test_metric[2]))
print('Ball Precision:', det_test_metric[2][0])
print('Goal Pillar Precision:', det_test_metric[2][1])
print('Robot Precision:', det_test_metric[2][2])
