def record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects):
if not os.path.exists(workplace + 'Classification/' + 'Result/' + dB +
'/'):
os.mkdir(workplace + 'Classification/' + 'Result/' + dB + '/')
with open(workplace + 'Classification/' + 'Result/' + dB + '/sub_CT.txt',
'a') as csvfile:
thewriter = csv.writer(csvfile, delimiter=' ')
thewriter.writerow('Sub ' + str(sub + 1))
thewriter = csv.writer(csvfile, dialect=csv.excel_tab)
for row in ct:
thewriter.writerow(row)
thewriter.writerow(order)
thewriter.writerow('\n')
if sub == subjects - 1:
# compute the accuracy, F1, P and R from the overall CT
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, p, r] = fpr(tot_mat, n_exp)
print(tot_mat)
print("F1-Score: " + str(f1))
# save into a .txt file
with open(
workplace + 'Classification/' + 'Result/' + dB +
'/final_CT.txt', 'w') as csvfile:
thewriter = csv.writer(csvfile, dialect=csv.excel_tab)
for row in tot_mat:
thewriter.writerow(row)
thewriter = csv.writer(csvfile, delimiter=' ')
thewriter.writerow('micro:' + str(microAcc))
thewriter.writerow('F1:' + str(f1))
thewriter.writerow('Precision:' + str(p))
thewriter.writerow('Recall:' + str(r))
model = load_model(filepath, compile=False)
predict = model.predict(testing_set, batch_size=16)
#print(predict)
print("cm subject" + str(sub))
testing_labels = numpy.argmax(testing_labels, axis=1)
predict = numpy.argmax(predict, axis=1)
ct = confusion_matrix(testing_labels, predict)
print(ct)
# check the order of the CT
order = numpy.unique(numpy.concatenate((predict, testing_labels)))
# create an array to hold the CT for each CV
mat = numpy.zeros((3, 3))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
print(tot_mat)
if sub == 20:
#microAcc = numpy.trace(tot_mat) / numpy.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, 3)
war = weighted_average_recall(tot_mat, 3, 164)
uar = unweighted_average_recall(tot_mat, 3)
print("f1:" + str(f1))
print("war: " + str(war))
print("uar: " + str(uar))
print("cm:")
print(tot_mat)
def test_samm(batch_size, spatial_epochs, temporal_epochs, train_id, dB,
spatial_size, flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/viprlab/01D31FFEF66D5170/"
workplace = root_db_path + dB + "/"
inputDir = root_db_path + dB + "/" + dB + "/"
######################################################
classes = 5
if dB == 'CASME2_TIM':
table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 2
n_exp = 5
# VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
listOfIgnoredSamples = []
VidPerSubject = [2, 1]
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
############################################
os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'CASME2_Optical':
table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 26
n_exp = 5
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 9
data_dim = r * w
pad_sequence = 9
channel = 3
############################################
# os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'SAMM_TIM10':
table, table_objective = loading_samm_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 29
n_exp = 8
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
classes = 8
#########################################################
elif dB == 'SAMM_CASME_Optical':
# total amount of videos 253
table, table_objective = loading_samm_table(root_db_path, dB)
table = table_objective
table2 = loading_casme_objective_table(root_db_path, dB)
# merge samm and casme tables
table = np.concatenate((table, table2), axis=1)
# print(table.shape)
# listOfIgnoredSamples, IgnoredSamples_index, sub_items = ignore_casme_samples(inputDir)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
sub_items = np.empty([0])
list_samples = filter_objective_samples(table)
r = w = spatial_size
subjects = 47 # some subjects were removed because of objective classes and ignore samples: 47
n_exp = 5
# TODO:
# 1) Further decrease the video amount, the one with objective classes >= 6
# list samples: samples with wanted objective class
VidPerSubject, list_samples = get_subfolders_num_crossdb(
inputDir, IgnoredSamples_index, sub_items, table, list_samples)
# print(VidPerSubject)
# print(len(VidPerSubject))
# print(sum(VidPerSubject))
timesteps_TIM = 9
data_dim = r * w
channel = 3
classes = 5
if os.path.isfile(workplace +
"Classification/SAMM_CASME_Optical_label.txt"):
os.remove(workplace +
"Classification/SAMM_CASME_Optical_label.txt")
##################### Variables ######################
######################################################
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
#########################################
############ Reading Images and Labels ################
SubperdB = Read_Input_Images_SAMM_CASME(inputDir, list_samples,
listOfIgnoredSamples, dB,
resizedFlag, table, workplace,
spatial_size, channel)
print("Loaded Images into the tray...")
labelperSub = label_matching(workplace, dB, subjects, VidPerSubject)
print("Loaded Labels into the tray...")
if channel_flag == 1:
SubperdB_strain = Read_Input_Images_SAMM_CASME(
inputDir, list_samples, listOfIgnoredSamples, 'SAMM_CASME_Strain',
resizedFlag, table, workplace, spatial_size, 1)
elif channel_flag == 2:
SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 1)
SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
#######################################################
########### Model Configurations #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
adam2 = optimizers.Adam(lr=0.00075, decay=0.0001)
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
else:
data_dim = 4096
########################################################
########### Training Process ############
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
# model checkpoint
spatial_weights_name = 'vgg_spatial_' + str(train_id) + '_casme2_'
temporal_weights_name = 'temporal_ID_' + str(train_id) + '_casme2_'
history = LossHistory()
stopping = EarlyStopping(monitor='loss',
min_delta=0,
mode='min',
patience=5)
# model checkpoint
if os.path.isdir('/media/viprlab/01D31FFEF66D5170/Weights/' +
str(train_id)) == False:
os.mkdir('/media/viprlab/01D31FFEF66D5170/Weights/' + str(train_id))
for sub in range(subjects):
# sub = sub + 25
# if sub > subjects:
# break
############### Reinitialization & weights reset of models ########################
spatial_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str(
train_id) + '/vgg_spatial_' + str(train_id) + '_' + str(
dB) + '_' + str(sub) + '.h5'
spatial_weights_name_strain = '/media/viprlab/01D31FFEF66D5170/Weights/' + str(
train_id) + '/vgg_spatial_strain_' + str(train_id) + '_' + str(
dB) + '_' + str(sub) + '.h5'
temporal_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str(
train_id) + '/temporal_ID_' + str(train_id) + '_' + str(
dB) + '_' + str(sub) + '.h5'
ae_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str(
train_id) + '/autoencoder_' + str(train_id) + '_' + str(
dB) + '_' + str(sub) + '.h5'
ae_weights_name_strain = '/media/viprlab/01D31FFEF66D5170/Weights/' + str(
train_id) + '/autoencoder_strain_' + str(train_id) + '_' + str(
dB) + '_' + str(sub) + '.h5'
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
weights_path=temporal_weights_name)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
if channel_flag == 1 or channel_flag == 2:
vgg_model = VGG_16_4_channels(spatial_size=spatial_size,
weights_path=spatial_weights_name)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
else:
vgg_model = VGG_16(spatial_size=spatial_size,
weights_path='VGG_Face_Deep_16.h5')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO(
sub, SubperdB, labelperSub, subjects)
# Rearrange Training labels into a vector of images, breaking sequence
Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r,
w, channel)
Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w,
channel)
# Special Loading for 4-Channel
if channel_flag == 1:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Test_X_Strain = Test_X_Strain.reshape(
Test_X.shape[0] * timesteps_TIM, r, w, 1)
# Concatenate Train X & Train_X_Strain
Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain),
axis=3)
Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain),
axis=3)
total_channel = 4
elif channel_flag == 2:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects, classes)
Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO(
sub, SubperdB_gray, labelperSub, subjects)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Test_X_Strain = Test_X_Strain.reshape(
Test_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Train_X_gray = Train_X_gray.reshape(
Train_X_gray.shape[0] * timesteps_TIM, r, w, 3)
Test_X_gray = Test_X_gray.reshape(
Test_X_gray.shape[0] * timesteps_TIM, r, w, 3)
# Concatenate Train_X_Strain & Train_X & Train_X_gray
Train_X_spatial = np.concatenate(
(Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3)
Test_X_spatial = np.concatenate(
(Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3)
total_channel = 7
if channel == 1:
# Duplicate channel of input image
Train_X_spatial = duplicate_channel(Train_X_spatial)
Test_X_spatial = duplicate_channel(Test_X_spatial)
# Extend Y labels 10 fold, so that all images have labels
Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0)
Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0)
print("Train_X_shape: " + str(np.shape(Train_X_spatial)))
print("Train_Y_shape: " + str(np.shape(Train_Y_spatial)))
print("Test_X_shape: " + str(np.shape(Test_X_spatial)))
print("Test_Y_shape: " + str(np.shape(Test_Y_spatial)))
# print(Train_X_spatial)
##################### Training & Testing #########################
X = Train_X_spatial.reshape(Train_X_spatial.shape[0], total_channel, r,
w)
y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes)
normalized_X = X.astype('float32') / 255.
test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], total_channel,
r, w)
test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X = test_X.astype('float32') / 255.
print(X.shape)
###### conv weights must be freezed for transfer learning ######
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
for layer in vgg_model_cam.layers[:31]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# record f1 and loss
file_loss = open(
workplace + 'Classification/' + 'Result/' + dB + '/loss_' +
str(train_id) + '.txt', 'a')
file_loss.write(str(history.losses) + "\n")
file_loss.close()
file_loss = open(
workplace + 'Classification/' + 'Result/' + dB + '/accuracy_' +
str(train_id) + '.txt', 'a')
file_loss.write(str(history.accuracy) + "\n")
file_loss.close()
model = Model(inputs=vgg_model.input,
outputs=vgg_model.layers[35].output)
plot_model(model,
to_file="spatial_module_FULL_TRAINING.png",
show_shapes=True)
# Testing
output = model.predict(test_X, batch_size=batch_size)
features = output.reshape(Test_X.shape[0], timesteps_TIM,
output.shape[1])
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects)
###############################################################################
# check the order of the CT
order = np.unique(np.concatenate((predict, test_y)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(workplace + 'Classification/' + 'Result/' + dB + '/f1.txt',
'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
if not os.path.exists(workplace + 'Classification/' + 'Result/' + dB +
'/'):
os.mkdir(workplace + 'Classification/' + 'Result/' + dB + '/')
with open(workplace + 'Classification/' + 'Result/' + dB + '/sub_CT.txt',
'a') as csvfile:
thewriter = csv.writer(csvfile, delimiter=' ')
def train_apex(batch_size, spatial_epochs, train_id, list_dB, spatial_size, objective_flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ostalo/MihaGarafolj/ME_data/"
#root_db_path = '/home/miha/Documents/ME_data/'
tensorboard_path = root_db_path + "tensorboard/"
if os.path.isdir(root_db_path + 'Weights/'+ str(train_id) ) == False:
os.mkdir(root_db_path + 'Weights/'+ str(train_id) )
######################################################
############## Variables ###################
dB = list_dB[0]
r, w, subjects, samples, n_exp, VidPerSubject, vidList, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples, db_home, db_images, cross_db_flag = load_db(root_db_path, list_dB, spatial_size, objective_flag)
# avoid confusion
if cross_db_flag == 1:
list_samples = listOfIgnoredSamples
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
history = LossHistory()
stopping = EarlyStopping(monitor='loss', min_delta = 0, mode = 'min', patience = 5)
############################################
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
#svm_flag = 0
#finetuning_flag = 0
cam_visualizer_flag = 0
#channel_flag = 0
#########################################
############ Reading Images and Labels ################
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(db_images, list_samples, listOfIgnoredSamples, dB, resizedFlag, table, db_home, spatial_size, channel)
else:
SubperdB = Read_Input_Images_Apex(db_images, listOfIgnoredSamples, dB, resizedFlag, table, db_home, spatial_size, channel, objective_flag)
labelperSub = label_matching(db_home, dB, subjects, VidPerSubject)
print(labelperSub)
print("Loaded Images into the tray.")
print("Loaded Labels into the tray.")
#######################################################
# PREPROCESSING STEPS
# optical flow
#import ipdb
#ipdb.set_trace()
#SubperdB = optical_flow_2d(SubperdB, samples, r, w, timesteps_TIM)
########### Model Configurations #######################
#K.set_image_dim_ordering('th')
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.8
# K.tensorflow_backend.set_session(tf.Session(config=config))
########################################################
print("Beginning training process.")
########### Training Process ############
subjects_todo = read_subjects_todo(db_home, dB, train_id, subjects)
for sub in subjects_todo:
print("**** starting subject " + str(sub) + " ****")
# gpu_observer()
#spatial_weights_name = root_db_path + 'Weights/'+ str(train_id) + '/c3d_'+ str(train_id) + '_' + str(dB) + '_'
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
############### Reinitialization & weights reset of models ########################
apex_model = apex_cnn(spatial_size=spatial_size, temporal_size=timesteps_TIM, classes=n_exp, channels=2, filters=16)
apex_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy])
#svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path2 = tensorboard_path + str(train_id) + str(sub) + "apex/"
if os.path.exists(cat_path2):
os.rmdir(cat_path2)
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2, write_graph=True)
#############################################
Train_X, Train_Y, Train_Y_gt, Test_X, Test_Y, Test_Y_gt = restructure_data_apex(sub, SubperdB, labelperSub, subjects, n_exp, r, w, timesteps_TIM, channel)
#Train_X, Train_Y, Train_Y_gt = upsample_training_set(Train_X, Train_Y, Train_Y_gt)
#Train_X, Test_X = optical_flow_2d_old(Train_X, Test_X, r, w, timesteps_TIM)
############### check gpu resources ####################
# gpu_observer()
########################################################
print("Beginning training & testing.")
##################### Training & Testing #########################
print("Beginning c3d training.")
# Spatial Training
input_u = Train_X[:,0,:,:].reshape(Train_X.shape[0],1,r,w)
input_v = Train_X[:,1,:,:].reshape(Train_X.shape[0],1,r,w)
if tensorboard_flag == 1:
apex_model.fit([input_u, input_v], Train_Y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[history,stopping,tbCallBack2])
else:
apex_model.fit([input_u, input_v], Train_Y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[history,stopping])
print(".record f1 and loss")
# record f1 and loss
record_loss_accuracy(db_home, train_id, dB, history)
print("Beginning testing.")
print(".predicting with c3d_model")
# Testing
input_u = Test_X[:,0,:,:].reshape(Test_X.shape[0],1,r,w)
input_v = Test_X[:,1,:,:].reshape(Test_X.shape[0],1,r,w)
predict_values = apex_model.predict([input_u, input_v], batch_size = batch_size)
predict = np.array([np.argmax(x) for x in predict_values])
##############################################################
#################### Confusion Matrix Construction #############
print (predict)
print (Test_Y_gt.astype(int))
print(".writing predicts to file")
file = open(db_home+'Classification/'+ 'Result/'+'/predicts_' + str(train_id) + '.txt', 'a')
for i in range(len(vidList[sub])):
file.write("sub_" + str(sub) + "," + str(vidList[sub][i]) + "," + str(predict.astype(list)[i]) + "," + str(Test_Y_gt.astype(int).astype(list)[i]) + "\n")
file.close()
file = open(db_home+'Classification/'+ 'Result/'+'/predictedvalues_' + str(train_id) + '.txt', 'a')
for i in range(len(vidList[sub])):
file.write("sub_" + str(sub) + "," + str(vidList[sub][i]) + "," + ','.join(str(e) for e in predict_values[i]) + "," + str(Test_Y_gt.astype(int).astype(list)[i]) + "\n")
file.close()
ct = confusion_matrix(Test_Y_gt,predict)
# check the order of the CT
order = np.unique(np.concatenate((predict,Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp,n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]),int(order[n])]=ct[m,n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1,precision,recall] = fpr(tot_mat,n_exp)
file = open(db_home+'Classification/'+ 'Result/'+'/f1_' + str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(db_home, dB, ct, sub, order, tot_mat, n_exp, subjects)
war = weighted_average_recall(tot_mat, n_exp, samples)
uar = unweighted_average_recall(tot_mat, n_exp)
print("war: " + str(war))
print("uar: " + str(uar))
###############################################################################
################## free memory ####################
del apex_model
del Train_X, Test_X, Train_Y, Test_Y, input_u, input_v
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
gc.collect()
os.mkdir(workplace + 'Classification/' + 'Result/' + dB + '/')
with open(workplace + 'Classification/' + 'Result/' + dB + '/sub_CT.txt',
'a') as csvfile:
thewriter = csv.writer(csvfile, delimiter=' ')
thewriter.writerow('Sub ' + str(sub + 1))
thewriter = csv.writer(csvfile, dialect=csv.excel_tab)
for row in ct:
thewriter.writerow(row)
thewriter.writerow(order)
thewriter.writerow('\n')
if sub == subjects - 1:
# compute the accuracy, F1, P and R from the overall CT
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, p, r] = fpr(tot_mat, n_exp)
# save into a .txt file
with open(
workplace + 'Classification/' + 'Result/' + dB +
'/final_CT.txt', 'w') as csvfile:
thewriter = csv.writer(csvfile, dialect=csv.excel_tab)
for row in tot_mat:
thewriter.writerow(row)
thewriter = csv.writer(csvfile, delimiter=' ')
thewriter.writerow('micro:' + str(microAcc))
thewriter.writerow('F1:' + str(f1))
thewriter.writerow('Precision:' + str(p))
thewriter.writerow('Recall:' + str(r))
outputs = net(inputs1, seg_map, inputs2, seg_map)
_, predicted = torch.max(outputs.data, 1)
total += targets.size(0)
correct += predicted.eq(targets.data).cpu().sum()
print(predicted.eq(targets.data).cpu().sum())
y_true.append(targets.cpu().numpy())
y_pre.append(predicted.cpu().numpy())
# calculate accuracy
print('total:', total, 'correct:', correct)
Test_acc = 100 * float(correct) / total
print('Test accuracy: %0.6f' % Test_acc)
# confusion matrix
y_true = np.concatenate(y_true, axis=0)
y_pre = np.concatenate(y_pre, axis=0)
cm = confusion_matrix_excel(y_true=y_true, y_pred=y_pre)
print(cm)
####################
f1_score, precision, recall = fpr(cm, n_exp=5)
print('f1_score:', f1_score, 'precision:', precision, 'recall:', recall)
WAR = weighted_average_recall(cm, n_exp=5, class_num_list=[32, 63, 27, 25, 99])
UAR = unweighted_average_recall(cm, n_exp=5)
plot_Matrix(cm, classes=5, title=None, cmap=plt.cm.Blues)
def train_vgg_lstm(batch_size, spatial_epochs, temporal_epochs, train_id,
list_dB, spatial_size, flag, objective_flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ostalo/MihaGarafolj/ME_data/"
tensorboard_path = root_db_path + "tensorboard/"
if os.path.isdir(root_db_path + 'Weights/' + str(train_id)) == False:
os.mkdir(root_db_path + 'Weights/' + str(train_id))
######################################################
############## Variables ###################
dB = list_dB[0]
r, w, subjects, samples, n_exp, VidPerSubject, vidList, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples, db_home, db_images, cross_db_flag = load_db(
root_db_path, list_dB, spatial_size, objective_flag)
# avoid confusion
if cross_db_flag == 1:
list_samples = listOfIgnoredSamples
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
history = LossHistory()
stopping = EarlyStopping(monitor='loss',
min_delta=0,
mode='min',
patience=3)
############################################
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4se' or flag == 'st4se_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7se' or flag == 'st7se_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
elif flag == 'st4te' or flag == 'st4te_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 3
elif flag == 'st7te' or flag == 'st7te_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 4
#########################################
############ Reading Images and Labels ################
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(db_images, list_samples,
listOfIgnoredSamples, dB,
resizedFlag, table, db_home,
spatial_size, channel)
else:
SubperdB = Read_Input_Images(db_images, listOfIgnoredSamples, dB,
resizedFlag, table, db_home, spatial_size,
channel, objective_flag)
labelperSub = label_matching(db_home, dB, subjects, VidPerSubject)
print("Loaded Images into the tray.")
print("Loaded Labels into the tray.")
#######################################################
# PREPROCESSING STEPS
# optical flow
#SubperdB = optical_flow_2d(SubperdB, samples, r, w, timesteps_TIM)
gc.collect()
if channel_flag == 1:
aux_db1 = list_dB[1]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 1)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 1,
objective_flag)
elif channel_flag == 2:
aux_db1 = list_dB[1]
aux_db2 = list_dB[2]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
db_gray_img = root_db_path + aux_db2 + "/" + aux_db2 + "/"
if cross_db_flag == 1:
SubperdB_strain = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 1)
SubperdB_gray = Read_Input_Images_SAMM_CASME(
db_gray_img, list_samples, listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home, spatial_size, 1)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 1,
objective_flag)
SubperdB_gray = Read_Input_Images(db_gray_img,
listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home,
spatial_size, 1, objective_flag)
elif channel_flag == 3:
aux_db1 = list_dB[1]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 3)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 3,
objective_flag)
elif channel_flag == 4:
aux_db1 = list_dB[1]
aux_db2 = list_dB[2]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
db_gray_img = root_db_path + aux_db2 + "/" + aux_db2 + "/"
if cross_db_flag == 1:
SubperdB_strain = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 3)
SubperdB_gray = Read_Input_Images_SAMM_CASME(
db_gray_img, list_samples, listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home, spatial_size, 3)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 3,
objective_flag)
SubperdB_gray = Read_Input_Images(db_gray_img,
listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home,
spatial_size, 3, objective_flag)
#######################################################
########### Model Configurations #######################
#K.set_image_dim_ordering('th')
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.8
# K.tensorflow_backend.set_session(tf.Session(config=config))
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
elif channel_flag == 3:
data_dim = 8192
elif channel_flag == 4:
data_dim = 12288
else:
data_dim = 4096
########################################################
print("Beginning training process.")
########### Training Process ############
subjects_todo = read_subjects_todo(db_home, dB, train_id, subjects)
for sub in subjects_todo:
#sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
print("**** starting subject " + str(sub) + " ****")
#gpu_obgpu_observer()
spatial_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_' + str(train_id) + '_' + str(dB) + '_'
spatial_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_strain_' + str(train_id) + '_' + str(
dB) + '_'
spatial_weights_name_gray = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_gray_' + str(train_id) + '_' + str(
dB) + '_'
temporal_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/temporal_ID_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_strain_' + str(train_id) + '_' + str(
dB) + '_'
############### Reinitialization & weights reset of models ########################
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
lstm1_size=3000,
classes=n_exp)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
if channel_flag == 1:
vgg_model = VGG_16_4_channels(classes=n_exp,
channels=4,
spatial_size=spatial_size)
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
elif channel_flag == 2:
vgg_model = VGG_16_4_channels(classes=n_exp,
channels=5,
spatial_size=spatial_size)
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
elif channel_flag == 3 or channel_flag == 4:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
vgg_model_strain = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
for layer in vgg_model_strain.layers[:33]:
layer.trainable = False
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
vgg_model_strain.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
if channel_flag == 4:
vgg_model_gray = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
if finetuning_flag == 1:
for layer in vgg_model_gray.layers[:33]:
layer.trainable = False
vgg_model_gray.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
else:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=channel,
channel_first=False,
weights_path='VGG_Face_Deep_16.h5')
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
#svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path = tensorboard_path + str(train_id) + str(sub) + "/"
if os.path.exists(cat_path):
os.rmdir(cat_path)
os.mkdir(cat_path)
tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path,
write_graph=True)
cat_path2 = tensorboard_path + str(train_id) + str(
sub) + "spatial/"
if os.path.exists(cat_path2):
os.rmdir(cat_path2)
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2,
write_graph=True)
#############################################
#import ipdb; ipdb.set_trace()
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt, X, y, test_X, test_y = restructure_data(
sub, SubperdB, labelperSub, subjects, n_exp, r, w, timesteps_TIM,
channel)
# Special Loading for 4-Channel
if channel_flag == 1:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
# verify
# sanity_check_image(Test_X_Strain, 1, spatial_size)
# Concatenate Train X & Train_X_Strain
X = np.concatenate((X, Train_X_Strain), axis=1)
test_X = np.concatenate((test_X, Test_X_Strain), axis=1)
total_channel = 4
elif channel_flag == 2:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
_, _, _, _, _, Train_X_Gray, Train_Y_Gray, Test_X_Gray, Test_Y_Gray = restructure_data(
sub, SubperdB_gray, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
# Concatenate Train_X_Strain & Train_X & Train_X_Gray
X = np.concatenate((X, Train_X_Strain, Train_X_Gray), axis=1)
test_X = np.concatenate((test_X, Test_X_Strain, Test_X_Gray),
axis=1)
total_channel = 5
elif channel_flag == 3:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
elif channel_flag == 4:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
_, _, _, _, _, Train_X_Gray, Train_Y_Gray, Test_X_Gray, Test_Y_Gray = restructure_data(
sub, SubperdB_gray, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
############### check gpu resources ####################
#gpu_observer()
########################################################
print("Beginning training & testing.")
##################### Training & Testing #########################
if train_spatial_flag == 1 and train_temporal_flag == 1:
print("Beginning spatial training.")
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping, tbCallBack2])
elif channel_flag == 3 or channel_flag == 4:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping])
vgg_model_strain.fit(Train_X_Strain,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[stopping])
model_strain = record_weights(vgg_model_strain,
spatial_weights_name_strain, sub,
flag)
output_strain = model_strain.predict(Train_X_Strain,
batch_size=batch_size)
if channel_flag == 4:
vgg_model_gray.fit(Train_X_Gray,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[stopping])
model_gray = record_weights(vgg_model_gray,
spatial_weights_name_gray, sub,
flag)
output_gray = model_gray.predict(Train_X_Gray,
batch_size=batch_size)
else:
#import ipdb; ipdb.set_trace()
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping])
print(".record f1 and loss")
# record f1 and loss
record_loss_accuracy(db_home, train_id, dB, history)
print(".save vgg weights")
# save vgg weights
# model = record_weights(vgg_model, spatial_weights_name, sub, flag)
print(".spatial encoding")
# Spatial Encoding
model_int = Model(inputs=vgg_model.input,
outputs=vgg_model.get_layer('dense_prvi').output)
#model = record_weights(cnn_model, spatial_weights_name, sub, flag)
features = model_int.predict(X, batch_size=batch_size)
features = features.reshape(int(Train_X.shape[0]), timesteps_TIM,
features.shape[1])
# concatenate features for temporal enrichment
if channel_flag == 3:
output = np.concatenate((output, output_strain), axis=1)
elif channel_flag == 4:
output = np.concatenate((output, output_strain, output_gray),
axis=1)
print("Beginning temporal training.")
# Temporal Training
if tensorboard_flag == 1:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs,
callbacks=[tbCallBack])
else:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs)
print(".save temportal weights")
# save temporal weights
temporal_model = record_weights(temporal_model,
temporal_weights_name, sub,
't') # let the flag be t
print("Beginning testing.")
print(".predicting with spatial model")
# Testing
features = model_int.predict(test_X, batch_size=batch_size)
features = features.reshape(int(Test_X.shape[0]), timesteps_TIM,
features.shape[1])
if channel_flag == 3 or channel_flag == 4:
output_strain = model_strain.predict(Test_X_Strain,
batch_size=batch_size)
if channel_flag == 4:
output_gray = model_gray.predict(Test_X_Gray,
batch_size=batch_size)
# concatenate features for temporal enrichment
if channel_flag == 3:
output = np.concatenate((output, output_strain), axis=1)
elif channel_flag == 4:
output = np.concatenate((output, output_strain, output_gray),
axis=1)
print(".outputing features")
print(".predicting with temporal model")
predict_values = temporal_model.predict(features,
batch_size=batch_size)
predict = np.array([np.argmax(x) for x in predict_values])
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt.astype(int))
print(".writing predicts to file")
file = open(
db_home + 'Classification/' + 'Result/' + '/predicts_' +
str(train_id) + '.txt', 'a')
for i in range(len(vidList[sub])):
file.write("sub_" + str(sub) + "," + str(vidList[sub][i]) + "," +
str(predict.astype(list)[i]) + "," +
str(Test_Y_gt.astype(int).astype(list)[i]) + "\n")
file.close()
file = open(
db_home + 'Classification/' + 'Result/' + '/predictedvalues_' +
str(train_id) + '.txt', 'a')
for i in range(len(vidList[sub])):
file.write("sub_" + str(sub) + "," + str(vidList[sub][i]) + "," +
','.join(str(e) for e in predict_values[i]) + "," +
str(Test_Y_gt.astype(int).astype(list)[i]) + "\n")
file.close()
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
db_home + 'Classification/' + 'Result/' + '/f1_' + str(train_id) +
'.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(db_home, dB, ct, sub, order, tot_mat, n_exp, subjects)
war = weighted_average_recall(tot_mat, n_exp, samples)
uar = unweighted_average_recall(tot_mat, n_exp)
print("war: " + str(war))
print("uar: " + str(uar))
###############################################################################
################## free memory ####################
del vgg_model
del temporal_model
del model_int
del Train_X, Test_X, X, y
if channel_flag == 1:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Test_Y_Strain
elif channel_flag == 2:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Test_Y_Strain, Train_X_Gray, Test_X_Gray, Train_Y_Gray, Test_Y_Gray
elif channel_flag == 3:
del vgg_model_strain, model_strain
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Test_Y_Strain
elif channel_flag == 4:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Test_Y_Strain, Train_X_Gray, Test_X_Gray, Train_Y_Gray, Test_Y_Gray
del vgg_model_gray, vgg_model_strain, model_gray, model_strain
K.get_session().close()
cfg = K.tf.ConfigProto()
cfg.gpu_options.allow_growth = True
K.set_session(K.tf.Session(config=cfg))
gc.collect()
def test_casme(batch_size, spatial_epochs, temporal_epochs, train_id, dB,
spatial_size, flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ice/OS/Datasets/"
workplace = root_db_path + dB + "/"
inputDir = root_db_path + dB + "/" + dB + "/"
######################################################
classes = 5
if dB == 'CASME2_TIM':
table = loading_casme_table(workplace + 'CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 2
samples = 246
n_exp = 5
# VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
listOfIgnoredSamples = []
VidPerSubject = [2, 1]
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
############################################
os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'CASME2_Optical':
table = loading_casme_table(workplace + 'CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index, _ = ignore_casme_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 26
samples = 246
n_exp = 5
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 9
data_dim = r * w
pad_sequence = 9
channel = 3
############################################
# os.remove(workplace + "Classification/CASME2_TIM_label.txt")
elif dB == 'CASME2_RGB':
# print(inputDir)
table = loading_casme_table(workplace +
'CASME2_RGB/CASME2_label_Ver_2.xls')
listOfIgnoredSamples, IgnoredSamples_index = ignore_casmergb_samples(
inputDir)
############## Variables ###################
r = w = spatial_size
subjects = 26
samples = 245 # not used, delete it later
n_exp = 5
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
############################################
elif dB == 'SMIC_TIM10':
table = loading_smic_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 16
samples = 164
n_exp = 3
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 1
classes = 3
#########################################################
elif dB == 'SAMM_Optical':
table, table_objective = loading_samm_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 29
samples = 159
n_exp = 8
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 9
data_dim = r * w
pad_sequence = 10
channel = 3
classes = 8
#########################################################
elif dB == 'SAMM_TIM10':
table, table_objective = loading_samm_table(root_db_path, dB)
listOfIgnoredSamples = []
IgnoredSamples_index = np.empty([0])
################# Variables #############################
r = w = spatial_size
subjects = 29
samples = 159
n_exp = 8
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
channel = 3
classes = 8
#########################################################
# print(VidPerSubject)
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
elif flag == 'st4vis':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 3
#########################################
############ Reading Images and Labels ################
SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB,
resizedFlag, table, workplace, spatial_size,
channel)
print("Loaded Images into the tray...")
labelperSub = label_matching(workplace, dB, subjects, VidPerSubject)
print("Loaded Labels into the tray...")
if channel_flag == 1:
inputDir = root_db_path + dB + "/" + dB + "/"
SubperdB_strain = Read_Input_Images(
root_db_path + 'CASME2_Strain_TIM10' + '/' +
'CASME2_Strain_TIM10' + '/', listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size,
3)
SubperdB_gray = Read_Input_Images(
root_db_path + 'CASME2_TIM' + '/' + 'CASME2_TIM' + '/',
listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace,
spatial_size, 3)
elif channel_flag == 3:
inputDir_strain = '/media/ice/OS/Datasets/CASME2_Strain_TIM10/CASME2_Strain_TIM10/'
SubperdB_strain = Read_Input_Images(inputDir_strain,
listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 3)
inputDir_gray = '/media/ice/OS/Datasets/CASME2_TIM/CASME2_TIM/'
SubperdB_gray = Read_Input_Images(inputDir_gray, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
elif channel_flag == 2:
SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 1)
SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
#######################################################
########### Model Configurations #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
else:
data_dim = 8192
########################################################
########### Image Data Generator ##############
image_generator = ImageDataGenerator(zca_whitening=True,
rotation_range=0.2,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
rescale=1.5)
###############################################
########### Training Process ############
# Todo:
# 1) LOSO (done)
# 2) call model (done)
# 3) saving model architecture
# 4) Saving Checkpoint (done)
# 5) make prediction (done)
if tensorboard_flag == 1:
tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/"
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
weights_dir = '/media/ice/OS/Datasets/Weights/53/'
image_path = '/home/ice/Documents/Micro-Expression/image/'
table_count = 0
for sub in range(subjects):
############### Reinitialization & weights reset of models ########################
temporal_model_weights = weights_dir + 'temporal_enrichment_ID_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
vgg_model_weights = weights_dir + 'vgg_spatial_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
vgg_model_strain_weights = weights_dir + 'vgg_spatial_strain_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
conv_ae_weights = weights_dir + 'autoencoder_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
conv_ae_strain_weights = weights_dir + 'autoencoder_strain_' + str(
train_id) + '_' + str(dB) + '_' + str(sub) + '.h5'
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
weights_path=temporal_model_weights)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
conv_ae = convolutional_autoencoder(spatial_size=spatial_size,
weights_path=conv_ae_weights)
conv_ae.compile(loss='binary_crossentropy', optimizer=adam)
conv_ae_strain = convolutional_autoencoder(
spatial_size=spatial_size, weights_path=conv_ae_strain_weights)
conv_ae_strain.compile(loss='binary_crossentropy', optimizer=adam)
vgg_model = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path=vgg_model_weights)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
vgg_model_strain = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path=vgg_model_strain_weights)
vgg_model_strain.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO(
sub, SubperdB, labelperSub, subjects, classes)
# Rearrange Training labels into a vector of images, breaking sequence
Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r,
w, channel)
Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w,
channel)
# Special Loading for 4-Channel
if channel_flag == 1 or channel_flag == 3:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects, classes)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 3)
Test_X_Strain = Test_X_Strain.reshape(
Test_X.shape[0] * timesteps_TIM, r, w, 3)
Train_X_Gray, _, Test_X_Gray, _, _ = data_loader_with_LOSO(
sub, SubperdB_gray, labelperSub, subjects, classes)
Test_X_Gray = Test_X_Gray.reshape(Test_X_Gray.shape[0] * 10, r, w,
3)
# print(Train_X_Strain.shape)
# Train_X_Strain = Train_X_Strain[0]
# Train_X_Strain = Train_X_Strain.reshape((224, 224, 3, 1))
# Train_X_Strain = Train_X_Strain.reshape((224, 224, 3))
# cv2.imwrite('steveharvey.png', Train_X_Strain)
# Concatenate Train X & Train_X_Strain
# Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3)
# Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3)
total_channel = 4
# Extend Y labels 10 fold, so that all images have labels
Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0)
Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0)
##################### Training & Testing #########################
# print(Train_X_spatial.shape)
test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w)
test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X = test_X.astype('float32') / 255.
Test_X_Strain = Test_X_Strain.reshape(Test_X_Strain.shape[0], channel,
r, w)
Test_X_Gray = Test_X_Gray.reshape(Test_X_Gray.shape[0], channel, r, w)
# test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X_strain = test_X.astype('float32') / 255.
# print(X.shape)
###### conv weights must be freezed for transfer learning ######
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# vgg
model = Model(inputs=vgg_model.input,
outputs=vgg_model.layers[35].output)
plot_model(model,
to_file="spatial_module_FULL_TRAINING.png",
show_shapes=True)
output = model.predict(test_X)
# vgg strain
model_strain = Model(inputs=vgg_model_strain.input,
outputs=vgg_model_strain.layers[35].output)
plot_model(model_strain,
to_file="spatial_module_FULL_TRAINING_strain.png",
show_shapes=True)
output_strain = model_strain.predict(Test_X_Strain)
# ae
# model_ae = Model(inputs=conv_ae.input, outputs=conv_ae.output)
# plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# output_ae = model_ae.predict(normalized_test_X)
# output_ae = model.predict(output_ae)
# ae strain
# model_ae_strain = Model(inputs=conv_ae_strain.input, outputs=conv_ae_strain.output)
# plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# output_ae_strain = model_ae_strain.predict(normalized_test_X_strain)
# output_ae_strain = model_ae_strain.predict(output_ae_strain)
# concatenate features
output = np.concatenate((output, output_strain), axis=1)
features = output.reshape(int(Test_X.shape[0]), timesteps_TIM,
output.shape[1])
# temporal
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
# visualize cam
countcam = 0
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/log_hde' +
str(train_id) + '.txt', 'a')
file.write(str(sub + 1) + "\n")
for item_idx in range(len(predict)):
test_strain = Test_X_Gray[item_idx + countcam]
test_strain = test_strain.reshape((224, 224, 3))
item = test_strain
cam_output = visualize_cam(model, 29, 0, item)
cam_output2 = visualize_cam(model, 29, 1, item)
cam_output3 = visualize_cam(model, 29, 2, item)
cam_output4 = visualize_cam(model, 29, 3, item)
cam_output5 = visualize_cam(model, 29, 4, item)
overlaying_cam = overlay(item, cam_output)
overlaying_cam2 = overlay(item, cam_output2)
overlaying_cam3 = overlay(item, cam_output3)
overlaying_cam4 = overlay(item, cam_output4)
overlaying_cam5 = overlay(item, cam_output5)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam0.png', overlaying_cam)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam1.png', overlaying_cam2)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam2.png', overlaying_cam3)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam3.png', overlaying_cam4)
cv2.imwrite(
image_path + '_' + str(sub) + '_' + str(item_idx) + '_' +
str(predict[item_idx]) + '_' + str(Test_Y_gt[item_idx]) +
'_coverlayingcam4.png', overlaying_cam5)
countcam += 9
######## write the log file for megc 2018 ############
result_string = table[table_count, 1] + ' ' + str(
int(Test_Y_gt[item_idx])) + ' ' + str(
predict[item_idx]) + '\n'
file.write(result_string)
######################################################
table_count += 1
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# print(type(ct))a
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects)
###############################################################################
tot_mat_cm = np.asarray(tot_mat, dtype=int)
plt.figure()
classes_test = [0, 1, 2, 3, 4]
plot_confusion_matrix(tot_mat_cm,
classes_test,
normalize=True,
title='Confusion matrix_single_db')
plt.show()
def train(batch_size, spatial_epochs, temporal_epochs, train_id, list_dB,
spatial_size, flag, objective_flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ice/OS/Datasets/"
tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/"
if os.path.isdir(root_db_path + 'Weights/' + str(train_id)) == False:
os.mkdir(root_db_path + 'Weights/' + str(train_id))
######################################################
############## Variables ###################
dB = list_dB[0]
r, w, subjects, samples, n_exp, VidPerSubject, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples, db_home, db_images, cross_db_flag = load_db(
root_db_path, list_dB, spatial_size, objective_flag)
# avoid confusion
if cross_db_flag == 1:
list_samples = listOfIgnoredSamples
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
history = LossHistory()
stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min')
############################################
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4se' or flag == 'st4se_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7se' or flag == 'st7se_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
elif flag == 'st4te' or flag == 'st4te_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 3
elif flag == 'st7te' or flag == 'st7te_cde':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 4
#########################################
############ Reading Images and Labels ################
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(db_images, list_samples,
listOfIgnoredSamples, dB,
resizedFlag, table, db_home,
spatial_size, channel)
else:
SubperdB = Read_Input_Images(db_images, listOfIgnoredSamples, dB,
resizedFlag, table, db_home, spatial_size,
channel, objective_flag)
labelperSub = label_matching(db_home, dB, subjects, VidPerSubject)
print("Loaded Images into the tray...")
print("Loaded Labels into the tray...")
if channel_flag == 1:
aux_db1 = list_dB[1]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 1)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 1,
objective_flag)
elif channel_flag == 2:
aux_db1 = list_dB[1]
aux_db2 = list_dB[2]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
db_gray_img = root_db_path + aux_db2 + "/" + aux_db2 + "/"
if cross_db_flag == 1:
SubperdB_strain = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 1)
SubperdB_gray = Read_Input_Images_SAMM_CASME(
db_gray_img, list_samples, listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home, spatial_size, 1)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 1,
objective_flag)
SubperdB_gray = Read_Input_Images(db_gray_img,
listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home,
spatial_size, 1, objective_flag)
elif channel_flag == 3:
aux_db1 = list_dB[1]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 3)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 3,
objective_flag)
elif channel_flag == 4:
aux_db1 = list_dB[1]
aux_db2 = list_dB[2]
db_strain_img = root_db_path + aux_db1 + "/" + aux_db1 + "/"
db_gray_img = root_db_path + aux_db2 + "/" + aux_db2 + "/"
if cross_db_flag == 1:
SubperdB_strain = Read_Input_Images_SAMM_CASME(
db_strain_img, list_samples, listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home, spatial_size, 3)
SubperdB_gray = Read_Input_Images_SAMM_CASME(
db_gray_img, list_samples, listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home, spatial_size, 3)
else:
SubperdB_strain = Read_Input_Images(db_strain_img,
listOfIgnoredSamples, aux_db1,
resizedFlag, table, db_home,
spatial_size, 3,
objective_flag)
SubperdB_gray = Read_Input_Images(db_gray_img,
listOfIgnoredSamples, aux_db2,
resizedFlag, table, db_home,
spatial_size, 3, objective_flag)
#######################################################
########### Model Configurations #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
elif channel_flag == 3:
data_dim = 8192
elif channel_flag == 4:
data_dim = 12288
else:
data_dim = 4096
########################################################
########### Training Process ############
for sub in range(subjects):
spatial_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_' + str(train_id) + '_' + str(dB) + '_'
spatial_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_strain_' + str(train_id) + '_' + str(
dB) + '_'
spatial_weights_name_gray = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_gray_' + str(train_id) + '_' + str(
dB) + '_'
temporal_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/temporal_ID_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_strain_' + str(train_id) + '_' + str(
dB) + '_'
############### Reinitialization & weights reset of models ########################
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
classes=n_exp)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
conv_ae = convolutional_autoencoder(spatial_size=spatial_size,
classes=n_exp)
conv_ae.compile(loss='binary_crossentropy', optimizer=adam)
if channel_flag == 1:
vgg_model = VGG_16_4_channels(classes=n_exp,
channels=4,
spatial_size=spatial_size)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
elif channel_flag == 2:
vgg_model = VGG_16_4_channels(classes=n_exp,
channels=5,
spatial_size=spatial_size)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
elif channel_flag == 3 or channel_flag == 4:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
vgg_model_strain = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
vgg_model_strain.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
if channel_flag == 4:
vgg_model_gray = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
vgg_model_gray.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
else:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path = tensorboard_path + str(sub) + "/"
os.mkdir(cat_path)
tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path,
write_graph=True)
cat_path2 = tensorboard_path + str(sub) + "spat/"
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2,
write_graph=True)
#############################################
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt, X, y, test_X, test_y = restructure_data(
sub, SubperdB, labelperSub, subjects, n_exp, r, w, timesteps_TIM,
channel)
# Special Loading for 4-Channel
if channel_flag == 1:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
# verify
# sanity_check_image(Test_X_Strain, 1, spatial_size)
# Concatenate Train X & Train_X_Strain
X = np.concatenate((X, Train_X_Strain), axis=1)
test_X = np.concatenate((test_X, Test_X_Strain), axis=1)
total_channel = 4
elif channel_flag == 2:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
_, _, _, _, _, Train_X_Gray, Train_Y_Gray, Test_X_Gray, Test_Y_Gray = restructure_data(
sub, SubperdB_gray, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 1)
# Concatenate Train_X_Strain & Train_X & Train_X_gray
X = np.concatenate((X, Train_X_Strain, Train_X_gray), axis=1)
test_X = np.concatenate((test_X, Test_X_Strain, Test_X_gray),
axis=1)
total_channel = 5
elif channel_flag == 3:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
elif channel_flag == 4:
_, _, _, _, _, Train_X_Strain, Train_Y_Strain, Test_X_Strain, Test_Y_Strain = restructure_data(
sub, SubperdB_strain, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
_, _, _, _, _, Train_X_Gray, Train_Y_Gray, Test_X_Gray, Test_Y_Gray = restructure_data(
sub, SubperdB_gray, labelperSub, subjects, n_exp, r, w,
timesteps_TIM, 3)
############### check gpu resources ####################
gpu_observer()
########################################################
##################### Training & Testing #########################
# conv weights must be freezed for transfer learning
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
if channel_flag == 3 or channel_flag == 4:
for layer in vgg_model_strain.layers[:33]:
layer.trainable = False
if channel_flag == 4:
for layer in vgg_model_gray.layers[:33]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[tbCallBack2])
elif channel_flag == 3 or channel_flag == 4:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping])
vgg_model_strain.fit(Train_X_Strain,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[stopping])
model_strain = record_weights(vgg_model_strain,
spatial_weights_name_strain, sub,
flag)
output_strain = model_strain.predict(Train_X_Strain,
batch_size=batch_size)
if channel_flag == 4:
vgg_model_gray.fit(Train_X_Gray,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[stopping])
model_gray = record_weights(vgg_model_gray,
spatial_weights_name_gray, sub,
flag)
output_gray = model_gray.predict(Train_X_Gray,
batch_size=batch_size)
else:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping])
# record f1 and loss
record_loss_accuracy(db_home, train_id, dB, history)
# save vgg weights
model = record_weights(vgg_model, spatial_weights_name, sub, flag)
# Spatial Encoding
output = model.predict(X, batch_size=batch_size)
# concatenate features for temporal enrichment
if channel_flag == 3:
output = np.concatenate((output, output_strain), axis=1)
elif channel_flag == 4:
output = np.concatenate((output, output_strain, output_gray),
axis=1)
features = output.reshape(int(Train_X.shape[0]), timesteps_TIM,
output.shape[1])
# Temporal Training
if tensorboard_flag == 1:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs,
callbacks=[tbCallBack])
else:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs)
# save temporal weights
temporal_model = record_weights(temporal_model,
temporal_weights_name, sub,
't') # let the flag be t
# Testing
output = model.predict(test_X, batch_size=batch_size)
if channel_flag == 3 or channel_flag == 4:
output_strain = model_strain.predict(Test_X_Strain,
batch_size=batch_size)
if channel_flag == 4:
output_gray = model_gray.predict(Test_X_Gray,
batch_size=batch_size)
# concatenate features for temporal enrichment
if channel_flag == 3:
output = np.concatenate((output, output_strain), axis=1)
elif channel_flag == 4:
output = np.concatenate((output, output_strain, output_gray),
axis=1)
features = output.reshape(Test_X.shape[0], timesteps_TIM,
output.shape[1])
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
db_home + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(db_home, dB, ct, sub, order, tot_mat, n_exp, subjects)
war = weighted_average_recall(tot_mat, n_exp, samples)
uar = unweighted_average_recall(tot_mat, n_exp)
print("war: " + str(war))
print("uar: " + str(uar))
###############################################################################
################## free memory ####################
del vgg_model
del temporal_model
del model
del Train_X, Test_X, X, y
if channel_flag == 1:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Train_Y_Strain
elif channel_flag == 2:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Train_Y_Strain, Train_X_Gray, Test_X_Gray, Train_Y_Gray, Test_Y_Gray
elif channel_flag == 3:
del vgg_model_strain, model_strain
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Train_Y_Strain
elif channel_flag == 4:
del Train_X_Strain, Test_X_Strain, Train_Y_Strain, Train_Y_Strain, Train_X_Gray, Test_X_Gray, Train_Y_Gray, Test_Y_Gray
del vgg_model_gray, vgg_model_strain, model_gray, model_strain
gc.collect()
###################################################
def train_cae_lstm(batch_size, spatial_epochs, temporal_epochs, train_id, dB,
spatial_size, flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/media/ice/OS/Datasets/"
workplace = root_db_path + dB + "/"
inputDir = root_db_path + dB + "/" + dB + "/"
######################################################
r, w, subjects, samples, n_exp, VidPerSubject, timesteps_TIM, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples = load_db(
root_db_path, dB, spatial_size)
# print(VidPerSubject)
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
elif flag == 's':
train_spatial_flag = 1
finetuning_flag = 1
elif flag == 't':
train_temporal_flag = 1
elif flag == 'nofine':
svm_flag = 1
elif flag == 'scratch':
train_spatial_flag = 1
train_temporal_flag = 1
elif flag == 'st4':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 1
elif flag == 'st7':
train_spatial_flag = 1
train_temporal_flag = 1
channel_flag = 2
#########################################
############ Reading Images and Labels ################
SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB,
resizedFlag, table, workplace, spatial_size,
channel)
print("Loaded Images into the tray...")
labelperSub = label_matching(workplace, dB, subjects, VidPerSubject)
print("Loaded Labels into the tray...")
if channel_flag == 1:
SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 1)
elif channel_flag == 2:
SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_Strain_TIM10', resizedFlag,
table, workplace, spatial_size, 1)
SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples,
'CASME2_TIM', resizedFlag, table,
workplace, spatial_size, 3)
#######################################################
########### Model Configurations #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
# Different Conditions for Temporal Learning ONLY
if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical':
data_dim = spatial_size * spatial_size
elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical':
data_dim = spatial_size * spatial_size * 3
else:
data_dim = 4096
########################################################
########### Training Process ############
# Todo:
# 1) LOSO (done)
# 2) call model (done)
# 3) saving model architecture
# 4) Saving Checkpoint (done)
# 5) make prediction (done)
if tensorboard_flag == 1:
tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/"
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
# model checkpoint
spatial_weights_name = 'vgg_spatial_' + str(train_id) + '_' + str(dB) + '_'
temporal_weights_name = 'temporal_ID_' + str(train_id) + '_' + str(
dB) + '_'
ae_weights_name = 'autoencoder_' + str(train_id) + '_' + str(dB) + '_'
history = LossHistory()
stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min')
for sub in range(subjects):
############### Reinitialization & weights reset of models ########################
vgg_model_cam = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path='VGG_Face_Deep_16.h5')
temporal_model = temporal_module(data_dim=data_dim,
classes=classes,
timesteps_TIM=timesteps_TIM)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
conv_ae = convolutional_autoencoder(spatial_size=spatial_size,
classes=classes)
conv_ae.compile(loss='binary_crossentropy', optimizer=adam)
if channel_flag == 1 or channel_flag == 2:
vgg_model = VGG_16_4_channels(classes=classes,
spatial_size=spatial_size)
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
else:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=classes,
weights_path='VGG_Face_Deep_16.h5')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
svm_classifier = SVC(kernel='linear', C=1)
####################################################################################
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path = tensorboard_path + str(sub) + "/"
os.mkdir(cat_path)
tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path,
write_graph=True)
cat_path2 = tensorboard_path + str(sub) + "spat/"
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2,
write_graph=True)
#############################################
image_label_mapping = np.empty([0])
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO(
sub, SubperdB, labelperSub, subjects, classes)
# Rearrange Training labels into a vector of images, breaking sequence
Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r,
w, channel)
Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w,
channel)
# Special Loading for 4-Channel
if channel_flag == 1:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects, classes)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Test_X_Strain = Test_X_Strain.reshape(
Test_X.shape[0] * timesteps_TIM, r, w, 1)
# Concatenate Train X & Train_X_Strain
Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain),
axis=3)
Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain),
axis=3)
channel = 4
elif channel_flag == 2:
Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(
sub, SubperdB_strain, labelperSub, subjects, classes)
Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO(
sub, SubperdB_gray, labelperSub, subjects)
Train_X_Strain = Train_X_Strain.reshape(
Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Test_X_Strain = Test_X_Strain.reshape(
Test_X_Strain.shape[0] * timesteps_TIM, r, w, 1)
Train_X_gray = Train_X_gray.reshape(
Train_X_gray.shape[0] * timesteps_TIM, r, w, 3)
Test_X_gray = Test_X_gray.reshape(
Test_X_gray.shape[0] * timesteps_TIM, r, w, 3)
# Concatenate Train_X_Strain & Train_X & Train_X_gray
Train_X_spatial = np.concatenate(
(Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3)
Test_X_spatial = np.concatenate(
(Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3)
channel = 7
if channel == 1:
# Duplicate channel of input image
Train_X_spatial = duplicate_channel(Train_X_spatial)
Test_X_spatial = duplicate_channel(Test_X_spatial)
# Extend Y labels 10 fold, so that all images have labels
Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0)
Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0)
# print ("Train_X_shape: " + str(np.shape(Train_X_spatial)))
# print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial)))
# print ("Test_X_shape: " + str(np.shape(Test_X_spatial)))
# print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial)))
# print(Train_X_spatial)
##################### Training & Testing #########################
# print(Train_X_spatial.shape)
X = Train_X_spatial.reshape(Train_X_spatial.shape[0], channel, r, w)
y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes)
normalized_X = X.astype('float32') / 255.
test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w)
test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes)
normalized_test_X = test_X.astype('float32') / 255.
print(X.shape)
###### conv weights must be freezed for transfer learning ######
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
for layer in vgg_model_cam.layers[:31]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# Autoencoder first training
conv_ae.fit(normalized_X,
normalized_X,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True)
# remove decoder
conv_ae.pop()
conv_ae.pop()
conv_ae.pop()
conv_ae.pop()
conv_ae.pop()
conv_ae.pop()
conv_ae.pop()
# append dense layers
conv_ae.add(Flatten())
conv_ae.add(Dense(4096, activation='relu'))
conv_ae.add(Dense(4096, activation='relu'))
conv_ae.add(Dense(n_exp, activation='sigmoid'))
model_ae = Model(inputs=conv_ae.input,
outputs=conv_ae.layers[9].output)
plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# freeze encoder
for layer in conv_ae.layers[:6]:
layer.trainable = False
# finetune dense layers
conv_ae.compile(loss='categorical_crossentropy', optimizer=adam)
conv_ae.fit(normalized_X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True)
model_ae = Model(inputs=conv_ae.input,
outputs=conv_ae.layers[8].output)
plot_model(model_ae, to_file='autoencoders.png', show_shapes=True)
# Autoencoding
output = model_ae.predict(normalized_X, batch_size=batch_size)
# print(output.shape)
features = output.reshape(int(Train_X.shape[0]), timesteps_TIM,
output.shape[1])
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs)
temporal_model.save_weights(temporal_weights_name + str(sub) +
".h5")
# Testing
output = model_ae.predict(test_X, batch_size=batch_size)
features = output.reshape(Test_X.shape[0], timesteps_TIM,
output.shape[1])
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
workplace + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects)
###############################################################################
model.compile(loss = 'categorical_crossentropy', optimizer =adam, metrics = ['accuracy'])
print(model.summary())
filepath="weights_microexpstcnn/SMIC_RandomSuiji_best.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_acc', verbose=2, save_best_only=True, mode='max')
callbacks_list = [checkpoint]
train_images, validation_images, train_labels, validation_labels = train_test_split(training_set, traininglabels, test_size=0.2, random_state=4)
numpy.save('numpy_validation_dataset/SMIC_RandomSuiji_val_images.npy', validation_images)
numpy.save('numpy_validation_dataset/SMIC_RandomSuiji_val_labels.npy', validation_labels)
hist = model.fit(train_images, train_labels, validation_data = (validation_images, validation_labels), callbacks=callbacks_list, batch_size = 16, nb_epoch = 100, shuffle=True)
model=load_model(filepath,compile=False)
predict=model.predict(validation_images, batch_size=16)
predict=numpy.argmax(predict,axis=1)
testing_labels=numpy.argmax(validation_labels,axis=1)
num = len(testing_labels)
ct = confusion_matrix(testing_labels,predict)
print(ct)
order = numpy.unique(numpy.concatenate((predict,testing_labels)))
mat = numpy.zeros((3,3))
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]),int(order[n])]=ct[m,n]
[f1,precision,recall] = fpr(mat,3)
war = weighted_average_recall(mat, 3, num)
uar = unweighted_average_recall(mat, 3)
print("f1:"+str(f1))
print("war: " + str(war))
print("uar: " + str(uar))
def train(batch_size, spatial_epochs, temporal_epochs, train_id, list_dB,
spatial_size, flag, objective_flag, tensorboard):
############## Path Preparation ######################
root_db_path = "/home/zhangke/database/"
tensorboard_path = root_db_path + "tensorboard/"
if os.path.isdir(root_db_path + 'Weights/' + str(train_id)) == False:
os.mkdir(root_db_path + 'Weights/' + str(train_id))
######################################################
############## Variables ###################
dB = list_dB[0]
r, w, subjects, samples, n_exp, VidPerSubject, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples, db_home, db_images, cross_db_flag = load_db(
root_db_path, list_dB, spatial_size, objective_flag)
# avoid confusion
if cross_db_flag == 1:
list_samples = listOfIgnoredSamples
# total confusion matrix to be used in the computation of f1 score
tot_mat = np.zeros((n_exp, n_exp))
history = LossHistory()
stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min')
############################################
############## Flags ####################
tensorboard_flag = tensorboard
resizedFlag = 1
train_spatial_flag = 0
train_temporal_flag = 0
svm_flag = 0
finetuning_flag = 0
cam_visualizer_flag = 0
channel_flag = 0
if flag == 'st':
train_spatial_flag = 1
train_temporal_flag = 1
finetuning_flag = 1
#########################################
############ Reading Images and Labels ################
if cross_db_flag == 1:
SubperdB = Read_Input_Images_SAMM_CASME(db_images, list_samples,
listOfIgnoredSamples, dB,
resizedFlag, table, db_home,
spatial_size, channel)
else:
SubperdB = Read_Input_Images(db_images, listOfIgnoredSamples, dB,
resizedFlag, table, db_home, spatial_size,
channel, objective_flag)
labelperSub = label_matching(db_home, dB, subjects, VidPerSubject)
print("Loaded Images into the tray.subjects", subjects)
print("Loaded Labels into the tray.")
########### Model Configurations #######################
K.set_image_dim_ordering('th')
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001, decay=0.000001)
data_dim = 4096
########################################################
print("Beginning training process.")
########### Training Process ############
aaa = 1
for sub in range(subjects):
print(".starting subject" + str(sub))
gpu_observer()
spatial_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_' + str(train_id) + '_' + str(dB) + '_'
spatial_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_strain_' + str(train_id) + '_' + str(
dB) + '_'
spatial_weights_name_gray = root_db_path + 'Weights/' + str(
train_id) + '/vgg_spatial_gray_' + str(train_id) + '_' + str(
dB) + '_'
temporal_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/temporal_ID_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_' + str(train_id) + '_' + str(dB) + '_'
ae_weights_name_strain = root_db_path + 'Weights/' + str(
train_id) + '/autoencoder_strain_' + str(train_id) + '_' + str(
dB) + '_'
############### Reinitialization & weights reset of models ########################
temporal_model = temporal_module(data_dim=data_dim,
timesteps_TIM=timesteps_TIM,
classes=n_exp)
temporal_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
if channel_flag == 0:
vgg_model = VGG_16(spatial_size=spatial_size,
classes=n_exp,
channels=3,
weights_path='VGG_Face_Deep_16.h5')
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.categorical_accuracy])
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path = tensorboard_path + str(sub) + "/"
os.mkdir(cat_path)
tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path,
write_graph=True)
cat_path2 = tensorboard_path + str(sub) + "spatial/"
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2,
write_graph=True)
#############################################
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt, X, y, test_X, test_y = restructure_data(
sub, SubperdB, labelperSub, subjects, n_exp, r, w, timesteps_TIM,
channel)
############### check gpu resources ####################
gpu_observer()
########################################################
print("Beginning training & testing.")
##################### Training & Testing #########################
if train_spatial_flag == 1 and train_temporal_flag == 1:
print("Beginning spatial training.")
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping, tbCallBack2])
else:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[history, stopping])
print(".record f1 and loss")
# record f1 and loss
record_loss_accuracy(db_home, train_id, dB, history)
print(".save vgg weights")
# save vgg weights
model = record_weights(vgg_model, spatial_weights_name, sub, flag)
print(".spatial encoding")
# Spatial Encoding
output = model.predict(X, batch_size=batch_size)
features = output.reshape(int(Train_X.shape[0]), timesteps_TIM,
output.shape[1])
print("Beginning temporal training.")
# Temporal Training
if tensorboard_flag == 1:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs,
callbacks=[tbCallBack])
else:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs)
print(".save temportal weights")
# save temporal weights
temporal_model = record_weights(temporal_model,
temporal_weights_name, sub,
't') # let the flag be t
print("Beginning testing.")
print(".predicting with spatial model")
# Testing
output = model.predict(test_X, batch_size=batch_size)
print(".outputing features")
features = output.reshape(Test_X.shape[0], timesteps_TIM,
output.shape[1])
print(".predicting with temporal model")
predict = temporal_model.predict(features)
predict = np.argmax(predict, axis=1)
#predict = temporal_model.predict_classes(features, batch_size=batch_size)
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt.astype(int))
print(".writing predicts to file")
file = open(
db_home + 'Classification/' + 'Result/' + dB + '/predicts_' +
str(train_id) + '.txt', 'a')
file.write("predicts_sub_" + str(sub) + "," +
(",".join(repr(e) for e in predict.astype(list))) + "\n")
file.write("actuals_sub_" + str(sub) + "," + (",".join(
repr(e) for e in Test_Y_gt.astype(int).astype(list))) + "\n")
file.close()
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(
db_home + 'Classification/' + 'Result/' + dB + '/f1_' +
str(train_id) + '.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(db_home, dB, ct, sub, order, tot_mat, n_exp, subjects)
war = weighted_average_recall(tot_mat, n_exp, samples)
uar = unweighted_average_recall(tot_mat, n_exp)
print("war: " + str(war))
print("uar: " + str(uar))
###############################################################################
################## free memory ####################
del vgg_model
del temporal_model
del model
del Train_X, Test_X, X, y
gc.collect()
def train_smic(batch_size, spatial_epochs, temporal_epochs, train_id):
############################## Loading Labels & Images ##############################
# /media/ice/OS/Datasets/SMIC_TIM10/SMIC_TIM10
root_db_path = "/media/ice/OS/Datasets/"
dB = "SMIC_TIM10"
inputDir = root_db_path + dB + "/" + dB + "/"
workplace = root_db_path + dB + "/"
subject, filename, label, num_frames = loading_smic_labels(
root_db_path, dB)
filename = filename.as_matrix()
label = label.as_matrix()
table = np.transpose(np.array([filename, label]))
# os.remove(workplace + "Classification/SMIC_label.txt")
################# Variables #############################
spatial_size = 224
r = w = spatial_size
subjects = 16
samples = 164
n_exp = 3
IgnoredSamples_index = np.empty([0])
VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index)
listOfIgnoredSamples = []
timesteps_TIM = 10
data_dim = r * w
pad_sequence = 10
#########################################################
############## Flags ####################
resizedFlag = 1
train_spatial_flag = 1
train_temporal_flag = 1
svm_flag = 0
finetuning_flag = 1
tensorboard_flag = 0
cam_visualizer_flag = 1
#########################################
############## Reading Images and Labels ################
# SubperdB = Read_SMIC_Images(inputDir, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size)
SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB,
resizedFlag, table, workplace, spatial_size)
labelperSub = label_matching(workplace, dB, subjects, VidPerSubject)
######################################################################################
########### Model #######################
sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True)
adam = optimizers.Adam(lr=0.00001)
if train_spatial_flag == 0 and train_temporal_flag == 1:
data_dim = spatial_size * spatial_size
else:
data_dim = 4096
temporal_model = temporal_module
#########################################
################# Pretrained Model ###################
vgg_model = VGG_16('VGG_Face_Deep_16.h5')
# keras_vgg = keras_vgg16(weights='imagenet')
# vgg_model = VGG_16('imagenet')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.sparse_categorical_accuracy])
plot_model(vgg_model, to_file='model.png', show_shapes=True)
svm_classifier = SVC(kernel='linear', C=1)
######################################################
# model checkpoint
spatial_weights_name = 'vgg_spatial_17a_smic_'
temporal_weights_name = 'temporal_ID_16_smic_'
# model checkpoint
root = "/home/viprlab/Documents/Micro-Expression/" + spatial_weights_name + "weights.{epoch:02d}-{val_loss:.2f}.hdf5"
root_temporal = "/home/viprlab/Documents/Micro-Expression/" + temporal_weights_name + "weights.{epoch:02d}-{val_loss:.2f}.hdf5"
model_checkpoint = keras.callbacks.ModelCheckpoint(root,
monitor='loss',
save_best_only=True,
save_weights_only=True)
model_checkpoint_temporal = keras.callbacks.ModelCheckpoint(
root_temporal,
monitor='loss',
save_best_only=True,
save_weights_only=True)
########### Training Process ############
# Todo:
# 1) LOSO (done)
# 2) call model (done)
# 3) saving model architecture
# 4) Saving Checkpoint
# 5) make prediction (done)
if tensorboard_flag == 1:
tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/"
tot_mat = np.zeros((n_exp, n_exp))
spatial_weights_name = 'vgg_spatial_ID_under_dev_smic.h5'
temporal_weights_name = 'temporal_ID_under_dev_smic.h5'
for sub in range(subjects):
vgg_model = VGG_16('VGG_Face_Deep_16.h5')
vgg_model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=[metrics.sparse_categorical_accuracy])
############ for tensorboard ###############
if tensorboard_flag == 1:
cat_path = tensorboard_path + str(sub) + "/"
os.mkdir(cat_path)
tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path,
write_graph=True)
cat_path2 = tensorboard_path + str(sub) + "spat/"
os.mkdir(cat_path2)
tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2,
write_graph=True)
#############################################
image_label_mapping = np.empty([0])
Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO(
sub, SubperdB, labelperSub, subjects)
# Rearrange Training labels into a vector of images, breaking sequence
Train_X_spatial = Train_X.reshape(Train_X.shape[0] * 10, r, w, 1)
Test_X_spatial = Test_X.reshape(Test_X.shape[0] * 10, r, w, 1)
# Extend Y labels 10 fold, so that all images have labels
Train_Y_spatial = np.repeat(Train_Y, 10, axis=0)
Test_Y_spatial = np.repeat(Test_Y, 10, axis=0)
# Duplicate channel of input image
Train_X_spatial = duplicate_channel(Train_X_spatial)
Test_X_spatial = duplicate_channel(Test_X_spatial)
# print ("Train_X_shape: " + str(np.shape(Train_X_spatial)))
# print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial)))
# print ("Test_X_shape: " + str(np.shape(Test_X_spatial)))
# print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial)))
# print(Train_X_spatial)
##################### Training & Testing #########################
X = Train_X_spatial.reshape(Train_X_spatial.shape[0], 3, r, w)
y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], 5)
test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], 3, r, w)
test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], 5)
###### conv weights must be freezed for transfer learning ######
if finetuning_flag == 1:
for layer in vgg_model.layers[:33]:
layer.trainable = False
if train_spatial_flag == 1 and train_temporal_flag == 1:
# trains encoder until fc, train temporal
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[tbCallBack2])
else:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[model_checkpoint])
# vgg_model.save_weights(spatial_weights_name)
model = Model(inputs=vgg_model.input,
outputs=vgg_model.layers[35].output)
plot_model(model,
to_file="spatial_module_FULL_TRAINING.png",
show_shapes=True)
# Spatial Encoding
output = model.predict(X, batch_size=batch_size)
features = output.reshape(int(output.shape[0] / 10), 10,
output.shape[1])
# Temporal Training
if tensorboard_flag == 1:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs,
callbacks=[tbCallBack])
else:
temporal_model.fit(features,
Train_Y,
batch_size=batch_size,
epochs=temporal_epochs,
callbacks=[model_checkpoint_temporal])
# temporal_model.save_weights(temporal_weights_name)
# Testing
output = model.predict(test_X, batch_size=batch_size)
features = output.reshape(int(output.shape[0] / 10), 10,
output.shape[1])
predict = temporal_model.predict_classes(features,
batch_size=batch_size)
elif train_spatial_flag == 1 and train_temporal_flag == 0 and cam_visualizer_flag == 0:
# trains spatial module ONLY, no escape
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[tbCallBack2])
else:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[model_checkpoint])
# vgg_model.save_weights(spatial_weights_name)
plot_model(vgg_model,
to_file="spatial_module_ONLY.png",
show_shapes=True)
# Testing
predict = vgg_model.predict(test_X, batch_size=batch_size)
Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0)
elif train_spatial_flag == 0 and train_temporal_flag == 1:
# trains temporal module ONLY.
# Temporal Training
if tensorboard_flag == 1:
temporal_model.fit(Train_X,
Train_Y,
batch_size=batch_size,
epochs=spatial_epochs,
callbacks=[tbCallBack])
else:
temporal_model.fit(Train_X,
Train_Y,
batch_size=batch_size,
epochs=spatial_epochs,
callbacks=[model_checkpoint_temporal])
# temporal_model.save_weights(temporal_weights_name)
# Testing
predict = temporal_model.predict_classes(Test_X,
batch_size=batch_size)
elif svm_flag == 1 and finetuning_flag == 0:
# no finetuning
X = vgg_model.predict(X, batch_size=batch_size)
y_for_svm = np.argmax(y, axis=1)
svm_classifier.fit(X, y_for_svm)
test_X = vgg_model.predict(test_X, batch_size=batch_size)
predict = svm_classifier.predict(test_X)
Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0)
elif train_spatial_flag == 1 and train_temporal_flag == 0 and cam_visualizer_flag == 1:
# trains spatial module & CAM ONLY
# Spatial Training
if tensorboard_flag == 1:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[tbCallBack2])
else:
vgg_model.fit(X,
y,
batch_size=batch_size,
epochs=spatial_epochs,
shuffle=True,
callbacks=[model_checkpoint])
# vgg_model.save_weights(spatial_weights_name)
plot_model(vgg_model,
to_file="spatial_module_CAM_ONLY.png",
show_shapes=True)
# Testing
predict = vgg_model.predict(test_X, batch_size=batch_size)
Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0)
##############################################################
#################### Confusion Matrix Construction #############
print(predict)
print(Test_Y_gt)
ct = confusion_matrix(Test_Y_gt, predict)
# check the order of the CT
order = np.unique(np.concatenate((predict, Test_Y_gt)))
# create an array to hold the CT for each CV
mat = np.zeros((n_exp, n_exp))
# put the order accordingly, in order to form the overall ConfusionMat
for m in range(len(order)):
for n in range(len(order)):
mat[int(order[m]), int(order[n])] = ct[m, n]
tot_mat = mat + tot_mat
################################################################
#################### cumulative f1 plotting ######################
microAcc = np.trace(tot_mat) / np.sum(tot_mat)
[f1, precision, recall] = fpr(tot_mat, n_exp)
file = open(workplace + 'Classification/' + 'Result/' + '/f1.txt', 'a')
file.write(str(f1) + "\n")
file.close()
##################################################################
################# write each CT of each CV into .txt file #####################
record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects)
###############################################################################
