def test_samm(batch_size, spatial_epochs, temporal_epochs, train_id, dB, spatial_size, flag, tensorboard): ############## Path Preparation ###################### root_db_path = "/media/viprlab/01D31FFEF66D5170/" workplace = root_db_path + dB + "/" inputDir = root_db_path + dB + "/" + dB + "/" ###################################################### classes = 5 if dB == 'CASME2_TIM': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples( inputDir) ############## Variables ################### r = w = spatial_size subjects = 2 n_exp = 5 # VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) listOfIgnoredSamples = [] VidPerSubject = [2, 1] timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 ############################################ os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'CASME2_Optical': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples( inputDir) ############## Variables ################### r = w = spatial_size subjects = 26 n_exp = 5 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 9 data_dim = r * w pad_sequence = 9 channel = 3 ############################################ # os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'SAMM_TIM10': table, table_objective = loading_samm_table(root_db_path, dB) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) ################# Variables ############################# r = w = spatial_size subjects = 29 n_exp = 8 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 classes = 8 ######################################################### elif dB == 'SAMM_CASME_Optical': # total amount of videos 253 table, table_objective = loading_samm_table(root_db_path, dB) table = table_objective table2 = loading_casme_objective_table(root_db_path, dB) # merge samm and casme tables table = np.concatenate((table, table2), axis=1) # print(table.shape) # listOfIgnoredSamples, IgnoredSamples_index, sub_items = ignore_casme_samples(inputDir) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) sub_items = np.empty([0]) list_samples = filter_objective_samples(table) r = w = spatial_size subjects = 47 # some subjects were removed because of objective classes and ignore samples: 47 n_exp = 5 # TODO: # 1) Further decrease the video amount, the one with objective classes >= 6 # list samples: samples with wanted objective class VidPerSubject, list_samples = get_subfolders_num_crossdb( inputDir, IgnoredSamples_index, sub_items, table, list_samples) # print(VidPerSubject) # print(len(VidPerSubject)) # print(sum(VidPerSubject)) timesteps_TIM = 9 data_dim = r * w channel = 3 classes = 5 if os.path.isfile(workplace + "Classification/SAMM_CASME_Optical_label.txt"): os.remove(workplace + "Classification/SAMM_CASME_Optical_label.txt") ##################### Variables ###################### ###################################################### ############## Flags #################### tensorboard_flag = tensorboard resizedFlag = 1 train_spatial_flag = 0 train_temporal_flag = 0 svm_flag = 0 finetuning_flag = 0 cam_visualizer_flag = 0 channel_flag = 0 if flag == 'st': train_spatial_flag = 1 train_temporal_flag = 1 finetuning_flag = 1 elif flag == 's': train_spatial_flag = 1 finetuning_flag = 1 elif flag == 't': train_temporal_flag = 1 elif flag == 'nofine': svm_flag = 1 elif flag == 'scratch': train_spatial_flag = 1 train_temporal_flag = 1 elif flag == 'st4': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 1 elif flag == 'st7': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 2 ######################################### ############ Reading Images and Labels ################ SubperdB = Read_Input_Images_SAMM_CASME(inputDir, list_samples, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size, channel) print("Loaded Images into the tray...") labelperSub = label_matching(workplace, dB, subjects, VidPerSubject) print("Loaded Labels into the tray...") if channel_flag == 1: SubperdB_strain = Read_Input_Images_SAMM_CASME( inputDir, list_samples, listOfIgnoredSamples, 'SAMM_CASME_Strain', resizedFlag, table, workplace, spatial_size, 1) elif channel_flag == 2: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace, spatial_size, 3) ####################################################### ########### Model Configurations ####################### sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True) adam = optimizers.Adam(lr=0.00001, decay=0.000001) adam2 = optimizers.Adam(lr=0.00075, decay=0.0001) # Different Conditions for Temporal Learning ONLY if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical': data_dim = spatial_size * spatial_size elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical': data_dim = spatial_size * spatial_size * 3 else: data_dim = 4096 ######################################################## ########### Training Process ############ # total confusion matrix to be used in the computation of f1 score tot_mat = np.zeros((n_exp, n_exp)) # model checkpoint spatial_weights_name = 'vgg_spatial_' + str(train_id) + '_casme2_' temporal_weights_name = 'temporal_ID_' + str(train_id) + '_casme2_' history = LossHistory() stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min', patience=5) # model checkpoint if os.path.isdir('/media/viprlab/01D31FFEF66D5170/Weights/' + str(train_id)) == False: os.mkdir('/media/viprlab/01D31FFEF66D5170/Weights/' + str(train_id)) for sub in range(subjects): # sub = sub + 25 # if sub > subjects: # break ############### Reinitialization & weights reset of models ######################## spatial_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str( train_id) + '/vgg_spatial_' + str(train_id) + '_' + str( dB) + '_' + str(sub) + '.h5' spatial_weights_name_strain = '/media/viprlab/01D31FFEF66D5170/Weights/' + str( train_id) + '/vgg_spatial_strain_' + str(train_id) + '_' + str( dB) + '_' + str(sub) + '.h5' temporal_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str( train_id) + '/temporal_ID_' + str(train_id) + '_' + str( dB) + '_' + str(sub) + '.h5' ae_weights_name = '/media/viprlab/01D31FFEF66D5170/Weights/' + str( train_id) + '/autoencoder_' + str(train_id) + '_' + str( dB) + '_' + str(sub) + '.h5' ae_weights_name_strain = '/media/viprlab/01D31FFEF66D5170/Weights/' + str( train_id) + '/autoencoder_strain_' + str(train_id) + '_' + str( dB) + '_' + str(sub) + '.h5' temporal_model = temporal_module(data_dim=data_dim, timesteps_TIM=timesteps_TIM, weights_path=temporal_weights_name) temporal_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) if channel_flag == 1 or channel_flag == 2: vgg_model = VGG_16_4_channels(spatial_size=spatial_size, weights_path=spatial_weights_name) vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) else: vgg_model = VGG_16(spatial_size=spatial_size, weights_path='VGG_Face_Deep_16.h5') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) svm_classifier = SVC(kernel='linear', C=1) #################################################################################### Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO( sub, SubperdB, labelperSub, subjects) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r, w, channel) Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w, channel) # Special Loading for 4-Channel if channel_flag == 1: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X.shape[0] * timesteps_TIM, r, w, 1) # Concatenate Train X & Train_X_Strain Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3) Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3) total_channel = 4 elif channel_flag == 2: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO( sub, SubperdB_gray, labelperSub, subjects) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Train_X_gray = Train_X_gray.reshape( Train_X_gray.shape[0] * timesteps_TIM, r, w, 3) Test_X_gray = Test_X_gray.reshape( Test_X_gray.shape[0] * timesteps_TIM, r, w, 3) # Concatenate Train_X_Strain & Train_X & Train_X_gray Train_X_spatial = np.concatenate( (Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3) Test_X_spatial = np.concatenate( (Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3) total_channel = 7 if channel == 1: # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) Test_X_spatial = duplicate_channel(Test_X_spatial) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0) Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0) print("Train_X_shape: " + str(np.shape(Train_X_spatial))) print("Train_Y_shape: " + str(np.shape(Train_Y_spatial))) print("Test_X_shape: " + str(np.shape(Test_X_spatial))) print("Test_Y_shape: " + str(np.shape(Test_Y_spatial))) # print(Train_X_spatial) ##################### Training & Testing ######################### X = Train_X_spatial.reshape(Train_X_spatial.shape[0], total_channel, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes) normalized_X = X.astype('float32') / 255. test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], total_channel, r, w) test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes) normalized_test_X = test_X.astype('float32') / 255. print(X.shape) ###### conv weights must be freezed for transfer learning ###### if finetuning_flag == 1: for layer in vgg_model.layers[:33]: layer.trainable = False for layer in vgg_model_cam.layers[:31]: layer.trainable = False if train_spatial_flag == 1 and train_temporal_flag == 1: # record f1 and loss file_loss = open( workplace + 'Classification/' + 'Result/' + dB + '/loss_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.losses) + "\n") file_loss.close() file_loss = open( workplace + 'Classification/' + 'Result/' + dB + '/accuracy_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.accuracy) + "\n") file_loss.close() model = Model(inputs=vgg_model.input, outputs=vgg_model.layers[35].output) plot_model(model, to_file="spatial_module_FULL_TRAINING.png", show_shapes=True) # Testing output = model.predict(test_X, batch_size=batch_size) features = output.reshape(Test_X.shape[0], timesteps_TIM, output.shape[1]) predict = temporal_model.predict_classes(features, batch_size=batch_size) ############################################################## #################### Confusion Matrix Construction ############# print(predict) print(Test_Y_gt) ct = confusion_matrix(Test_Y_gt, predict) # check the order of the CT order = np.unique(np.concatenate((predict, Test_Y_gt))) # create an array to hold the CT for each CV mat = np.zeros((n_exp, n_exp)) # put the order accordingly, in order to form the overall ConfusionMat for m in range(len(order)): for n in range(len(order)): mat[int(order[m]), int(order[n])] = ct[m, n] tot_mat = mat + tot_mat ################################################################ #################### cumulative f1 plotting ###################### microAcc = np.trace(tot_mat) / np.sum(tot_mat) [f1, precision, recall] = fpr(tot_mat, n_exp) file = open( workplace + 'Classification/' + 'Result/' + dB + '/f1_' + str(train_id) + '.txt', 'a') file.write(str(f1) + "\n") file.close() ################################################################## ################# write each CT of each CV into .txt file ##################### record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects) ###############################################################################
image_label_mapping = np.empty([0]) Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO( sub, SubperdB, labelperSub, subjects) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0] * 10, r, w, 1) Test_X_spatial = Test_X.reshape(Test_X.shape[0] * 10, r, w, 1) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, 10, axis=0) Test_Y_spatial = np.repeat(Test_Y, 10, axis=0) # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) Test_X_spatial = duplicate_channel(Test_X_spatial) # temp = Test_X_spatial[0] # temp = cv2.ims(temp) # cv2.imwrite("concat_image.png", temp) # print(temp.shape) ##################### VGG FACE 16 ######################### X = Train_X_spatial.reshape(Train_X_spatial.shape[0], 3, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], 5) test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], 3, r, w) # test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], 5) test_y = np.repeat(Test_Y_gt, 10, axis=0)
def train_samm_cross(batch_size, spatial_epochs, temporal_epochs, train_id, dB, spatial_size, flag, tensorboard): ############## Path Preparation ###################### root_db_path = "/media/ice/OS/Datasets/" workplace = root_db_path + dB + "/" inputDir = root_db_path + dB + "/" + dB + "/" ###################################################### classes = 5 if dB == 'CASME2_TIM': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(inputDir) ############## Variables ################### r = w = spatial_size subjects=2 n_exp = 5 # VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) listOfIgnoredSamples = [] VidPerSubject = [2,1] timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 ############################################ os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'CASME2_Optical': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples(inputDir) ############## Variables ################### r = w = spatial_size subjects=26 n_exp = 5 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 9 data_dim = r * w pad_sequence = 9 channel = 3 ############################################ # os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'SAMM_TIM10': table, table_objective = loading_samm_table(root_db_path, dB) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) ################# Variables ############################# r = w = spatial_size subjects = 29 n_exp = 8 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 classes = 8 ######################################################### elif dB == 'SAMM_CASME_Optical': # total amount of videos 253 # table, table_objective = loading_samm_table(root_db_path, dB) # table = table_objective table = loading_casme_objective_table(root_db_path, dB) # merge samm and casme tables # table = np.concatenate((table, table2), axis=1) # print(table.shape) # listOfIgnoredSamples, IgnoredSamples_index, sub_items = ignore_casme_samples(inputDir) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) sub_items = np.empty([0]) list_samples = filter_objective_samples(table) r = w = spatial_size subjects = 26 # some subjects were removed because of objective classes and ignore samples: 47 n_exp = 5 # TODO: # 1) Further decrease the video amount, the one with objective classes >= 6 # list samples: samples with wanted objective class VidPerSubject, list_samples = get_subfolders_num_crossdb(inputDir, IgnoredSamples_index, sub_items, table, list_samples) # print(VidPerSubject) # print(len(VidPerSubject)) # print(sum(VidPerSubject)) timesteps_TIM = 9 data_dim = r * w channel = 3 classes = 5 if os.path.isfile(workplace + "Classification/SAMM_CASME_Optical_label.txt"): os.remove(workplace + "Classification/SAMM_CASME_Optical_label.txt") ##################### Variables ###################### ###################################################### ############## Flags #################### tensorboard_flag = tensorboard resizedFlag = 1 train_spatial_flag = 0 train_temporal_flag = 0 svm_flag = 0 finetuning_flag = 0 cam_visualizer_flag = 0 channel_flag = 0 if flag == 'st': train_spatial_flag = 1 train_temporal_flag = 1 finetuning_flag = 1 elif flag == 's': train_spatial_flag = 1 finetuning_flag = 1 elif flag == 't': train_temporal_flag = 1 elif flag == 'nofine': svm_flag = 1 elif flag == 'scratch': train_spatial_flag = 1 train_temporal_flag = 1 elif flag == 'st4': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 1 elif flag == 'st7': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 2 ######################################### ############ Reading Images and Labels ################ SubperdB = Read_Input_Images_SAMM_CASME(inputDir, list_samples, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size, channel) print("Loaded Images into the tray...") labelperSub = label_matching(workplace, dB, subjects, VidPerSubject) print("Loaded Labels into the tray...") if channel_flag == 1: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) elif channel_flag == 2: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace, spatial_size, 3) ####################################################### ########### Model Configurations ####################### sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True) adam = optimizers.Adam(lr=0.00001, decay=0.000001) adam2 = optimizers.Adam(lr= 0.00075, decay= 0.0001) # Different Conditions for Temporal Learning ONLY if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical': data_dim = spatial_size * spatial_size elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical': data_dim = spatial_size * spatial_size * 3 else: data_dim = 4096 ######################################################## ########### Training Process ############ # total confusion matrix to be used in the computation of f1 score tot_mat = np.zeros((n_exp,n_exp)) # model checkpoint spatial_weights_name = 'vgg_spatial_'+ str(train_id) + '_casme2_' temporal_weights_name = 'temporal_ID_' + str(train_id) + '_casme2_' history = LossHistory() stopping = EarlyStopping(monitor='loss', min_delta = 0, mode = 'min') ############### Reinitialization & weights reset of models ######################## vgg_model_cam = VGG_16(spatial_size=spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') temporal_model = temporal_module(data_dim=data_dim, classes=classes, timesteps_TIM=timesteps_TIM) temporal_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) conv_ae = convolutional_autoencoder(spatial_size = spatial_size, classes = classes) conv_ae.compile(loss='binary_crossentropy', optimizer=adam) if channel_flag == 1 or channel_flag == 2: vgg_model = VGG_16_4_channels(classes=classes, spatial_size = spatial_size) vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) else: vgg_model = VGG_16(spatial_size = spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) svm_classifier = SVC(kernel='linear', C=1) #################################################################################### ############ for tensorboard ############### if tensorboard_flag == 1: cat_path = tensorboard_path + str(sub) + "/" os.mkdir(cat_path) tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path, write_graph=True) cat_path2 = tensorboard_path + str(sub) + "spat/" os.mkdir(cat_path2) tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2, write_graph=True) ############################################# image_label_mapping = np.empty([0]) Train_X, Train_Y= standard_data_loader(SubperdB, labelperSub, subjects, classes) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0]*timesteps_TIM, r, w, channel) # Test_X_spatial = Test_X.reshape(Test_X.shape[0]* timesteps_TIM, r, w, channel) # Special Loading for 4-Channel if channel_flag == 1: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_Strain = Train_X_Strain.reshape(Train_X_Strain.shape[0]*timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape(Test_X.shape[0]*timesteps_TIM, r, w, 1) # Concatenate Train X & Train_X_Strain Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3) Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3) channel = 4 elif channel_flag == 2: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO(sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO(sub, SubperdB_gray, labelperSub, subjects) Train_X_Strain = Train_X_Strain.reshape(Train_X_Strain.shape[0]*timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape(Test_X_Strain.shape[0]*timesteps_TIM, r, w, 1) Train_X_gray = Train_X_gray.reshape(Train_X_gray.shape[0]*timesteps_TIM, r, w, 3) Test_X_gray = Test_X_gray.reshape(Test_X_gray.shape[0]*timesteps_TIM, r, w, 3) # Concatenate Train_X_Strain & Train_X & Train_X_gray Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3) Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3) channel = 7 if channel == 1: # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) # Test_X_spatial = duplicate_channel(Test_X_spatial) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0) # Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0) # print ("Train_X_shape: " + str(np.shape(Train_X_spatial))) # print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial))) # print ("Test_X_shape: " + str(np.shape(Test_X_spatial))) # print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial))) # print(Train_X_spatial) ##################### Training & Testing ######################### X = Train_X_spatial.reshape(Train_X_spatial.shape[0], channel, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes) normalized_X = X.astype('float32') / 255. # test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w) # test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes) # normalized_test_X = test_X.astype('float32') / 255. print(X.shape) ###### conv weights must be freezed for transfer learning ###### if finetuning_flag == 1: for layer in vgg_model.layers[:33]: layer.trainable = False for layer in vgg_model_cam.layers[:31]: layer.trainable = False if train_spatial_flag == 1 and train_temporal_flag == 1: # Autoencoder features # conv_ae.fit(normalized_X, normalized_X, batch_size=batch_size, epochs=spatial_epochs, shuffle=True) # Spatial Training if tensorboard_flag == 1: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[tbCallBack2]) else: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[history, stopping]) # record f1 and loss file_loss = open(workplace+'Classification/'+ 'Result/'+dB+'/loss_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.losses) + "\n") file_loss.close() file_loss = open(workplace+'Classification/'+ 'Result/'+dB+'/accuracy_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.accuracy) + "\n") file_loss.close() vgg_model.save_weights(spatial_weights_name + 'HDE'+ ".h5") model = Model(inputs=vgg_model.input, outputs=vgg_model.layers[35].output) plot_model(model, to_file="spatial_module_FULL_TRAINING.png", show_shapes=True) model_ae = Model(inputs=conv_ae.input, outputs=conv_ae.output) plot_model(model_ae, to_file='autoencoders.png', show_shapes=True) # Autoencoding output_ae = model_ae.predict(normalized_X, batch_size = batch_size) for i in range(batch_size): visual_ae = output_ae[i].reshape(224,224,channel) # de-normalize visual_ae = ( ( visual_ae - min(visual_ae) ) / ( max(visual_ae) - min(visual_ae) ) ) * 255 fname = '{prefix}_{index}_{hash}.{format}'.format(prefix='AE_output', index=str(sub), hash=np.random.randint(1e7), format='png') cv2.imwrite(workplace+'Classification/Result/ae_train/'+fname, visual_ae) output_ae = model.predict(output_ae, batch_size = batch_size) # Spatial Encoding output = model.predict(X, batch_size = batch_size) # features = output.reshape(int(Train_X.shape[0]), timesteps_TIM, output.shape[1]) # merging autoencoded features and spatial features output = np.concatenate((output, output_ae), axis=1) # print(output.shape) features = output.reshape(int(Train_X.shape[0]), timesteps_TIM, output.shape[1]) # Temporal Training if tensorboard_flag == 1: temporal_model.fit(features, Train_Y, batch_size=batch_size, epochs=temporal_epochs, callbacks=[tbCallBack]) else: temporal_model.fit(features, Train_Y, batch_size=batch_size, epochs=temporal_epochs) temporal_model.save_weights(temporal_weights_name + 'HDE' + ".h5") # # Testing # output = model.predict(test_X, batch_size = batch_size) # output_ae = model_ae.predict(normalized_test_X, batch_size = batch_size) # for i in range(batch_size): # visual_ae = output_ae[i].reshape(224,224,channel) # # de-normalize # visual_ae = ( ( visual_ae - min(visual_ae) ) / ( max(visual_ae) - min(visual_ae) ) ) * 255 # fname = '{prefix}_{index}_{hash}.{format}'.format(prefix='AE_output', index=str(sub), # hash=np.random.randint(1e7), format='png') # cv2.imwrite(workplace+'Classification/Result/ae_train/'+fname, visual_ae) # output_ae = model.predict(output_ae, batch_size = batch_size) # output = np.concatenate((output, output_ae), axis=1) # features = output.reshape(Test_X.shape[0], timesteps_TIM, output.shape[1]) # predict = temporal_model.predict_classes(features, batch_size=batch_size) # ############################################################## # #################### Confusion Matrix Construction ############# # print (predict) # print (Test_Y_gt) # ct = confusion_matrix(Test_Y_gt,predict) # # check the order of the CT # order = np.unique(np.concatenate((predict,Test_Y_gt))) # # create an array to hold the CT for each CV # mat = np.zeros((n_exp,n_exp)) # # put the order accordingly, in order to form the overall ConfusionMat # for m in range(len(order)): # for n in range(len(order)): # mat[int(order[m]),int(order[n])]=ct[m,n] # tot_mat = mat + tot_mat # ################################################################ # #################### cumulative f1 plotting ###################### # microAcc = np.trace(tot_mat) / np.sum(tot_mat) # [f1,precision,recall] = fpr(tot_mat,n_exp) # file = open(workplace+'Classification/'+ 'Result/'+dB+'/f1_' + str(train_id) + '.txt', 'a') # file.write(str(f1) + "\n") # file.close() ################################################################## ################# write each CT of each CV into .txt file ##################### # record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects) ###############################################################################
def train_cae_lstm(batch_size, spatial_epochs, temporal_epochs, train_id, dB, spatial_size, flag, tensorboard): ############## Path Preparation ###################### root_db_path = "/media/ice/OS/Datasets/" workplace = root_db_path + dB + "/" inputDir = root_db_path + dB + "/" + dB + "/" ###################################################### r, w, subjects, samples, n_exp, VidPerSubject, timesteps_TIM, timesteps_TIM, data_dim, channel, table, listOfIgnoredSamples = load_db( root_db_path, dB, spatial_size) # print(VidPerSubject) ############## Flags #################### tensorboard_flag = tensorboard resizedFlag = 1 train_spatial_flag = 0 train_temporal_flag = 0 svm_flag = 0 finetuning_flag = 0 cam_visualizer_flag = 0 channel_flag = 0 if flag == 'st': train_spatial_flag = 1 train_temporal_flag = 1 finetuning_flag = 1 elif flag == 's': train_spatial_flag = 1 finetuning_flag = 1 elif flag == 't': train_temporal_flag = 1 elif flag == 'nofine': svm_flag = 1 elif flag == 'scratch': train_spatial_flag = 1 train_temporal_flag = 1 elif flag == 'st4': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 1 elif flag == 'st7': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 2 ######################################### ############ Reading Images and Labels ################ SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size, channel) print("Loaded Images into the tray...") labelperSub = label_matching(workplace, dB, subjects, VidPerSubject) print("Loaded Labels into the tray...") if channel_flag == 1: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) elif channel_flag == 2: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace, spatial_size, 3) ####################################################### ########### Model Configurations ####################### sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True) adam = optimizers.Adam(lr=0.00001, decay=0.000001) # Different Conditions for Temporal Learning ONLY if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical': data_dim = spatial_size * spatial_size elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical': data_dim = spatial_size * spatial_size * 3 else: data_dim = 4096 ######################################################## ########### Training Process ############ # Todo: # 1) LOSO (done) # 2) call model (done) # 3) saving model architecture # 4) Saving Checkpoint (done) # 5) make prediction (done) if tensorboard_flag == 1: tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/" # total confusion matrix to be used in the computation of f1 score tot_mat = np.zeros((n_exp, n_exp)) # model checkpoint spatial_weights_name = 'vgg_spatial_' + str(train_id) + '_' + str(dB) + '_' temporal_weights_name = 'temporal_ID_' + str(train_id) + '_' + str( dB) + '_' ae_weights_name = 'autoencoder_' + str(train_id) + '_' + str(dB) + '_' history = LossHistory() stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min') for sub in range(subjects): ############### Reinitialization & weights reset of models ######################## vgg_model_cam = VGG_16(spatial_size=spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') temporal_model = temporal_module(data_dim=data_dim, classes=classes, timesteps_TIM=timesteps_TIM) temporal_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) conv_ae = convolutional_autoencoder(spatial_size=spatial_size, classes=classes) conv_ae.compile(loss='binary_crossentropy', optimizer=adam) if channel_flag == 1 or channel_flag == 2: vgg_model = VGG_16_4_channels(classes=classes, spatial_size=spatial_size) vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) else: vgg_model = VGG_16(spatial_size=spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) svm_classifier = SVC(kernel='linear', C=1) #################################################################################### ############ for tensorboard ############### if tensorboard_flag == 1: cat_path = tensorboard_path + str(sub) + "/" os.mkdir(cat_path) tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path, write_graph=True) cat_path2 = tensorboard_path + str(sub) + "spat/" os.mkdir(cat_path2) tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2, write_graph=True) ############################################# image_label_mapping = np.empty([0]) Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO( sub, SubperdB, labelperSub, subjects, classes) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r, w, channel) Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w, channel) # Special Loading for 4-Channel if channel_flag == 1: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X.shape[0] * timesteps_TIM, r, w, 1) # Concatenate Train X & Train_X_Strain Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3) Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3) channel = 4 elif channel_flag == 2: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO( sub, SubperdB_gray, labelperSub, subjects) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Train_X_gray = Train_X_gray.reshape( Train_X_gray.shape[0] * timesteps_TIM, r, w, 3) Test_X_gray = Test_X_gray.reshape( Test_X_gray.shape[0] * timesteps_TIM, r, w, 3) # Concatenate Train_X_Strain & Train_X & Train_X_gray Train_X_spatial = np.concatenate( (Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3) Test_X_spatial = np.concatenate( (Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3) channel = 7 if channel == 1: # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) Test_X_spatial = duplicate_channel(Test_X_spatial) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0) Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0) # print ("Train_X_shape: " + str(np.shape(Train_X_spatial))) # print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial))) # print ("Test_X_shape: " + str(np.shape(Test_X_spatial))) # print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial))) # print(Train_X_spatial) ##################### Training & Testing ######################### # print(Train_X_spatial.shape) X = Train_X_spatial.reshape(Train_X_spatial.shape[0], channel, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes) normalized_X = X.astype('float32') / 255. test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w) test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes) normalized_test_X = test_X.astype('float32') / 255. print(X.shape) ###### conv weights must be freezed for transfer learning ###### if finetuning_flag == 1: for layer in vgg_model.layers[:33]: layer.trainable = False for layer in vgg_model_cam.layers[:31]: layer.trainable = False if train_spatial_flag == 1 and train_temporal_flag == 1: # Autoencoder first training conv_ae.fit(normalized_X, normalized_X, batch_size=batch_size, epochs=spatial_epochs, shuffle=True) # remove decoder conv_ae.pop() conv_ae.pop() conv_ae.pop() conv_ae.pop() conv_ae.pop() conv_ae.pop() conv_ae.pop() # append dense layers conv_ae.add(Flatten()) conv_ae.add(Dense(4096, activation='relu')) conv_ae.add(Dense(4096, activation='relu')) conv_ae.add(Dense(n_exp, activation='sigmoid')) model_ae = Model(inputs=conv_ae.input, outputs=conv_ae.layers[9].output) plot_model(model_ae, to_file='autoencoders.png', show_shapes=True) # freeze encoder for layer in conv_ae.layers[:6]: layer.trainable = False # finetune dense layers conv_ae.compile(loss='categorical_crossentropy', optimizer=adam) conv_ae.fit(normalized_X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True) model_ae = Model(inputs=conv_ae.input, outputs=conv_ae.layers[8].output) plot_model(model_ae, to_file='autoencoders.png', show_shapes=True) # Autoencoding output = model_ae.predict(normalized_X, batch_size=batch_size) # print(output.shape) features = output.reshape(int(Train_X.shape[0]), timesteps_TIM, output.shape[1]) temporal_model.fit(features, Train_Y, batch_size=batch_size, epochs=temporal_epochs) temporal_model.save_weights(temporal_weights_name + str(sub) + ".h5") # Testing output = model_ae.predict(test_X, batch_size=batch_size) features = output.reshape(Test_X.shape[0], timesteps_TIM, output.shape[1]) predict = temporal_model.predict_classes(features, batch_size=batch_size) #################### Confusion Matrix Construction ############# print(predict) print(Test_Y_gt) ct = confusion_matrix(Test_Y_gt, predict) # check the order of the CT order = np.unique(np.concatenate((predict, Test_Y_gt))) # create an array to hold the CT for each CV mat = np.zeros((n_exp, n_exp)) # put the order accordingly, in order to form the overall ConfusionMat for m in range(len(order)): for n in range(len(order)): mat[int(order[m]), int(order[n])] = ct[m, n] tot_mat = mat + tot_mat ################################################################ #################### cumulative f1 plotting ###################### microAcc = np.trace(tot_mat) / np.sum(tot_mat) [f1, precision, recall] = fpr(tot_mat, n_exp) file = open( workplace + 'Classification/' + 'Result/' + dB + '/f1_' + str(train_id) + '.txt', 'a') file.write(str(f1) + "\n") file.close() ################################################################## ################# write each CT of each CV into .txt file ##################### record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects) ###############################################################################
def train_samm(batch_size, spatial_epochs, temporal_epochs, train_id, dB, spatial_size, flag, tensorboard): ############## Path Preparation ###################### root_db_path = "/media/ice/OS/Datasets/" workplace = root_db_path + dB + "/" inputDir = root_db_path + dB + "/" + dB + "/" ###################################################### classes = 5 if dB == 'CASME2_TIM': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples( inputDir) ############## Variables ################### r = w = spatial_size subjects = 2 n_exp = 5 # VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) listOfIgnoredSamples = [] VidPerSubject = [2, 1] timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 ############################################ os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'CASME2_Optical': table = loading_casme_table(workplace + 'CASME2-ObjectiveClasses.xlsx') listOfIgnoredSamples, IgnoredSamples_index = ignore_casme_samples( inputDir) ############## Variables ################### r = w = spatial_size subjects = 26 n_exp = 5 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 9 data_dim = r * w pad_sequence = 9 channel = 3 ############################################ # os.remove(workplace + "Classification/CASME2_TIM_label.txt") elif dB == 'SAMM_TIM10': table, table_objective = loading_samm_table(root_db_path, dB) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) ################# Variables ############################# r = w = spatial_size subjects = 29 n_exp = 8 VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 channel = 3 classes = 8 ######################################################### elif dB == 'SAMM_CASME_Strain': # total amount of videos 253 table, table_objective = loading_samm_table(root_db_path, dB) table = table_objective table2 = loading_casme_objective_table(root_db_path, dB) # merge samm and casme tables table = np.concatenate((table, table2), axis=1) # print(table.shape) # listOfIgnoredSamples, IgnoredSamples_index, sub_items = ignore_casme_samples(inputDir) listOfIgnoredSamples = [] IgnoredSamples_index = np.empty([0]) sub_items = np.empty([0]) list_samples = filter_objective_samples(table) r = w = spatial_size subjects = 47 # some subjects were removed because of objective classes and ignore samples: 47 n_exp = 5 # TODO: # 1) Further decrease the video amount, the one with objective classes >= 6 # list samples: samples with wanted objective class VidPerSubject, list_samples = get_subfolders_num_crossdb( inputDir, IgnoredSamples_index, sub_items, table, list_samples) # print(VidPerSubject) # print(len(VidPerSubject)) # print(sum(VidPerSubject)) timesteps_TIM = 9 data_dim = r * w channel = 3 classes = 5 if os.path.isfile(workplace + "Classification/SAMM_CASME_Optical_label.txt"): os.remove(workplace + "Classification/SAMM_CASME_Optical_label.txt") ##################### Variables ###################### ###################################################### ############## Flags #################### tensorboard_flag = tensorboard resizedFlag = 1 train_spatial_flag = 0 train_temporal_flag = 0 svm_flag = 0 finetuning_flag = 0 cam_visualizer_flag = 0 channel_flag = 0 if flag == 'st': train_spatial_flag = 1 train_temporal_flag = 1 finetuning_flag = 1 elif flag == 's': train_spatial_flag = 1 finetuning_flag = 1 elif flag == 't': train_temporal_flag = 1 elif flag == 'nofine': svm_flag = 1 elif flag == 'scratch': train_spatial_flag = 1 train_temporal_flag = 1 elif flag == 'st4': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 1 elif flag == 'st7': train_spatial_flag = 1 train_temporal_flag = 1 channel_flag = 2 ######################################### ############ Reading Images and Labels ################ SubperdB = Read_Input_Images_SAMM_CASME(inputDir, list_samples, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size, channel) print("Loaded Images into the tray...") labelperSub = label_matching(workplace, dB, subjects, VidPerSubject) print("Loaded Labels into the tray...") if channel_flag == 1: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) elif channel_flag == 2: SubperdB_strain = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_Strain_TIM10', resizedFlag, table, workplace, spatial_size, 1) SubperdB_gray = Read_Input_Images(inputDir, listOfIgnoredSamples, 'CASME2_TIM', resizedFlag, table, workplace, spatial_size, 3) ####################################################### ########### Model Configurations ####################### sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True) adam = optimizers.Adam(lr=0.00001, decay=0.000001) adam2 = optimizers.Adam(lr=0.00075, decay=0.0001) # Different Conditions for Temporal Learning ONLY if train_spatial_flag == 0 and train_temporal_flag == 1 and dB != 'CASME2_Optical': data_dim = spatial_size * spatial_size elif train_spatial_flag == 0 and train_temporal_flag == 1 and dB == 'CASME2_Optical': data_dim = spatial_size * spatial_size * 3 else: data_dim = 4096 ######################################################## ########### Training Process ############ # total confusion matrix to be used in the computation of f1 score tot_mat = np.zeros((n_exp, n_exp)) # model checkpoint spatial_weights_name = 'vgg_spatial_' + str(train_id) + '_casme2_' temporal_weights_name = 'temporal_ID_' + str(train_id) + '_casme2_' history = LossHistory() stopping = EarlyStopping(monitor='loss', min_delta=0, mode='min') for sub in range(subjects): ############### Reinitialization & weights reset of models ######################## vgg_model_cam = VGG_16(spatial_size=spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') temporal_model = temporal_module(data_dim=data_dim, classes=classes, timesteps_TIM=timesteps_TIM) temporal_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) conv_ae = convolutional_autoencoder(spatial_size=spatial_size, classes=classes) conv_ae.compile(loss='binary_crossentropy', optimizer=adam) if channel_flag == 1 or channel_flag == 2: vgg_model = VGG_16_4_channels(classes=classes, spatial_size=spatial_size) vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) else: vgg_model = VGG_16(spatial_size=spatial_size, classes=classes, weights_path='VGG_Face_Deep_16.h5') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.categorical_accuracy]) svm_classifier = SVC(kernel='linear', C=1) #################################################################################### ############ for tensorboard ############### if tensorboard_flag == 1: cat_path = tensorboard_path + str(sub) + "/" os.mkdir(cat_path) tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path, write_graph=True) cat_path2 = tensorboard_path + str(sub) + "spat/" os.mkdir(cat_path2) tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2, write_graph=True) ############################################# image_label_mapping = np.empty([0]) Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO( sub, SubperdB, labelperSub, subjects, classes) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0] * timesteps_TIM, r, w, channel) Test_X_spatial = Test_X.reshape(Test_X.shape[0] * timesteps_TIM, r, w, channel) # Special Loading for 4-Channel if channel_flag == 1: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X.shape[0] * timesteps_TIM, r, w, 1) # Concatenate Train X & Train_X_Strain Train_X_spatial = np.concatenate((Train_X_spatial, Train_X_Strain), axis=3) Test_X_spatial = np.concatenate((Test_X_spatial, Test_X_Strain), axis=3) channel = 4 elif channel_flag == 2: Train_X_Strain, _, Test_X_Strain, _, _ = data_loader_with_LOSO( sub, SubperdB_strain, labelperSub, subjects, classes) Train_X_gray, _, Test_X_gray, _, _ = data_loader_with_LOSO( sub, SubperdB_gray, labelperSub, subjects) Train_X_Strain = Train_X_Strain.reshape( Train_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Test_X_Strain = Test_X_Strain.reshape( Test_X_Strain.shape[0] * timesteps_TIM, r, w, 1) Train_X_gray = Train_X_gray.reshape( Train_X_gray.shape[0] * timesteps_TIM, r, w, 3) Test_X_gray = Test_X_gray.reshape( Test_X_gray.shape[0] * timesteps_TIM, r, w, 3) # Concatenate Train_X_Strain & Train_X & Train_X_gray Train_X_spatial = np.concatenate( (Train_X_spatial, Train_X_Strain, Train_X_gray), axis=3) Test_X_spatial = np.concatenate( (Test_X_spatial, Test_X_Strain, Test_X_gray), axis=3) channel = 7 if channel == 1: # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) Test_X_spatial = duplicate_channel(Test_X_spatial) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, timesteps_TIM, axis=0) Test_Y_spatial = np.repeat(Test_Y, timesteps_TIM, axis=0) # print ("Train_X_shape: " + str(np.shape(Train_X_spatial))) # print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial))) # print ("Test_X_shape: " + str(np.shape(Test_X_spatial))) # print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial))) # print(Train_X_spatial) ##################### Training & Testing ######################### X = Train_X_spatial.reshape(Train_X_spatial.shape[0], channel, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], classes) normalized_X = X.astype('float32') / 255. test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], channel, r, w) test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], classes) normalized_test_X = test_X.astype('float32') / 255. print(X.shape) ###### conv weights must be freezed for transfer learning ###### if finetuning_flag == 1: for layer in vgg_model.layers[:33]: layer.trainable = False if train_spatial_flag == 1 and train_temporal_flag == 1: # Autoencoder features # conv_ae.fit(normalized_X, normalized_X, batch_size=batch_size, epochs=spatial_epochs, shuffle=True) # Spatial Training if tensorboard_flag == 1: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[tbCallBack2]) else: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[history, stopping]) # record f1 and loss file_loss = open( workplace + 'Classification/' + 'Result/' + dB + '/loss_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.losses) + "\n") file_loss.close() file_loss = open( workplace + 'Classification/' + 'Result/' + dB + '/accuracy_' + str(train_id) + '.txt', 'a') file_loss.write(str(history.accuracy) + "\n") file_loss.close() vgg_model.save_weights(spatial_weights_name + str(sub) + ".h5")
def train_smic(batch_size, spatial_epochs, temporal_epochs, train_id): ############################## Loading Labels & Images ############################## # /media/ice/OS/Datasets/SMIC_TIM10/SMIC_TIM10 root_db_path = "/media/ice/OS/Datasets/" dB = "SMIC_TIM10" inputDir = root_db_path + dB + "/" + dB + "/" workplace = root_db_path + dB + "/" subject, filename, label, num_frames = loading_smic_labels( root_db_path, dB) filename = filename.as_matrix() label = label.as_matrix() table = np.transpose(np.array([filename, label])) # os.remove(workplace + "Classification/SMIC_label.txt") ################# Variables ############################# spatial_size = 224 r = w = spatial_size subjects = 16 samples = 164 n_exp = 3 IgnoredSamples_index = np.empty([0]) VidPerSubject = get_subfolders_num(inputDir, IgnoredSamples_index) listOfIgnoredSamples = [] timesteps_TIM = 10 data_dim = r * w pad_sequence = 10 ######################################################### ############## Flags #################### resizedFlag = 1 train_spatial_flag = 1 train_temporal_flag = 1 svm_flag = 0 finetuning_flag = 1 tensorboard_flag = 0 cam_visualizer_flag = 1 ######################################### ############## Reading Images and Labels ################ # SubperdB = Read_SMIC_Images(inputDir, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size) SubperdB = Read_Input_Images(inputDir, listOfIgnoredSamples, dB, resizedFlag, table, workplace, spatial_size) labelperSub = label_matching(workplace, dB, subjects, VidPerSubject) ###################################################################################### ########### Model ####################### sgd = optimizers.SGD(lr=0.0001, decay=1e-7, momentum=0.9, nesterov=True) adam = optimizers.Adam(lr=0.00001) if train_spatial_flag == 0 and train_temporal_flag == 1: data_dim = spatial_size * spatial_size else: data_dim = 4096 temporal_model = temporal_module ######################################### ################# Pretrained Model ################### vgg_model = VGG_16('VGG_Face_Deep_16.h5') # keras_vgg = keras_vgg16(weights='imagenet') # vgg_model = VGG_16('imagenet') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.sparse_categorical_accuracy]) plot_model(vgg_model, to_file='model.png', show_shapes=True) svm_classifier = SVC(kernel='linear', C=1) ###################################################### # model checkpoint spatial_weights_name = 'vgg_spatial_17a_smic_' temporal_weights_name = 'temporal_ID_16_smic_' # model checkpoint root = "/home/viprlab/Documents/Micro-Expression/" + spatial_weights_name + "weights.{epoch:02d}-{val_loss:.2f}.hdf5" root_temporal = "/home/viprlab/Documents/Micro-Expression/" + temporal_weights_name + "weights.{epoch:02d}-{val_loss:.2f}.hdf5" model_checkpoint = keras.callbacks.ModelCheckpoint(root, monitor='loss', save_best_only=True, save_weights_only=True) model_checkpoint_temporal = keras.callbacks.ModelCheckpoint( root_temporal, monitor='loss', save_best_only=True, save_weights_only=True) ########### Training Process ############ # Todo: # 1) LOSO (done) # 2) call model (done) # 3) saving model architecture # 4) Saving Checkpoint # 5) make prediction (done) if tensorboard_flag == 1: tensorboard_path = "/home/ice/Documents/Micro-Expression/tensorboard/" tot_mat = np.zeros((n_exp, n_exp)) spatial_weights_name = 'vgg_spatial_ID_under_dev_smic.h5' temporal_weights_name = 'temporal_ID_under_dev_smic.h5' for sub in range(subjects): vgg_model = VGG_16('VGG_Face_Deep_16.h5') vgg_model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=[metrics.sparse_categorical_accuracy]) ############ for tensorboard ############### if tensorboard_flag == 1: cat_path = tensorboard_path + str(sub) + "/" os.mkdir(cat_path) tbCallBack = keras.callbacks.TensorBoard(log_dir=cat_path, write_graph=True) cat_path2 = tensorboard_path + str(sub) + "spat/" os.mkdir(cat_path2) tbCallBack2 = keras.callbacks.TensorBoard(log_dir=cat_path2, write_graph=True) ############################################# image_label_mapping = np.empty([0]) Train_X, Train_Y, Test_X, Test_Y, Test_Y_gt = data_loader_with_LOSO( sub, SubperdB, labelperSub, subjects) # Rearrange Training labels into a vector of images, breaking sequence Train_X_spatial = Train_X.reshape(Train_X.shape[0] * 10, r, w, 1) Test_X_spatial = Test_X.reshape(Test_X.shape[0] * 10, r, w, 1) # Extend Y labels 10 fold, so that all images have labels Train_Y_spatial = np.repeat(Train_Y, 10, axis=0) Test_Y_spatial = np.repeat(Test_Y, 10, axis=0) # Duplicate channel of input image Train_X_spatial = duplicate_channel(Train_X_spatial) Test_X_spatial = duplicate_channel(Test_X_spatial) # print ("Train_X_shape: " + str(np.shape(Train_X_spatial))) # print ("Train_Y_shape: " + str(np.shape(Train_Y_spatial))) # print ("Test_X_shape: " + str(np.shape(Test_X_spatial))) # print ("Test_Y_shape: " + str(np.shape(Test_Y_spatial))) # print(Train_X_spatial) ##################### Training & Testing ######################### X = Train_X_spatial.reshape(Train_X_spatial.shape[0], 3, r, w) y = Train_Y_spatial.reshape(Train_Y_spatial.shape[0], 5) test_X = Test_X_spatial.reshape(Test_X_spatial.shape[0], 3, r, w) test_y = Test_Y_spatial.reshape(Test_Y_spatial.shape[0], 5) ###### conv weights must be freezed for transfer learning ###### if finetuning_flag == 1: for layer in vgg_model.layers[:33]: layer.trainable = False if train_spatial_flag == 1 and train_temporal_flag == 1: # trains encoder until fc, train temporal # Spatial Training if tensorboard_flag == 1: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[tbCallBack2]) else: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[model_checkpoint]) # vgg_model.save_weights(spatial_weights_name) model = Model(inputs=vgg_model.input, outputs=vgg_model.layers[35].output) plot_model(model, to_file="spatial_module_FULL_TRAINING.png", show_shapes=True) # Spatial Encoding output = model.predict(X, batch_size=batch_size) features = output.reshape(int(output.shape[0] / 10), 10, output.shape[1]) # Temporal Training if tensorboard_flag == 1: temporal_model.fit(features, Train_Y, batch_size=batch_size, epochs=temporal_epochs, callbacks=[tbCallBack]) else: temporal_model.fit(features, Train_Y, batch_size=batch_size, epochs=temporal_epochs, callbacks=[model_checkpoint_temporal]) # temporal_model.save_weights(temporal_weights_name) # Testing output = model.predict(test_X, batch_size=batch_size) features = output.reshape(int(output.shape[0] / 10), 10, output.shape[1]) predict = temporal_model.predict_classes(features, batch_size=batch_size) elif train_spatial_flag == 1 and train_temporal_flag == 0 and cam_visualizer_flag == 0: # trains spatial module ONLY, no escape # Spatial Training if tensorboard_flag == 1: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[tbCallBack2]) else: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[model_checkpoint]) # vgg_model.save_weights(spatial_weights_name) plot_model(vgg_model, to_file="spatial_module_ONLY.png", show_shapes=True) # Testing predict = vgg_model.predict(test_X, batch_size=batch_size) Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0) elif train_spatial_flag == 0 and train_temporal_flag == 1: # trains temporal module ONLY. # Temporal Training if tensorboard_flag == 1: temporal_model.fit(Train_X, Train_Y, batch_size=batch_size, epochs=spatial_epochs, callbacks=[tbCallBack]) else: temporal_model.fit(Train_X, Train_Y, batch_size=batch_size, epochs=spatial_epochs, callbacks=[model_checkpoint_temporal]) # temporal_model.save_weights(temporal_weights_name) # Testing predict = temporal_model.predict_classes(Test_X, batch_size=batch_size) elif svm_flag == 1 and finetuning_flag == 0: # no finetuning X = vgg_model.predict(X, batch_size=batch_size) y_for_svm = np.argmax(y, axis=1) svm_classifier.fit(X, y_for_svm) test_X = vgg_model.predict(test_X, batch_size=batch_size) predict = svm_classifier.predict(test_X) Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0) elif train_spatial_flag == 1 and train_temporal_flag == 0 and cam_visualizer_flag == 1: # trains spatial module & CAM ONLY # Spatial Training if tensorboard_flag == 1: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[tbCallBack2]) else: vgg_model.fit(X, y, batch_size=batch_size, epochs=spatial_epochs, shuffle=True, callbacks=[model_checkpoint]) # vgg_model.save_weights(spatial_weights_name) plot_model(vgg_model, to_file="spatial_module_CAM_ONLY.png", show_shapes=True) # Testing predict = vgg_model.predict(test_X, batch_size=batch_size) Test_Y_gt = np.repeat(Test_Y_gt, 10, axis=0) ############################################################## #################### Confusion Matrix Construction ############# print(predict) print(Test_Y_gt) ct = confusion_matrix(Test_Y_gt, predict) # check the order of the CT order = np.unique(np.concatenate((predict, Test_Y_gt))) # create an array to hold the CT for each CV mat = np.zeros((n_exp, n_exp)) # put the order accordingly, in order to form the overall ConfusionMat for m in range(len(order)): for n in range(len(order)): mat[int(order[m]), int(order[n])] = ct[m, n] tot_mat = mat + tot_mat ################################################################ #################### cumulative f1 plotting ###################### microAcc = np.trace(tot_mat) / np.sum(tot_mat) [f1, precision, recall] = fpr(tot_mat, n_exp) file = open(workplace + 'Classification/' + 'Result/' + '/f1.txt', 'a') file.write(str(f1) + "\n") file.close() ################################################################## ################# write each CT of each CV into .txt file ##################### record_scores(workplace, dB, ct, sub, order, tot_mat, n_exp, subjects) ###############################################################################