def summary(maml_result, set): print(set) print_str = 'Iteration ' + str(itr) print(print_str) y_hata = np.vstack(np.array(maml_result[-2][0])) # length = num_of_task * N * K y_laba = np.vstack(np.array(maml_result[-2][1])) save_path = "./logs/result/train/" + trained_model_dir + "/" + str(FLAGS.au_idx) + "/" if FLAGS.keep_train_dir: retrained_model_dir = 'sbjt' + str(FLAGS.sbjt_start_idx) + '.ubs_' + str( FLAGS.train_update_batch_size) + '.numstep' + str( FLAGS.num_updates) + '.updatelr' + str( FLAGS.train_update_lr) + '.metalr' + str(FLAGS.meta_lr) save_path += retrained_model_dir if not os.path.exists(save_path): os.makedirs(save_path) print_summary(y_hata, y_laba, log_dir=save_path + "/outa_" + set + "_" + str(itr) + ".txt") print("------------------------------------------------------------------------------------") recent_y_hatb = np.array(maml_result[-1][0][ FLAGS.num_updates - 1]) y_hatb = np.vstack(recent_y_hatb) recent_y_labb = np.array(maml_result[-1][1][FLAGS.num_updates - 1]) y_labb = np.vstack(recent_y_labb) print_summary(y_hatb, y_labb, log_dir=save_path + "/outb_" + set + "_" + str(itr) + ".txt") print("================================================================================")
def main(): os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu TOTAL_NUM_AU = 8 all_au = ['au1', 'au2', 'au4', 'au6', 'au9', 'au12', 'au25', 'au26'] if not FLAGS.train: orig_meta_batch_size = FLAGS.meta_batch_size # always use meta batch size of 1 when testing. FLAGS.meta_batch_size = 1 temp_kshot = FLAGS.update_batch_size FLAGS.update_batch_size = 1 if FLAGS.model.startswith('m2'): temp_num_updates = FLAGS.num_updates FLAGS.num_updates = 1 data_generator = DataGenerator() dim_output = data_generator.num_classes dim_input = data_generator.dim_input inputa, inputb, labela, labelb = data_generator.make_data_tensor() metatrain_input_tensors = {'inputa': inputa, 'inputb': inputb, 'labela': labela, 'labelb': labelb} model = MAML(dim_input, dim_output) model.construct_model(input_tensors=metatrain_input_tensors, prefix='metatrain_') model.summ_op = tf.summary.merge_all() saver = loader = tf.train.Saver(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES), max_to_keep=20) sess = tf.InteractiveSession() if not FLAGS.train: # change to original meta batch size when loading model. FLAGS.update_batch_size = temp_kshot FLAGS.meta_batch_size = orig_meta_batch_size if FLAGS.model.startswith('m2'): FLAGS.num_updates = temp_num_updates if FLAGS.train_update_batch_size == -1: FLAGS.train_update_batch_size = FLAGS.update_batch_size if FLAGS.train_update_lr == -1: FLAGS.train_update_lr = FLAGS.update_lr tf.global_variables_initializer().run() tf.train.start_queue_runners() print('initial weights: ', sess.run('model/b1:0')) print("========================================================================================") ################## Test ################## def _load_weight_m(trained_model_dir): all_au = ['au1', 'au2', 'au4', 'au6', 'au9', 'au12', 'au25', 'au26'] if FLAGS.au_idx < TOTAL_NUM_AU: all_au = [all_au[FLAGS.au_idx]] w_arr = None b_arr = None for au in all_au: model_file = None print('model file dir: ', FLAGS.logdir + '/' + au + '/' + trained_model_dir) model_file = tf.train.latest_checkpoint(FLAGS.logdir + '/' + au + '/' + trained_model_dir) print("model_file from ", au, ": ", model_file) if (model_file == None): print( "############################################################################################") print("####################################################################### None for ", au) print( "############################################################################################") else: if FLAGS.test_iter > 0: files = os.listdir(model_file[:model_file.index('model')]) if 'model' + str(FLAGS.test_iter) + '.index' in files: model_file = model_file[:model_file.index('model')] + 'model' + str(FLAGS.test_iter) print("model_file by test_iter > 0: ", model_file) else: print(" >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", files) print("Restoring model weights from " + model_file) saver.restore(sess, model_file) w = sess.run('model/w1:0') b = sess.run('model/b1:0') print("updated weights from ckpt: ", b) print('----------------------------------------------------------') if w_arr is None: w_arr = w b_arr = b else: w_arr = np.hstack((w_arr, w)) b_arr = np.vstack((b_arr, b)) return w_arr, b_arr def _load_weight_s(sbjt_start_idx): batch_size = 10 # 모든 au 를 이용하여 한 모델을 만든경우 그 한 모델만 로드하면됨. if FLAGS.model.startswith('s1'): three_layers = feature_layer(batch_size, TOTAL_NUM_AU) three_layers.loadWeight(FLAGS.vae_model_to_test, FLAGS.au_idx, num_au_for_rm=TOTAL_NUM_AU) # 각 au별로 다른 모델인 경우 au별 weight을 쌓아줘야함 else: three_layers = feature_layer(batch_size, 1) all_au = ['au1', 'au2', 'au4', 'au6', 'au9', 'au12', 'au25', 'au26'] if FLAGS.au_idx < TOTAL_NUM_AU: all_au = [all_au[FLAGS.au_idx]] w_arr = None b_arr = None for au in all_au: if FLAGS.model.startswith('s3'): load_model_path = FLAGS.vae_model_to_test + '/' + FLAGS.model + '_' + au + '_kshot' + str( FLAGS.update_batch_size) + '_iter100' elif FLAGS.model.startswith('s4'): load_model_path = FLAGS.vae_model_to_test + '/' + FLAGS.model + '_' + au + '_subject' + str( sbjt_start_idx + 1) + '_kshot' + str( FLAGS.update_batch_size) + '_iter10_maml_adad' + str(FLAGS.test_iter) else: load_model_path = FLAGS.vae_model_to_test + '/' + FLAGS.model + '_' + au + '_kshot' + str( FLAGS.update_batch_size) + '_iter200_kshot10_iter10_nobatch_adam_noinit' three_layers.loadWeight(load_model_path, au) print('=============== Model S loaded from ', load_model_path) w = three_layers.model_intensity.layers[-1].get_weights()[0] b = three_layers.model_intensity.layers[-1].get_weights()[1] print('----------------------------------------------------------') if w_arr is None: w_arr = w b_arr = b else: w_arr = np.hstack((w_arr, w)) b_arr = np.vstack((b_arr, b)) return w_arr, b_arr def _load_weight_m0(trained_model_dir): model_file = None print('--------- model file dir: ', FLAGS.logdir + trained_model_dir) model_file = tf.train.latest_checkpoint(FLAGS.logdir + trained_model_dir) print(">>>> model_file from all_aus: ", model_file) if (model_file == None): print("####################################################################### None for all_aus") else: if FLAGS.test_iter > 0: files = os.listdir(model_file[:model_file.index('model')]) if 'model' + str(FLAGS.test_iter) + '.index' in files: model_file = model_file[:model_file.index('model')] + 'model' + str(FLAGS.test_iter) print(">>>> model_file2: ", model_file) else: print(" >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>", files) print("Restoring model weights from " + model_file) saver.restore(sess, model_file) w = sess.run('model/w1:0') b = sess.run('model/b1:0') print("updated weights from ckpt: ", b) print('----------------------------------------------------------') return w, b print("<<<<<<<<<<<< CONCATENATE >>>>>>>>>>>>>>") save_path = "./logs/result/" y_hat = [] y_lab = [] if FLAGS.all_sub_model: # 모델이 모든 subjects를 이용해 train된 경우 print('---------------- all sub model ----------------') # weight load를 한번만 실행해도됨. subject별로 모델이 다르지 않기 때문 if FLAGS.model.startswith('m'): trained_model_dir = '/cls_' + str(FLAGS.num_classes) + '.mbs_' + str( FLAGS.meta_batch_size) + '.ubs_' + str( FLAGS.train_update_batch_size) + '.numstep' + str(FLAGS.num_updates) + '.updatelr' + str( FLAGS.train_update_lr) + '.metalr' + str(FLAGS.meta_lr) if FLAGS.model.startswith('m0'): w_arr, b_arr = _load_weight_m0(trained_model_dir) else: w_arr, b_arr = _load_weight_m(trained_model_dir) # au별로 모델이 다르게됨 ### test per each subject and concatenate for i in range(FLAGS.sbjt_start_idx, FLAGS.sbjt_start_idx + FLAGS.num_test_tasks): if FLAGS.model.startswith('s'): w_arr, b_arr = _load_weight_s(i) result = test_each_subject(w_arr, b_arr, i) y_hat.append(result[0]) y_lab.append(result[1]) print("y_hat shape:", result[0].shape) print("y_lab shape:", result[1].shape) print(">> y_hat_all shape:", np.vstack(y_hat).shape) print(">> y_lab_all shape:", np.vstack(y_lab).shape) print_summary(np.vstack(y_hat), np.vstack(y_lab), log_dir=save_path + "/" + "test.txt") else: # 모델이 각 subject 별로 train된 경우: vae와 MAML의 train_test두 경우에만 존재 가능 + local weight test의 경우 for subj_idx in range(FLAGS.sbjt_start_idx, FLAGS.sbjt_start_idx + FLAGS.num_test_tasks): if FLAGS.model.startswith('s'): w_arr, b_arr = _load_weight_s(subj_idx) else: trained_model_dir = '/sbjt' + str(subj_idx) + '.ubs_' + str( FLAGS.train_update_batch_size) + '.numstep' + str(FLAGS.num_updates) + '.updatelr' + str( FLAGS.train_update_lr) + '.metalr' + str(FLAGS.meta_lr) w_arr, b_arr = _load_weight_m(trained_model_dir) result = test_each_subject(w_arr, b_arr, subj_idx) y_hat.append(result[0]) y_lab.append(result[1]) print("y_hat shape:", result[0].shape) print("y_lab shape:", result[1].shape) print(">> y_hat_all shape:", np.vstack(y_hat).shape) print(">> y_lab_all shape:", np.vstack(y_lab).shape) print_summary(np.vstack(y_hat), np.vstack(y_lab), log_dir=save_path + "/test.txt") end_time = datetime.now() elapse = end_time - start_time print("=======================================================") print(">>>>>> elapse time: " + str(elapse)) print("=======================================================")
#path = r_path + iter + '/' + kshot + 'kshot/seed' + str(seed) y_lab_all = [] y_hat_all = [] f1_scores_per_seed = [] for subject_idx in range(13): print("=============================subject_idx: ", subject_idx) file = pickle.load(open( path + '/predicted_subject' + str(subject_idx) + '.pkl', 'rb'), encoding='latin1') y_lab = file['y_lab'] y_hat = file['y_hat'] y_lab_all.append(y_lab) y_hat_all.append(y_hat) out = print_summary(y_hat, y_lab, log_dir="./logs/result/" + "/test.txt") f1_score = list(out['data'][5]) # add avg throughout all AUs as the last elt f1_score.append(np.average(f1_score)) # stack each subject's f1-score f1_scores_per_seed.append(f1_score) print("-- num of samples:", len(file['all_used_frame_set'])) print(">> y_lab_all shape:", np.vstack(y_lab_all).shape) print(">> y_hat_all shape:", np.vstack(y_hat_all).shape) print('-------------------- avg --------------------') averaged_f1 = np.average(f1_scores_per_seed, axis=0) print(averaged_f1) print('---------------- concatenated ---------------')
def train(model, data_generator, saver, sess, trained_model_dir, resume_itr=0): print("===============> Final in weight: ", sess.run('model/w1:0').shape, sess.run('model/b1:0').shape) SUMMARY_INTERVAL = 10 SAVE_INTERVAL = 5000 if FLAGS.log: train_writer = tf.summary.FileWriter( FLAGS.logdir + '/' + trained_model_dir, sess.graph) feed_dict = {} print('Done initializing, starting training.') aus = ['au1', 'au2', 'au4', 'au6', 'au9', 'au12', 'au25', 'au26'] total_val_stat = [] total_mse = [] two_layer = feature_layer(1, FLAGS.num_au) data_generator.get_validation_data() all_val_feat_vec = data_generator.val_feat_vec all_val_frame = data_generator.val_frame val_subjects = os.listdir(FLAGS.val_data_folder) val_subjects.sort() print('total validation subjects: ', val_subjects) from sklearn.metrics import f1_score for itr in range(resume_itr + 1, FLAGS.metatrain_iterations + 1): if FLAGS.shuffle_batch > 0 and itr % FLAGS.shuffle_batch == 0: print( '=============================================================shuffle data, iteration:', itr) inputa, inputb, labela, labelb, _ = data_generator.shuffle_data( itr, FLAGS.update_batch_size, aus) feed_dict = { model.inputa: inputa, model.inputb: inputb, model.labela: labela, model.labelb: labelb } if itr <= 1000: SAVE_INTERVAL = 100 else: SAVE_INTERVAL = 100 input_tensors = [model.train_op] if (itr % SUMMARY_INTERVAL == 0): input_tensors.extend([model.summ_op]) input_tensors.extend([model.fast_weight_w]) input_tensors.extend([model.fast_weight_b]) result = sess.run(input_tensors, feed_dict) if (itr % SUMMARY_INTERVAL == 0): train_writer.add_summary(result[1], itr) if (itr % SAVE_INTERVAL == 0) or (itr == FLAGS.metatrain_iterations): print("================================================ iter:", itr) print() saver.save( sess, FLAGS.logdir + '/' + trained_model_dir + '/model' + str(itr)) w = [] b = [] w.append(sess.run('model/w1:0')) w.append(sess.run('model/w2:0')) w.append(sess.run('model/w3:0')) b.append(sess.run('model/b1:0')) b.append(sess.run('model/b2:0')) b.append(sess.run('model/b3:0')) ### save global weight ### with open( FLAGS.logdir + '/' + trained_model_dir + "/two_layers" + str(itr) + ".pkl", 'wb') as out: pickle.dump({'w': w, 'b': b}, out, protocol=2) ### save local weight ### with open( FLAGS.logdir + '/' + trained_model_dir + "/per_sub_weight" + str(itr) + ".pkl", 'wb') as out: pickle.dump({ 'w': result[-2], 'b': result[-1] }, out, protocol=2) ### validation ### two_layer.loadWeight(FLAGS.vae_model, w=w, b=b) print('--------------------------------------------------------') print("[Main] loaded soft bias to be evaluated: ", b[0]) print("[Main] loaded z_mean bias to be evaluated : ", b[1][:4]) print('--------------------------------------------------------') total_val_cnt = 0 f1_scores = [] mse = [] for i in range(len(val_subjects)): eval_vec = all_val_feat_vec[i] eval_frame = all_val_frame[i] y_lab = data_generator.labels[i][eval_frame] print( '---------------- len of eval_frame ---------------------') print(len(eval_frame)) y_reconst = two_layer.model_reconst.predict(eval_vec) one_sub_mse = np.average( np.power((np.array(eval_vec) - y_reconst), 2)) print('==================== mse of {} ===================='. format(val_subjects[i])) print(one_sub_mse) print( '=========================================================' ) y_true = np.array([np.eye(2)[label] for label in y_lab]) y_pred = two_layer.model_intensity.predict(eval_vec) print('y_true shape: ', y_true.shape) print('y_pred shape: ', y_pred.shape) total_val_cnt += int(y_true.shape[0]) out = print_summary(y_pred, y_true, log_dir="./logs/result/" + "/test.txt") f1_score = np.average(list(out['data'][5])) f1_scores.append(f1_score) mse.append(one_sub_mse) means = np.mean(f1_scores, 0) stds = np.std(f1_scores, 0) ci95 = 1.96 * stds / np.sqrt(total_val_cnt) mean_mse = np.average(mse) total_val_stat.append((means, stds, ci95)) total_mse.append(mean_mse) print( '================================================================' ) print('total_val_cnt: ', total_val_cnt) print( '(Mean validation f1-score, stddev, and confidence intervals), Mean reconst. loss' ) for i in range(len(total_val_stat)): print('iter:', (i + 1) * SAVE_INTERVAL, total_val_stat[i], total_mse[i])