def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker DATASET = args.dataset if DATASET == 'ferplus': emotion_classes = 8 else: emotion_classes = 7 paths, emotion_label = load_data(DATASET) n_fold = 1 print('[K-FOLD] Started...') kf = KFold(n_splits=5, shuffle=True, random_state=1) kf_split = kf.split(emotion_label) for train_idx, test_idx in kf_split: model = None if MODEL == 'vggFace': model = Net(MODEL, 1, 0, emotion_classes, 2, 2) if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL, 1, 0, emotion_classes, 2, 2) train_paths = paths[train_idx] train_emotion = emotion_label[train_idx] test_paths = paths[test_idx] test_emotion = emotion_label[test_idx] print(len(train_paths), len(test_paths)) losses = { "emotion_prediction": "categorical_crossentropy", } metrics = { "emotion_prediction": "acc", } if MODEL == 'ssrnet': losses = { "emotion_prediction": "mae", } metrics = { "emotion_prediction": "mae", } # -{epoch:02d}-{val_loss:.4f}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f} model.summary() weights_path = './train_weights/emotion/{}/{}/'.format( DATASET, model.name) logs_path = './train_log/emotion/{}/{}/'.format(DATASET, model.name) # plot_model(model,to_file= logs_path+'.jpg',show_shapes=True) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) model_names = weights_path + '.{epoch:02d}-{val_acc:.2f}.hdf5' csv_name = logs_path + '{}.log'.format(n_fold) board_name = logs_path + '{}'.format(n_fold) checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=True) csvlogger = CSVLogger(csv_name) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE / 2), verbose=1) tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] if MODEL == 'ssrnet': callbacks = [ ModelCheckpoint(MODEL, 'val_loss', verbose=1, save_best_only=True), CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)), DecayLearningRate([30, 60]) ] model.compile(optimizer='adam', loss=losses, metrics=metrics) model.fit_generator(DataGenerator(model, train_paths, train_emotion, BATCH_SIZE), validation_data=DataGenerator( model, test_paths, test_emotion, BATCH_SIZE), epochs=args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) n_fold += 1 del train_paths, train_emotion del test_paths, test_emotion
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker EMOTION_DATASET = args.dataset_emotion GENDER_AGE_DATASET = args.dataset_gender_age if EMOTION_DATASET == 'ferplus': emotion_classes = 8 else: emotion_classes = 7 gender_classes = 2 age_classes = 8 pose_classes = 5 # emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5' # pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5' # age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5' paths_emotion, paths_pose, paths_gender_age, emotion_label, pose_label, gender_label, age_label = load_data( EMOTION_DATASET, GENDER_AGE_DATASET, args.dataset_pose) kf = KFold(n_splits=5, shuffle=True, random_state=1) kf_split_gender_age = kf.split(paths_gender_age) kf_split_emotion = kf.split(paths_emotion) kf_split_pose = kf.split(paths_pose) emotion_kf = [[emotion_train_idx, emotion_test_idx] for emotion_train_idx, emotion_test_idx in kf_split_emotion] gender_age_kf = [[ gender_age_train_idx, gender_age_test_idx ] for gender_age_train_idx, gender_age_test_idx in kf_split_gender_age] pose_kf = [[pose_train_idx, pose_test_idx] for pose_train_idx, pose_test_idx in kf_split_pose] emotion_train_idx, emotion_test_idx = emotion_kf[0] pose_train_idx, pose_test_idx = pose_kf[0] gender_age_train_idx, gender_age_test_idx = gender_age_kf[0] print(len(emotion_train_idx), len(gender_age_train_idx), len(pose_train_idx)) print(len(emotion_test_idx), len(gender_age_test_idx), len(pose_test_idx)) train_emotion_paths = paths_emotion[emotion_train_idx] train_emotion = emotion_label[emotion_train_idx] test_emotion_paths = paths_emotion[emotion_test_idx] test_emotion = emotion_label[emotion_test_idx] train_pose_paths = paths_pose[pose_train_idx] train_pose = pose_label[pose_train_idx] test_pose_paths = paths_pose[pose_test_idx] test_pose = pose_label[pose_test_idx] train_gender_age_paths = paths_gender_age[gender_age_train_idx] train_gender = gender_label[gender_age_train_idx] train_age = age_label[gender_age_train_idx] test_gender_age_paths = paths_gender_age[gender_age_test_idx] test_gender = gender_label[gender_age_test_idx] test_age = age_label[gender_age_test_idx] model = None if MODEL == 'vggFace': print('based on vggface') model = Net(MODEL, 1, 12, emotion_classes, pose_classes, age_classes, gender_classes, args.is_dropout, args.is_bn, args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL, 1, 12, emotion_classes, pose_classes, age_classes, gender_classes, args.is_dropout, args.is_bn, args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) loss_dict = {} metrics_dict = {} prediction = [] loss = [] acc = [] loss_weights = [] prediction.append('emotion_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.E_loss_weights) prediction.append('pose_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.P_loss_weights) prediction.append('gender_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.G_loss_weights) prediction.append('age_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.A_loss_weights) loss_dict = dict(zip(prediction, loss)) metrics_dict = dict(zip(prediction, acc)) #pesudo label if args.is_naive: is_naive = 'naive_true' print('this is naive method') else: is_naive = 'naive_false' if args.is_distilled: is_distilled = 'distilled_true' print('this is distiiled method') else: is_distilled = 'distilled_false' if args.is_pesudo: is_pesudo = 'pesudo_true' print('this is pesudo method') else: is_pesudo = 'pesudo_false' if args.is_interpolation: is_interpolation = 'interpolation_true' print('this is interpolation method') else: is_interpolation = 'interpolation_false' if args.selection_threshold > 0: print('this is pesudo selection method') #training trich if args.is_augmentation: augmentation = 'augmentation_true' else: augmentation = 'augmentation_false' if args.is_dropout: dropout = 'dropout_true' else: dropout = 'dropout_false' if args.is_bn: bn = 'bn_true' else: bn = 'bn_false' #import parameter if args.is_freezing: freezing = 'freezing_true' else: freezing = 'freezing_false' main_path = '{}-{}-{}/{}/'.format(EMOTION_DATASET, GENDER_AGE_DATASET, args.dataset_pose, MODEL) appendex_path = 'net-{}_{}_{}_{}-pesudo-{}_{}_{}_{}_{}_{}-lr-{}_{}_{}_{}_{}_'.format( augmentation, dropout, bn, args.weights_decay, is_naive, is_distilled, is_pesudo, is_interpolation, args.selection_threshold, args.interpolation_weights, args.E_loss_weights, args.G_loss_weights, args.A_loss_weights, freezing, args.no_freezing_epoch) weights_path = './train_weights/CONFUSION/' + main_path + appendex_path logs_path = './train_log/CONFUSION/' + main_path + appendex_path matrix_path = './train_log/CONFUSION/' + main_path + appendex_path + '/matrix/' if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) if not os.path.exists(matrix_path): os.makedirs(matrix_path) model_names = weights_path + '.{epoch:02d}-{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}-{val_gender_prediction_my_acc:.4f}-{val_age_prediction_my_acc:.4f}.hdf5' csv_name = logs_path + '.log' checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=False) csvlogger = CSVLogger(filename=csv_name, append=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE), verbose=1) tensorboard = TensorBoard(log_dir=logs_path, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] print('whether freezing:', args.is_freezing) if not args.is_freezing: adam = keras.optimizers.Adam(lr=0.0001) model = no_freeze_all(model) model.compile(optimizer=adam, loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch == 0 or args.is_naive: model_pre = None else: model_pre = keras.models.clone_model(model) model_pre.set_weights(model.get_weights()) model.fit_generator( DataGenerator(model, model_pre, train_emotion_paths, train_pose_paths, train_gender_age_paths, train_emotion, train_pose, train_gender, train_age, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, is_augmentation=args.is_augmentation), validation_data=DataGenerator(model, None, test_emotion_paths, test_pose_paths, test_gender_age_paths, test_emotion, test_pose, test_gender, test_age, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, is_augmentation=False), epochs=1, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) print('{}_train_finished'.format(epoch)) emotion_predict = model.predict_generator( emotion_DataGenerator(model, test_emotion_paths, test_emotion, args.batch_size)) emotion_reuslt = np.argmax(emotion_predict[0], axis=1) emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt) age_predict = model.predict_generator( age_DataGenerator(model, test_gender_age_paths, test_age, args.batch_size)) age_reuslt = np.argmax(age_predict[2], axis=1) age_matrix = confusion_matrix(test_age, age_reuslt) print('emotion confusion matrix:', emotion_matrix) print('age confusion matrix:', age_matrix) np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch), emotion_matrix) np.savetxt(matrix_path + '{}_age_matrix.txt'.format(epoch), age_matrix) else: model.compile(optimizer='adam', loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch == 0 or args.is_naive: model_pre = None else: model_pre = keras.models.clone_model(model) model_pre.set_weights(model.get_weights()) model.fit_generator( DataGenerator(model, model_pre, train_emotion_paths, train_pose_paths, train_gender_age_paths, train_emotion, train_pose, train_gender, train_age, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, is_augmentation=args.is_augmentation), validation_data=DataGenerator(model, None, test_emotion_paths, test_pose_paths, test_gender_age_paths, test_emotion, test_pose, test_gender, test_age, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, is_augmentation=False), epochs=1, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) if epoch == args.no_freezing_epoch: print('no freezing and start to changing learning rate') model = no_freeze_all(model) adam = keras.optimizers.Adam(lr=1 * 1e-4) print('lr changing to:', 1 * 1e-4) model.compile(optimizer=adam, loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) print('epoch is{}'.format(epoch)) model.summary() print('{}_train_finished'.format(epoch)) emotion_predict = model.predict_generator( emotion_DataGenerator(model, test_emotion_paths, test_emotion, args.batch_size)) emotion_reuslt = np.argmax(emotion_predict[0], axis=1) emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt) age_predict = model.predict_generator( age_DataGenerator(model, test_gender_age_paths, test_age, args.batch_size)) age_reuslt = np.argmax(age_predict[2], axis=1) age_matrix = confusion_matrix(test_age, age_reuslt) print('emotion confusion matrix:', emotion_matrix) print('age confusion matrix:', age_matrix) np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch), emotion_matrix) np.savetxt(matrix_path + '{}_age_matrix.txt'.format(epoch), age_matrix)
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker loss_dict = {} metrics_dict = {} attr_predcition = [] loss = [] acc = [] for i in range(40): attr_predcition.append('attr{}_predition'.format(i)) loss.append('binary_crossentropy') acc.append('acc') # print(attr_predcition,loss,acc) loss_dict = dict(zip(attr_predcition, loss)) metrics_dict = dict(zip(attr_predcition, acc)) losses = {"age_prediction": mae} metrics = {"age_prediction": mae} model = None if MODEL == 'vggFace': model = Net(MODEL, 1, 6, 8, 2, 8) model = freeze_all_but_mid_and_top(model) MODEL = model.name else: model = Net(MODEL, 1, 6, 8, 2, 8) MODEL = model.name paths, attr_label = load_data(DATASET) # model.summary() n_fold = 1 print('[K-FOLD] Started...') kf = KFold(n_splits=10, shuffle=True, random_state=1) kf_split = kf.split(attr_label) for train_idx, test_idx in kf_split: print('length:train test:', len(train_idx), len(test_idx)) train_paths = paths[train_idx] train_attr = attr_label[train_idx] test_paths = paths[test_idx] test_attr = attr_label[test_idx] if MODEL == 'ssrnet': losses = { "age_prediction": "mae", "gender_prediction": "mae", } metrics = { "age_prediction": "mae", "gender_prediction": "binary_accuracy", } weights_path = './train_weights/attr/{}/{}/'.format(DATASET, MODEL) logs_path = './train_log/attr/{}/{}/'.format(DATASET, MODEL) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) # model_names = weights_path + '{epoch:02d}-{val_acc:.2f}.hdf5' model_names = weights_path + '{epoch:02d}.hdf5' csv_name = logs_path + '{}.log'.format(n_fold) board_name = logs_path + '{}'.format(n_fold) checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=True) csvlogger = CSVLogger(csv_name) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE / 2), verbose=1) tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] model.compile(optimizer='adam', loss=loss_dict, metrics=metrics_dict) model.fit_generator(DataGenerator(model, train_paths, train_attr, BATCH_SIZE), validation_data=DataGenerator( model, test_paths, test_attr, BATCH_SIZE), epochs=EPOCH, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) n_fold += 1 del train_paths, train_attr del test_paths, test_attr
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker EMOTION_DATASET = args.dataset_emotion POSE_DATASET = args.dataset_pose if EMOTION_DATASET == 'ferplus': emotion_classes = 8 else: emotion_classes = 7 gender_classes = 2 age_classes = 8 pose_classes = 5 # emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5' # pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5' # age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5' paths_emotion, paths_pose, emotion_label,pose_label = load_data(EMOTION_DATASET,POSE_DATASET) kf = KFold(n_splits=5, shuffle=True, random_state=1) kf_split_emotion = kf.split(paths_emotion) kf_split_pose = kf.split(paths_pose) emotion_kf = [[emotion_train_idx,emotion_test_idx] for emotion_train_idx,emotion_test_idx in kf_split_emotion] pose_kf = [[pose_train_idx,pose_test_idx] for pose_train_idx,pose_test_idx in kf_split_pose] # for emotion_train_idx,emotion_test_idx in kf_split_emotion: # for gender_age_train_idx,gender_age_test_idx in kf_split_gender_age: # print(emotion_train_idx,emotion_test_idx,gender_age_train_idx,gender_age_test_idx) emotion_train_idx,emotion_test_idx = emotion_kf[0] pose_train_idx,pose_test_idx = pose_kf[0] print(len(emotion_train_idx),len(pose_train_idx)) print(len(emotion_test_idx),len(pose_test_idx)) train_emotion_paths = paths_emotion[emotion_train_idx] train_emotion = emotion_label[emotion_train_idx] test_emotion_paths = paths_emotion[emotion_test_idx] test_emotion = emotion_label[emotion_test_idx] train_pose_paths = paths_pose[pose_train_idx] train_pose = pose_label[pose_train_idx] test_pose_paths = paths_pose[pose_test_idx] test_pose = pose_label[pose_test_idx] model = None if MODEL == 'vggFace': print('based on vggface') model = Net(MODEL,1,11,emotion_classes,pose_classes,age_classes,gender_classes,args.is_dropout,args.is_bn,args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL,1,11,emotion_classes,pose_classes,age_classes,gender_classes,args.is_dropout,args.is_bn,args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) def my_cross_loss(y_true, y_pred): mask = K.all(K.equal(y_true, 0), axis=-1) mask = 1 - K.cast(mask, K.floatx()) loss = K.categorical_crossentropy(y_true, y_pred) * mask return K.sum(loss) / K.sum(mask) def my_bin_loss(y_true, y_pred): mask = K.all(K.equal(y_true, 0), axis=-1) mask = 1 - K.cast(mask, K.floatx()) loss = K.binary_crossentropy(y_true, y_pred) * mask return K.sum(loss) / K.sum(mask) def my_mean_square(y_true,y_pred): mask = K.all(K.equal(y_true, 0), axis=-1) mask = 1 - K.cast(mask, K.floatx()) loss = keras.losses.mean_squared_error(y_true,y_pred)* mask return K.sum(loss) / K.sum(mask) def my_acc(y_true, y_pred): mask = K.all(K.equal(y_true, 0), axis=-1) mask = 1 - K.cast(mask, K.floatx()) acc = (K.cast(K.equal(K.argmax(y_true, axis=-1),K.argmax(y_pred, axis=-1)),K.floatx()))*mask return K.sum(acc)/K.sum(mask) # def pose_metric(y_true,y_pred): # # y_true = K.dot(180/np.pi*K.ones(K.shape(y_true)),y_true) # # y_pred = K.dot(180/np.pi*K.ones(K.shape(y_pred)),y_pred) # y_true = 180/3.14*y_true # y_pred = 180/3.14*y_pred # comp = 15*K.ones(K.shape(y_pred)) # comp = K.cast(comp, K.floatx()) # sub = K.cast(K.abs(y_true-y_pred), K.floatx()) # diff = K.greater(comp,sub) # diff = K.cast(diff, K.floatx()) # result = K.sum(diff,axis=0)/K.cast(K.shape(y_pred)[0], K.floatx()) # return result[0] # def my_pose_metric(y_true,y_pred): # mask = K.all(K.equal(y_true, 0), axis=-1) # mask = 1 - K.cast(mask, K.floatx()) # acc = pose_metric(y_true,y_pred)*mask # return K.sum(acc) / K.sum(mask) loss_dict = {} metrics_dict = {} prediction = [] loss =[] acc =[] loss_weights = [] prediction.append('emotion_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.E_loss_weights) prediction.append('pose_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.P_loss_weights) loss_dict = dict(zip(prediction, loss)) metrics_dict = dict(zip(prediction, acc)) #pesudo label if args.is_naive: is_naive='naive_true' print('this is naive method') else: is_naive='naive_false' if args.is_distilled: is_distilled='distilled_true' print('this is distiiled method') else: is_distilled='distilled_false' if args.is_pesudo: is_pesudo='pesudo_true' print('this is pesudo method') else: is_pesudo='pesudo_false' if args.is_pesudo_density: is_pesudo_density='pesudo_density_true' else: is_pesudo_density='pesudo_density_false' if args.is_pesudo_distribution: is_pesudo_distribution='pesudo_distribution_true' else: is_pesudo_distribution='pesudo_distribution_false' if args.is_interpolation: is_soft_weight = 'soft_weight' print('this is overall soft weight') else: is_soft_weight = 'hard_weight' print('this is overall hard weight') if args.selection_threshold>0: print('this is pesudo selection method') #training trich if args.is_augmentation: augmentation='augmentation_true' else: augmentation='augmentation_false' if args.is_dropout: dropout='dropout_true' else: dropout='dropout_false' if args.is_bn: bn = 'bn_true' else: bn = 'bn_false' #import parameter if args.is_freezing: freezing='freezing_true' else: freezing='freezing_false' main_path='{}-{}/{}/'.format(EMOTION_DATASET,POSE_DATASET,MODEL) appendex_path='net-{}_{}_{}_{}-pesudo-{}_{}_{}_{}_{}_{}_{}_threshold-{}_cluster_k-{}-lr-{}_{}_{}_{}'.format(augmentation,dropout,bn,args.weights_decay, is_naive,is_distilled,is_soft_weight,is_pesudo,is_pesudo_density,is_pesudo_distribution,args.selection_threshold,args.interpolation_weights,args.cluster_k, args.E_loss_weights,args.P_loss_weights,freezing,args.no_freezing_epoch) weights_path = './train_weights/CONFUSION/'+main_path+appendex_path logs_path = './train_log/CONFUSION/'+main_path+appendex_path matrix_path = './train_log/CONFUSION/'+main_path+appendex_path+'/matrix/' if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) if not os.path.exists(matrix_path): os.makedirs(matrix_path) model_names = weights_path +'.{epoch:02d}-{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}.hdf5' csv_name = logs_path + '.log' checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True) csvlogger=CSVLogger(filename=csv_name,append=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE), verbose=1) tensorboard = TensorBoard(log_dir=logs_path,batch_size=BATCH_SIZE) callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard] print('whether freezing:',args.is_freezing) if args.is_pesudo or args.is_interpolation: each_term_epoch = 4 else: each_term_epoch = 1 if not args.is_freezing: adam=keras.optimizers.Adam(lr=0.0001) model = no_freeze_all(model) model.compile(optimizer=adam, loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch==0 or args.is_naive: model_pre = None else: model_pre = Model(inputs=model.input,outputs=model.output) model_pre.set_weights(model.get_weights()) if args.is_pesudo or args.is_interpolation: is_hard_weight = not args.is_interpolation pesudo_emotion = assign_weights(model,False,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights) pesudo_pose = assign_weights(model,True,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights) else: pesudo_emotion=None pesudo_pose=None model.fit_generator( DataGenerator(model,model_pre,train_emotion_paths,train_pose_paths,train_emotion,train_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,args.is_augmentation,pesudo_emotion,pesudo_pose), validation_data = DataGenerator(model, None,test_emotion_paths,test_pose_paths, test_emotion,test_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,False,pesudo_emotion,pesudo_pose), epochs=each_term_epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) print('{}_train_finished'.format(epoch)) emotion_predict = model.predict_generator(emotion_DataGenerator(model,test_emotion_paths,test_emotion,args.batch_size)) emotion_reuslt=np.argmax(emotion_predict[0],axis=1) emotion_matrix=confusion_matrix(test_emotion,emotion_reuslt) pose_predict = model.predict_generator(pose_DataGenerator(model,test_pose_paths,test_pose,args.batch_size)) pose_reuslt=np.argmax(pose_predict[1],axis=1) pose_matrix=confusion_matrix(test_pose,pose_reuslt) print('emotion confusion matrix:',emotion_matrix) print('pose confusion matrix:',pose_matrix) np.savetxt(matrix_path+'{}_emotion_matrix.txt'.format(epoch),emotion_matrix) np.savetxt(matrix_path+'{}_pose_matrix.txt'.format(epoch),pose_matrix) else: model.compile(optimizer='adam', loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch == 0 or args.is_naive: model_pre = None else: model_pre = Model(inputs=model.input,outputs=model.output) model_pre.set_weights(model.get_weights()) if args.is_pesudo or args.is_interpolation: is_hard_weight = not args.is_interpolation pesudo_emotion = assign_weights(model,False,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights) pesudo_pose = assign_weights(model,True,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights) else: pesudo_emotion=None pesudo_pose=None model.fit_generator( DataGenerator(model,model_pre,train_emotion_paths,train_pose_paths,train_emotion,train_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,args.is_augmentation,pesudo_emotion,pesudo_pose), validation_data = DataGenerator(model, None,test_emotion_paths,test_pose_paths, test_emotion,test_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,False,pesudo_emotion,pesudo_pose), epochs=each_term_epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) if epoch==args.no_freezing_epoch: print('no freezing and start to changing learning rate') model = no_freeze_all(model) adam = keras.optimizers.Adam(lr=1*1e-4) print('lr changing to:',1*1e-4) model.compile(optimizer=adam, loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict) print('epoch is{}'.format(epoch)) model.summary() print('{}_train_finished'.format(epoch)) emotion_predict = model.predict_generator(emotion_DataGenerator(model,test_emotion_paths,test_emotion,args.batch_size)) emotion_reuslt=np.argmax(emotion_predict[0],axis=1) emotion_matrix=confusion_matrix(test_emotion,emotion_reuslt) pose_predict = model.predict_generator(pose_DataGenerator(model,test_pose_paths,test_pose,args.batch_size)) pose_reuslt=np.argmax(pose_predict[1],axis=1) pose_matrix=confusion_matrix(test_pose,pose_reuslt) print('emotion confusion matrix:',emotion_matrix) print('pose confusion matrix:',pose_matrix) np.savetxt(matrix_path+'{}_emotion_matrix.txt'.format(epoch),emotion_matrix) np.savetxt(matrix_path+'{}_pose_matrix.txt'.format(epoch),pose_matrix) del train_emotion_paths, train_pose_paths,train_emotion,train_pose del test_emotion_paths, test_pose_paths,test_emotion, test_pose
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker EMOTION_DATASET = args.dataset_emotion POSE_DATASET = args.dataset_pose if EMOTION_DATASET == 'ferplus': emotion_classes = 8 else: emotion_classes = 7 gender_classes = 2 age_classes = 8 pose_classes = 5 # emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5' # pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5' # age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5' paths_emotion, paths_pose, emotion_label, pose_label = load_data( EMOTION_DATASET, POSE_DATASET) kf = KFold(n_splits=100, shuffle=True, random_state=1) kf_split_emotion = kf.split(paths_emotion) kf_split_pose = kf.split(paths_pose) emotion_kf = [[emotion_train_idx, emotion_test_idx] for emotion_train_idx, emotion_test_idx in kf_split_emotion] pose_kf = [[pose_train_idx, pose_test_idx] for pose_train_idx, pose_test_idx in kf_split_pose] # for emotion_train_idx,emotion_test_idx in kf_split_emotion: # for gender_age_train_idx,gender_age_test_idx in kf_split_gender_age: # print(emotion_train_idx,emotion_test_idx,gender_age_train_idx,gender_age_test_idx) emotion_train_idx, emotion_test_idx = emotion_kf[0] pose_train_idx, pose_test_idx = pose_kf[0] print(len(emotion_train_idx), len(pose_train_idx)) print(len(emotion_test_idx), len(pose_test_idx)) train_emotion_paths = paths_emotion[emotion_train_idx] train_emotion = emotion_label[emotion_train_idx] test_emotion_paths = paths_emotion[emotion_test_idx] test_emotion = emotion_label[emotion_test_idx] train_pose_paths = paths_pose[pose_train_idx] train_pose = pose_label[pose_train_idx] test_pose_paths = paths_pose[pose_test_idx] test_pose = pose_label[pose_test_idx] model = None if MODEL == 'vggFace': print('based on vggface') model = Net(MODEL, 1, 11, emotion_classes, pose_classes, age_classes, gender_classes, args.is_dropout, args.is_bn, args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL, 1, 11, emotion_classes, pose_classes, age_classes, gender_classes, args.is_dropout, args.is_bn, args.weights_decay) MODEL = model.name if args.is_freezing: model = freeze_all_but_mid_and_top(model) loss_dict = {} metrics_dict = {} prediction = [] loss = [] acc = [] loss_weights = [] prediction.append('emotion_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.E_loss_weights) prediction.append('pose_prediction') loss.append(my_cross_loss) acc.append(my_acc) loss_weights.append(args.P_loss_weights) loss_dict = dict(zip(prediction, loss)) metrics_dict = dict(zip(prediction, acc)) #pesudo label if args.is_naive: is_naive = 'naive_true' print('this is naive method') else: is_naive = 'naive_false' if args.is_distilled: is_distilled = 'distilled_true' print('this is distiiled method') else: is_distilled = 'distilled_false' if args.is_pesudo: is_pesudo = 'pesudo_true' print('this is pesudo method') else: is_pesudo = 'pesudo_false' if args.is_pesudo_confidence: is_pesudo_confidence = 'pesudo_confidence_true' else: is_pesudo_confidence = 'pesudo_confidence_false' if args.is_pesudo_density: is_pesudo_density = 'pesudo_density_true' else: is_pesudo_density = 'pesudo_density_false' if args.is_pesudo_distribution: is_pesudo_distribution = 'pesudo_distribution_true' else: is_pesudo_distribution = 'pesudo_distribution_false' if args.is_interpolation: is_soft_weight = 'soft_weight' print('this is overall soft weight') else: is_soft_weight = 'hard_weight' print('this is overall hard weight') #training trich if args.is_augmentation: augmentation = 'augmentation_true' else: augmentation = 'augmentation_false' if args.is_dropout: dropout = 'dropout_true' else: dropout = 'dropout_false' if args.is_bn: bn = 'bn_true' else: bn = 'bn_false' #import parameter if args.is_freezing: freezing = 'freezing_true' else: freezing = 'freezing_false' main_path = '{}-{}/{}/'.format(EMOTION_DATASET, POSE_DATASET, MODEL) # appendex_path='Net-{}_{}_{}_{}-Pesudo-{}_{}_{}_{}_{}_{}_{}_distill_t-{}_threshold_d-{}_interpolation_w-{}_cluster_k-{}_density_t-{}-LR-{}_{}_{}_{}'.format(augmentation,dropout,bn,args.weights_decay, # is_naive,is_distilled,is_soft_weight,is_pesudo,is_pesudo_confidence,is_pesudo_density,is_pesudo_distribution, # args.distill_t,args.selection_threshold,args.interpolation_weights,args.cluster_k,args.density_t, # args.E_loss_weights,args.P_loss_weights,freezing,args.no_freezing_epoch) appendex_path = 'Pesudo-{}_{}_{}_{}_{}_{}_{}_distill_t-{}_threshold_d-{}_interpolation_w-{}_cluster_k-{}_density_t-{}-LR-{}_{}_{}_{}'.format( is_naive, is_distilled, is_soft_weight, is_pesudo, is_pesudo_confidence, is_pesudo_density, is_pesudo_distribution, args.distill_t, args.selection_threshold, args.interpolation_weights, args.cluster_k, args.density_t, args.E_loss_weights, args.P_loss_weights, freezing, args.no_freezing_epoch) weights_path = './train_weights/ITERATION/' + main_path + appendex_path logs_path = './train_log/ITERATION/' + main_path + appendex_path matrix_path = './train_log/ITERATION/' + main_path + appendex_path + '/matrix/' if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) if not os.path.exists(matrix_path): os.makedirs(matrix_path) model_names = weights_path + '.{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}.hdf5' csv_name = logs_path + '.log' checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=True) csvlogger = CSVLogger(filename=csv_name, append=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE), verbose=1) tensorboard = TensorBoard(log_dir=logs_path, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] print('whether freezing:', args.is_freezing) # model_path = './train_weights/ITERATION/ferplus-aflw/EmotionPoseNetVGGFace_vgg16/Pesudo-naive_false_distilled_false_soft_weight_pesudo_true_pesudo_confidence_true_pesudo_density_true_pesudo_distribution_true_distill_t-2_threshold_d-0.0_interpolation_w-0.0_cluster_k-4_density_t-0.6-LR-1.0_1.0_freezing_false_0.0.7856-0.7687.hdf5' # model.summary() # model.load_weights(model_path) if args.is_pesudo or args.is_interpolation or args.is_distilled: each_term_epoch = 1 else: each_term_epoch = 1 if not args.is_freezing: adam = keras.optimizers.Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) # adam = keras.optimizers.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False) model = no_freeze_all(model) model.compile(optimizer=adam, loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch == -1: print('initializing') model.fit_generator(Test_DataGenerator( model, None, train_emotion_paths, train_pose_paths, train_emotion, train_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, args.is_augmentation, None, None), validation_data=Test_DataGenerator( model, None, test_emotion_paths, test_pose_paths, test_emotion, test_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, False, None, None), epochs=4, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) print('initializing finished') else: # model_pre = keras.models.clone_model(model) # print('load:',model.get_weights()[3][:10]) model_pre = Model(inputs=model.input, outputs=model.output) # model_pre.set_weights(model.get_weights()) # print('pre:',model_pre.get_weights()[3][:10]) if epoch % 2 == 0: is_emotion = True if args.is_naive: model.compile(optimizer=adam, loss=loss_dict, loss_weights=[1, 0], metrics=metrics_dict) else: is_emotion = False if args.is_naive: model.compile(optimizer=adam, loss=loss_dict, loss_weights=[0, 1], metrics=metrics_dict) if args.is_pesudo or args.is_interpolation or args.is_distilled: is_hard_weight = not args.is_interpolation pesudo_data = assign_weights( model, model_pre, is_emotion, train_emotion_paths, train_emotion, train_pose_paths, train_pose, args.cluster_k, args.is_augmentation, args.selection_threshold, args.is_distilled, args.is_pesudo_confidence, args.is_pesudo_density, args.is_pesudo_distribution, is_hard_weight, args.interpolation_weights, args.distill_t, args.density_t) else: pesudo_data = None model.fit_generator( DataGenerator(model, model_pre, train_emotion_paths, train_pose_paths, train_emotion, train_pose, args.batch_size, is_emotion, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, args.is_augmentation, pesudo_data), validation_data=Test_DataGenerator( model, None, test_emotion_paths, test_pose_paths, test_emotion, test_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, False, None, None), epochs=each_term_epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) # print('after:',model.get_weights()[3][:10]) if epoch % 2 == 0: print('{}_emotion_train_finished'.format(epoch)) else: print('{}_pose_train_finished'.format(epoch)) emotion_predict = model.predict_generator( emotion_DataGenerator(model, test_emotion_paths, test_emotion, args.batch_size)) emotion_reuslt = np.argmax(emotion_predict[0], axis=1) emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt) pose_predict = model.predict_generator( pose_DataGenerator(model, test_pose_paths, test_pose, args.batch_size)) pose_reuslt = np.argmax(pose_predict[1], axis=1) pose_matrix = confusion_matrix(test_pose, pose_reuslt) print('emotion confusion matrix:', emotion_matrix) print('pose confusion matrix:', pose_matrix) np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch), emotion_matrix) np.savetxt(matrix_path + '{}_pose_matrix.txt'.format(epoch), pose_matrix) else: model.compile(optimizer='adam', loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) model.summary() for epoch in range(EPOCH): if epoch == 0: print('initializing') model.fit_generator(Test_DataGenerator( model, None, train_emotion_paths, train_pose_paths, train_emotion, train_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, args.is_augmentation, None, None), validation_data=Test_DataGenerator( model, None, test_emotion_paths, test_pose_paths, test_emotion, test_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, False, None, None), epochs=3, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) print('initializing finished') else: # model_pre = keras.models.clone_model(model) model_pre = Model(inputs=model.input, outputs=model.output) model_pre.set_weights(model.get_weights()) if epoch % 2 == 0: is_emotion = True if args.is_naive: if epoch < args.no_freezing_epoch: model.compile(optimizer='adam', loss=loss_dict, loss_weights=[1, 0], metrics=metrics_dict) else: adam = keras.optimizers.Adam(lr=1 * 1e-4) model.compile(optimizer=adam, loss=loss_dict, loss_weights=[1, 0], metrics=metrics_dict) else: is_emotion = False if args.is_naive: if epoch < args.no_freezing_epoch: model.compile(optimizer='adam', loss=loss_dict, loss_weights=[0, 1], metrics=metrics_dict) else: adam = keras.optimizers.Adam(lr=1 * 1e-4) model.compile(optimizer=adam, loss=loss_dict, loss_weights=[0, 1], metrics=metrics_dict) if args.is_pesudo or args.is_interpolation or args.is_distilled: is_hard_weight = not args.is_interpolation pesudo_data = assign_weights( model, model_pre, is_emotion, train_emotion_paths, train_emotion, train_pose_paths, train_pose, args.cluster_k, args.is_augmentation, args.selection_threshold, args.is_distilled, args.is_pesudo_confidence, args.is_pesudo_density, args.is_pesudo_distribution, is_hard_weight, args.interpolation_weights, args.distill_t, args.density_t) else: pesudo_data = None model.fit_generator( DataGenerator(model, model_pre, train_emotion_paths, train_pose_paths, train_emotion, train_pose, args.batch_size, is_emotion, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, args.is_augmentation, pesudo_data), validation_data=Test_DataGenerator( model, None, test_emotion_paths, test_pose_paths, test_emotion, test_pose, args.batch_size, args.is_distilled, args.is_pesudo, args.is_interpolation, args.selection_threshold, args.interpolation_weights, False, None, None), epochs=each_term_epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) if epoch % 2 == 0: print('{}_emotion_train_finished'.format(epoch)) else: print('{}_pose_train_finished'.format(epoch)) if epoch == args.no_freezing_epoch: print('no freezing and start to changing learning rate') model = no_freeze_all(model) adam = keras.optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) print('lr changing to:', 1 * 1e-5) model.compile(optimizer=adam, loss=loss_dict, loss_weights=loss_weights, metrics=metrics_dict) print('epoch is{}'.format(epoch)) model.summary() emotion_predict = model.predict_generator( emotion_DataGenerator(model, test_emotion_paths, test_emotion, args.batch_size)) emotion_reuslt = np.argmax(emotion_predict[0], axis=1) emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt) pose_predict = model.predict_generator( pose_DataGenerator(model, test_pose_paths, test_pose, args.batch_size)) pose_reuslt = np.argmax(pose_predict[1], axis=1) pose_matrix = confusion_matrix(test_pose, pose_reuslt) print('emotion confusion matrix:', emotion_matrix) print('pose confusion matrix:', pose_matrix) np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch), emotion_matrix) np.savetxt(matrix_path + '{}_pose_matrix.txt'.format(epoch), pose_matrix)
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker if args.task_type == 0: task_name = 'emotion' elif args.task_type ==1: task_name = 'age' elif args.task_type == 5: task_name = 'pose' elif args.task_type == 10: task_name = 'gender' paths, task_label = load_data(DATASET,task_name) print('[K-FOLD] Started...') kf = KFold(n_splits=5, shuffle=True, random_state=1) kf_split_task = kf.split(task_label) task_kf = [[task_train_idx,task_test_idx] for task_train_idx,task_test_idx in kf_split_task] task_train_idx,task_test_idx = task_kf[0] train_paths = paths[task_train_idx] train_task = task_label[task_train_idx] test_paths = paths[task_test_idx] test_task = task_label[task_test_idx] model = None if MODEL == 'vggFace': if args.dataset=='ferplus': model = Net(MODEL,1,args.task_type,8,5,8,2,args.is_dropout,args.is_bn,args.weights_decay) else: model = Net(MODEL,1,args.task_type,7,5,8,2,args.is_dropout,args.is_bn,args.weights_decay) if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: if args.dataset=='ferplus': model = Net(MODEL,1,args.task_type,8,5,8,2,args.is_dropout,args.is_bn,args.weights_decay) else: model = Net(MODEL,1,args.task_type,7,5,8,2,args.is_dropout,args.is_bn,args.weights_decay) if args.is_freezing: model = freeze_all_but_mid_and_top(model) if args.task_type == 0: task_classes = model.emotion_classes losses = {"emotion_prediction": 'categorical_crossentropy' } metrics = { "emotion_prediction": 'acc' } elif args.task_type ==1: task_classes = model.age_classes losses = {"age_prediction": 'categorical_crossentropy' } metrics = { "age_prediction": 'acc' } elif args.task_type == 5: task_classes = model.pose_classes losses = {"pose_prediction": 'categorical_crossentropy' } metrics = { "pose_prediction": 'acc' } elif args.task_type == 10: task_classes = model.gender_classes losses = {"gender_prediction": 'categorical_crossentropy' } metrics = { "gender_prediction": 'acc' } #training trick if args.is_augmentation: augmentation='augmentation_true' else: augmentation='augmentation_false' if args.is_dropout: dropout='dropout_true' else: dropout='dropout_false' if args.is_bn: bn = 'bn_true' else: bn = 'bn_false' # key parameter if args.is_freezing: freezing='freezing_true' else: freezing='freezing_false' weights_path = './train_weights/{}/{}/net-{}_{}_{}_{}-lr-{}_{}'.format(model.name,DATASET,augmentation,dropout,bn,args.weights_decay,freezing,args.no_freezing_epoch) logs_path = './train_log/{}/{}/net-{}_{}_{}_{}-lr-{}_{}'.format(model.name,DATASET,augmentation,dropout,bn,args.weights_decay,freezing,args.no_freezing_epoch) matrix_path = logs_path+'/matrix/' if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) if not os.path.exists(matrix_path): os.makedirs(matrix_path) model_names = weights_path + '{epoch:02d}-{val_acc:.4f}.hdf5' csv_name = logs_path + '.log' board_name = logs_path checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True) csvlogger=CSVLogger(filename=csv_name,separator=',',append=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE), verbose=1) tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE) callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard] if args.is_freezing: model.compile(optimizer='adam', loss=losses, metrics=metrics) model.summary() model.fit_generator( DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation), validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False), epochs=args.no_freezing_epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks ) if args.epoch>args.no_freezing_epoch: adam = keras.optimizers.Adam(lr=1*1e-5) print('lr changing to:',1*1e-5) model = no_freeze_all(model) model.compile(optimizer=adam, loss=losses, metrics=metrics) model.summary() model.fit_generator( DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation), validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False), epochs=args.epoch-args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) predicted_result = model.predict_generator(DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False)) task_reuslt=np.argmax(np.array(predicted_result),axis=1) matrix=confusion_matrix(test_task,task_reuslt) print('confusion matrix:',matrix) np.savetxt(matrix_path+'.txt',matrix) else: adam = keras.optimizers.Adam(lr=0.00001) model.compile(optimizer=adam, loss=losses, metrics=metrics) model.summary() model.fit_generator( DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation), validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False), epochs=args.epoch, verbose=2, workers=1, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) predicted_result = model.predict_generator(DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False)) task_reuslt=np.argmax(np.array(predicted_result),axis=1) matrix=confusion_matrix(test_task,task_reuslt) print('confusion matrix:',matrix) np.savetxt(matrix_path+'.txt',matrix) del train_paths, train_task del test_paths, test_task
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker paths, pose_label = load_data(DATASET) n_fold = 1 losses = {"pose_prediction": custom_mse_pose} metrics = {"pose_prediction": pose_metric} print('[K-FOLD] Started...') kf = KFold(n_splits=10, shuffle=True, random_state=1) kf_split = kf.split(pose_label) for train_idx, test_idx in kf_split: model = None if MODEL == 'vggFace': model = Net(MODEL,1,5,8,2,8) model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL,1,5,8,2,8) train_paths = paths[train_idx] train_label = pose_label[train_idx] test_paths = paths[test_idx] test_label = pose_label[test_idx] print(len(train_paths),len(test_paths)) model.summary() weights_path = './train_weights/pose/{}/{}/'.format(DATASET,model.name) logs_path = './train_log/pose/{}/{}/'.format(DATASET,model.name) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) model_names = weights_path + '.{epoch:02d}-{val_pose_metric: 0.4f}.hdf5' csv_name = logs_path + '{}.log'.format(n_fold) board_name = logs_path + '{}'.format(n_fold) checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True) csvlogger=CSVLogger(csv_name) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE/2), verbose=1) tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE) callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard] if MODEL == 'ssrnet': callbacks = [ # ModelCheckpoint( # "train_weight/{}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f}.h5".format( # MODEL), # verbose=1, save_best_only=True, save_weights_only=True), ModelCheckpoint(MODEL, 'val_loss', verbose=1,save_best_only=True), CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)), DecayLearningRate([30, 60])] model.compile(optimizer='adam', loss=losses, metrics = metrics) model.fit_generator( DataGenerator(model, train_paths, train_label, BATCH_SIZE), validation_data=DataGenerator(model, test_paths, test_label, BATCH_SIZE), epochs=EPOCH, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks ) n_fold += 1 del train_paths, train_roll,train_pictch,train_yaw del test_paths, test_roll,test_pitch,test_yaw
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker if DATASET == 'fgnet': age_classes = 70 losses = {"age_prediction": mae} metrics = {"age_prediction": mae} elif DATASET == 'megaage_asian': age_classes = 71 losses = {"age_prediction": mae} metrics = {"age_prediction": mae} else: gender_classes = 2 age_classes = 8 losses = {"age_prediction": 'categorical_crossentropy'} metrics = {"age_prediction": 'acc'} paths, age_label = load_data(DATASET) n_fold = 1 print('[K-FOLD] Started...') kf = KFold(n_splits=10, shuffle=True, random_state=1) kf_split = kf.split(age_label) for train_idx, test_idx in kf_split: model = None if MODEL == 'vggFace': model = Net(MODEL, 1, 1, 8, 2, age_classes) if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: model = Net(MODEL, 1, 1, 8, 2, age_classes) train_paths = paths[train_idx] train_age = age_label[train_idx] test_paths = paths[test_idx] test_age = age_label[test_idx] if MODEL == 'ssrnet': losses = { "age_prediction": "mae", "gender_prediction": "mae", } metrics = { "age_prediction": "mae", "gender_prediction": "binary_accuracy", } weights_path = './train_weights/age/{}/{}/'.format(DATASET, model.name) logs_path = './train_log/age/{}/{}/'.format(DATASET, model.name) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) model_names = weights_path + '{}__'.format( n_fold) + '{epoch:02d}-{val_acc:.2f}.hdf5' csv_name = logs_path + '{}.log'.format(n_fold) board_name = logs_path + '{}'.format(n_fold) checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE / 2), verbose=1) tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] model.compile(optimizer='adam', loss=losses, metrics=metrics) # emotion_model = Net(MODEL,1,1,8,2,age_classes) # emotion_model = updateTargetmodel(model,emotion_model) # emotion_model = keras.models.clone_model(model) # emotion_model.set_weights(model.get_weights()) # emotion_model = freeze_all(emotion_model) # emotion_model.compile(optimizer='adam', loss=losses, metrics=metrics) print('freezing model') model.summary() model.fit_generator(DataGenerator(model, train_paths, train_age, age_classes, BATCH_SIZE), validation_data=DataGenerator( model, test_paths, test_age, age_classes, BATCH_SIZE), epochs=args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) model = no_freeze_all(model) print('no freezing') model.summary() model.fit_generator(DataGenerator(model, train_paths, train_age, age_classes, BATCH_SIZE), validation_data=DataGenerator( model, test_paths, test_age, age_classes, BATCH_SIZE), epochs=EPOCH - args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) n_fold += 1 print( np.array_equal(model.get_weights()[0][0][0][0], emotion_model.get_weights()[0][0][0][0])) if np.array_equal(model.get_weights()[0], emotion_model.get_weights()[0]): print('please check code!') del train_paths, train_age del test_paths, test_age
def main(): config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.tensorflow_backend.set_session(sess) args = parser.parse_args() DATASET = args.dataset MODEL = args.model EPOCH = args.epoch PATIENCE = args.patience BATCH_SIZE = args.batch_size NUM_WORKER = args.num_worker if args.task_type == 0: task_name = 'emotion' elif args.task_type == 1: task_name = 'age' elif args.task_type == 5: task_name = 'pose' elif args.task_type == 10: task_name = 'gender' paths, task_label = load_data(DATASET, task_name) print('[K-FOLD] Started...') kf = KFold(n_splits=5, shuffle=True, random_state=1) kf_split_task = kf.split(task_label) task_kf = [[task_train_idx, task_test_idx] for task_train_idx, task_test_idx in kf_split_task] task_train_idx, task_test_idx = task_kf[0] train_paths = paths[task_train_idx] train_task = task_label[task_train_idx] test_paths = paths[task_test_idx] test_task = task_label[task_test_idx] model = None if MODEL == 'vggFace': if args.dataset == 'ferplus': model = Net(MODEL, 1, args.task_type, args.is_dropout, 8, 5, 8, 2) else: model = Net(MODEL, 1, args.task_type, args.is_dropout, 7, 5, 8, 2) if args.is_freezing: model = freeze_all_but_mid_and_top(model) else: if args.dataset == 'ferplus': model = Net(MODEL, 1, args.task_type, args.is_dropout, 8, 5, 8, 2) else: model = Net(MODEL, 1, args.task_type, args.is_dropout, 7, 5, 8, 2) if args.is_freezing: model = freeze_all_but_mid_and_top(model) if args.task_type == 0: task_classes = model.emotion_classes losses = {"emotion_prediction": 'categorical_crossentropy'} metrics = {"emotion_prediction": 'acc'} elif args.task_type == 1: task_classes = model.age_classes losses = {"age_prediction": 'categorical_crossentropy'} metrics = {"age_prediction": 'acc'} elif args.task_type == 5: task_classes = model.pose_classes losses = {"pose_prediction": 'categorical_crossentropy'} metrics = {"pose_prediction": 'acc'} elif args.task_type == 10: task_classes = model.gender_classes losses = {"gender_prediction": 'categorical_crossentropy'} metrics = {"gender_prediction": 'acc'} if args.is_freezing: freezing = 'freezing_true' else: freezing = 'freezing_false' if args.is_dropout: dropout = 'dropout_true' else: dropout = 'drouout_false' weights_path = './train_weights/{}/{}/{}-{}_'.format( model.name, DATASET, freezing, dropout) logs_path = './train_log/{}/{}/{}-{}_'.format(model.name, DATASET, freezing, dropout) if not os.path.exists(weights_path): os.makedirs(weights_path) if not os.path.exists(logs_path): os.makedirs(logs_path) model_names = weights_path + '{epoch:02d}-{val_acc:.2f}.hdf5' csv_name = logs_path + '.log' board_name = logs_path checkpoint = ModelCheckpoint(model_names, verbose=1, save_weights_only=True, save_best_only=True) csvlogger = CSVLogger(filename=csv_name, separator=',', append=True) early_stop = EarlyStopping('val_loss', patience=PATIENCE) reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE / 2), verbose=1) tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE) callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard] adam = keras.optimizer.adam(lr=0.0001) model.compile(optimizer=adam, loss=losses, metrics=metrics) print('starting model') model.summary() model.fit_generator(DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE), validation_data=DataGenerator(model, test_paths, test_task, task_classes, BATCH_SIZE), epochs=args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks) if args.is_freezing: reduce_lr = LearningRateScheduler(scheduler) else: reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1, patience=int(PATIENCE / 2), verbose=1) if args.is_freezing: before_freezing_weights = model.get_weights()[1] model = no_freeze_all(model) model.compile(optimizer=adam, loss=losses, metrics=metrics) callbacks_new = [ checkpoint, csvlogger, early_stop, reduce_lr, tensorboard ] no_freezing_weights = model.get_weights()[1] print('combile whether change', np.equal(before_freezing_weights, no_freezing_weights)) model.summary() model.fit_generator(DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE), validation_data=DataGenerator(model, test_paths, test_task, task_classes, BATCH_SIZE), epochs=EPOCH - args.no_freezing_epoch, verbose=2, workers=NUM_WORKER, use_multiprocessing=False, max_queue_size=int(BATCH_SIZE * 2), callbacks=callbacks_new) train_no_freezing_weights = model.get_weights()[1] print('second stage training change', np.equal(no_freezing_weights, train_no_freezing_weights)) del train_paths, train_task del test_paths, test_task