def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
DATASET = args.dataset
if DATASET == 'ferplus':
emotion_classes = 8
else:
emotion_classes = 7
paths, emotion_label = load_data(DATASET)
n_fold = 1
print('[K-FOLD] Started...')
kf = KFold(n_splits=5, shuffle=True, random_state=1)
kf_split = kf.split(emotion_label)
for train_idx, test_idx in kf_split:
model = None
if MODEL == 'vggFace':
model = Net(MODEL, 1, 0, emotion_classes, 2, 2)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL, 1, 0, emotion_classes, 2, 2)
train_paths = paths[train_idx]
train_emotion = emotion_label[train_idx]
test_paths = paths[test_idx]
test_emotion = emotion_label[test_idx]
print(len(train_paths), len(test_paths))
losses = {
"emotion_prediction": "categorical_crossentropy",
}
metrics = {
"emotion_prediction": "acc",
}
if MODEL == 'ssrnet':
losses = {
"emotion_prediction": "mae",
}
metrics = {
"emotion_prediction": "mae",
}
# -{epoch:02d}-{val_loss:.4f}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f}
model.summary()
weights_path = './train_weights/emotion/{}/{}/'.format(
DATASET, model.name)
logs_path = './train_log/emotion/{}/{}/'.format(DATASET, model.name)
# plot_model(model,to_file= logs_path+'.jpg',show_shapes=True)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
model_names = weights_path + '.{epoch:02d}-{val_acc:.2f}.hdf5'
csv_name = logs_path + '{}.log'.format(n_fold)
board_name = logs_path + '{}'.format(n_fold)
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=True)
csvlogger = CSVLogger(csv_name)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE / 2),
verbose=1)
tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
if MODEL == 'ssrnet':
callbacks = [
ModelCheckpoint(MODEL,
'val_loss',
verbose=1,
save_best_only=True),
CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)),
DecayLearningRate([30, 60])
]
model.compile(optimizer='adam', loss=losses, metrics=metrics)
model.fit_generator(DataGenerator(model, train_paths, train_emotion,
BATCH_SIZE),
validation_data=DataGenerator(
model, test_paths, test_emotion, BATCH_SIZE),
epochs=args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
n_fold += 1
del train_paths, train_emotion
del test_paths, test_emotion
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
EMOTION_DATASET = args.dataset_emotion
GENDER_AGE_DATASET = args.dataset_gender_age
if EMOTION_DATASET == 'ferplus':
emotion_classes = 8
else:
emotion_classes = 7
gender_classes = 2
age_classes = 8
pose_classes = 5
# emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5'
# pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5'
# age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5'
paths_emotion, paths_pose, paths_gender_age, emotion_label, pose_label, gender_label, age_label = load_data(
EMOTION_DATASET, GENDER_AGE_DATASET, args.dataset_pose)
kf = KFold(n_splits=5, shuffle=True, random_state=1)
kf_split_gender_age = kf.split(paths_gender_age)
kf_split_emotion = kf.split(paths_emotion)
kf_split_pose = kf.split(paths_pose)
emotion_kf = [[emotion_train_idx, emotion_test_idx]
for emotion_train_idx, emotion_test_idx in kf_split_emotion]
gender_age_kf = [[
gender_age_train_idx, gender_age_test_idx
] for gender_age_train_idx, gender_age_test_idx in kf_split_gender_age]
pose_kf = [[pose_train_idx, pose_test_idx]
for pose_train_idx, pose_test_idx in kf_split_pose]
emotion_train_idx, emotion_test_idx = emotion_kf[0]
pose_train_idx, pose_test_idx = pose_kf[0]
gender_age_train_idx, gender_age_test_idx = gender_age_kf[0]
print(len(emotion_train_idx), len(gender_age_train_idx),
len(pose_train_idx))
print(len(emotion_test_idx), len(gender_age_test_idx), len(pose_test_idx))
train_emotion_paths = paths_emotion[emotion_train_idx]
train_emotion = emotion_label[emotion_train_idx]
test_emotion_paths = paths_emotion[emotion_test_idx]
test_emotion = emotion_label[emotion_test_idx]
train_pose_paths = paths_pose[pose_train_idx]
train_pose = pose_label[pose_train_idx]
test_pose_paths = paths_pose[pose_test_idx]
test_pose = pose_label[pose_test_idx]
train_gender_age_paths = paths_gender_age[gender_age_train_idx]
train_gender = gender_label[gender_age_train_idx]
train_age = age_label[gender_age_train_idx]
test_gender_age_paths = paths_gender_age[gender_age_test_idx]
test_gender = gender_label[gender_age_test_idx]
test_age = age_label[gender_age_test_idx]
model = None
if MODEL == 'vggFace':
print('based on vggface')
model = Net(MODEL, 1, 12, emotion_classes, pose_classes, age_classes,
gender_classes, args.is_dropout, args.is_bn,
args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL, 1, 12, emotion_classes, pose_classes, age_classes,
gender_classes, args.is_dropout, args.is_bn,
args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
loss_dict = {}
metrics_dict = {}
prediction = []
loss = []
acc = []
loss_weights = []
prediction.append('emotion_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.E_loss_weights)
prediction.append('pose_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.P_loss_weights)
prediction.append('gender_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.G_loss_weights)
prediction.append('age_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.A_loss_weights)
loss_dict = dict(zip(prediction, loss))
metrics_dict = dict(zip(prediction, acc))
#pesudo label
if args.is_naive:
is_naive = 'naive_true'
print('this is naive method')
else:
is_naive = 'naive_false'
if args.is_distilled:
is_distilled = 'distilled_true'
print('this is distiiled method')
else:
is_distilled = 'distilled_false'
if args.is_pesudo:
is_pesudo = 'pesudo_true'
print('this is pesudo method')
else:
is_pesudo = 'pesudo_false'
if args.is_interpolation:
is_interpolation = 'interpolation_true'
print('this is interpolation method')
else:
is_interpolation = 'interpolation_false'
if args.selection_threshold > 0:
print('this is pesudo selection method')
#training trich
if args.is_augmentation:
augmentation = 'augmentation_true'
else:
augmentation = 'augmentation_false'
if args.is_dropout:
dropout = 'dropout_true'
else:
dropout = 'dropout_false'
if args.is_bn:
bn = 'bn_true'
else:
bn = 'bn_false'
#import parameter
if args.is_freezing:
freezing = 'freezing_true'
else:
freezing = 'freezing_false'
main_path = '{}-{}-{}/{}/'.format(EMOTION_DATASET, GENDER_AGE_DATASET,
args.dataset_pose, MODEL)
appendex_path = 'net-{}_{}_{}_{}-pesudo-{}_{}_{}_{}_{}_{}-lr-{}_{}_{}_{}_{}_'.format(
augmentation, dropout, bn, args.weights_decay, is_naive, is_distilled,
is_pesudo, is_interpolation, args.selection_threshold,
args.interpolation_weights, args.E_loss_weights, args.G_loss_weights,
args.A_loss_weights, freezing, args.no_freezing_epoch)
weights_path = './train_weights/CONFUSION/' + main_path + appendex_path
logs_path = './train_log/CONFUSION/' + main_path + appendex_path
matrix_path = './train_log/CONFUSION/' + main_path + appendex_path + '/matrix/'
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
if not os.path.exists(matrix_path):
os.makedirs(matrix_path)
model_names = weights_path + '.{epoch:02d}-{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}-{val_gender_prediction_my_acc:.4f}-{val_age_prediction_my_acc:.4f}.hdf5'
csv_name = logs_path + '.log'
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=False)
csvlogger = CSVLogger(filename=csv_name, append=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE),
verbose=1)
tensorboard = TensorBoard(log_dir=logs_path, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
print('whether freezing:', args.is_freezing)
if not args.is_freezing:
adam = keras.optimizers.Adam(lr=0.0001)
model = no_freeze_all(model)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch == 0 or args.is_naive:
model_pre = None
else:
model_pre = keras.models.clone_model(model)
model_pre.set_weights(model.get_weights())
model.fit_generator(
DataGenerator(model,
model_pre,
train_emotion_paths,
train_pose_paths,
train_gender_age_paths,
train_emotion,
train_pose,
train_gender,
train_age,
args.batch_size,
args.is_distilled,
args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
is_augmentation=args.is_augmentation),
validation_data=DataGenerator(model,
None,
test_emotion_paths,
test_pose_paths,
test_gender_age_paths,
test_emotion,
test_pose,
test_gender,
test_age,
args.batch_size,
args.is_distilled,
args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
is_augmentation=False),
epochs=1,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
print('{}_train_finished'.format(epoch))
emotion_predict = model.predict_generator(
emotion_DataGenerator(model, test_emotion_paths, test_emotion,
args.batch_size))
emotion_reuslt = np.argmax(emotion_predict[0], axis=1)
emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt)
age_predict = model.predict_generator(
age_DataGenerator(model, test_gender_age_paths, test_age,
args.batch_size))
age_reuslt = np.argmax(age_predict[2], axis=1)
age_matrix = confusion_matrix(test_age, age_reuslt)
print('emotion confusion matrix:', emotion_matrix)
print('age confusion matrix:', age_matrix)
np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch),
emotion_matrix)
np.savetxt(matrix_path + '{}_age_matrix.txt'.format(epoch),
age_matrix)
else:
model.compile(optimizer='adam',
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch == 0 or args.is_naive:
model_pre = None
else:
model_pre = keras.models.clone_model(model)
model_pre.set_weights(model.get_weights())
model.fit_generator(
DataGenerator(model,
model_pre,
train_emotion_paths,
train_pose_paths,
train_gender_age_paths,
train_emotion,
train_pose,
train_gender,
train_age,
args.batch_size,
args.is_distilled,
args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
is_augmentation=args.is_augmentation),
validation_data=DataGenerator(model,
None,
test_emotion_paths,
test_pose_paths,
test_gender_age_paths,
test_emotion,
test_pose,
test_gender,
test_age,
args.batch_size,
args.is_distilled,
args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
is_augmentation=False),
epochs=1,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
if epoch == args.no_freezing_epoch:
print('no freezing and start to changing learning rate')
model = no_freeze_all(model)
adam = keras.optimizers.Adam(lr=1 * 1e-4)
print('lr changing to:', 1 * 1e-4)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
print('epoch is{}'.format(epoch))
model.summary()
print('{}_train_finished'.format(epoch))
emotion_predict = model.predict_generator(
emotion_DataGenerator(model, test_emotion_paths, test_emotion,
args.batch_size))
emotion_reuslt = np.argmax(emotion_predict[0], axis=1)
emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt)
age_predict = model.predict_generator(
age_DataGenerator(model, test_gender_age_paths, test_age,
args.batch_size))
age_reuslt = np.argmax(age_predict[2], axis=1)
age_matrix = confusion_matrix(test_age, age_reuslt)
print('emotion confusion matrix:', emotion_matrix)
print('age confusion matrix:', age_matrix)
np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch),
emotion_matrix)
np.savetxt(matrix_path + '{}_age_matrix.txt'.format(epoch),
age_matrix)
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
loss_dict = {}
metrics_dict = {}
attr_predcition = []
loss = []
acc = []
for i in range(40):
attr_predcition.append('attr{}_predition'.format(i))
loss.append('binary_crossentropy')
acc.append('acc')
# print(attr_predcition,loss,acc)
loss_dict = dict(zip(attr_predcition, loss))
metrics_dict = dict(zip(attr_predcition, acc))
losses = {"age_prediction": mae}
metrics = {"age_prediction": mae}
model = None
if MODEL == 'vggFace':
model = Net(MODEL, 1, 6, 8, 2, 8)
model = freeze_all_but_mid_and_top(model)
MODEL = model.name
else:
model = Net(MODEL, 1, 6, 8, 2, 8)
MODEL = model.name
paths, attr_label = load_data(DATASET)
# model.summary()
n_fold = 1
print('[K-FOLD] Started...')
kf = KFold(n_splits=10, shuffle=True, random_state=1)
kf_split = kf.split(attr_label)
for train_idx, test_idx in kf_split:
print('length:train test:', len(train_idx), len(test_idx))
train_paths = paths[train_idx]
train_attr = attr_label[train_idx]
test_paths = paths[test_idx]
test_attr = attr_label[test_idx]
if MODEL == 'ssrnet':
losses = {
"age_prediction": "mae",
"gender_prediction": "mae",
}
metrics = {
"age_prediction": "mae",
"gender_prediction": "binary_accuracy",
}
weights_path = './train_weights/attr/{}/{}/'.format(DATASET, MODEL)
logs_path = './train_log/attr/{}/{}/'.format(DATASET, MODEL)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
# model_names = weights_path + '{epoch:02d}-{val_acc:.2f}.hdf5'
model_names = weights_path + '{epoch:02d}.hdf5'
csv_name = logs_path + '{}.log'.format(n_fold)
board_name = logs_path + '{}'.format(n_fold)
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=True)
csvlogger = CSVLogger(csv_name)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE / 2),
verbose=1)
tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
model.compile(optimizer='adam', loss=loss_dict, metrics=metrics_dict)
model.fit_generator(DataGenerator(model, train_paths, train_attr,
BATCH_SIZE),
validation_data=DataGenerator(
model, test_paths, test_attr, BATCH_SIZE),
epochs=EPOCH,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
n_fold += 1
del train_paths, train_attr
del test_paths, test_attr
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
EMOTION_DATASET = args.dataset_emotion
POSE_DATASET = args.dataset_pose
if EMOTION_DATASET == 'ferplus':
emotion_classes = 8
else:
emotion_classes = 7
gender_classes = 2
age_classes = 8
pose_classes = 5
# emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5'
# pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5'
# age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5'
paths_emotion, paths_pose, emotion_label,pose_label = load_data(EMOTION_DATASET,POSE_DATASET)
kf = KFold(n_splits=5, shuffle=True, random_state=1)
kf_split_emotion = kf.split(paths_emotion)
kf_split_pose = kf.split(paths_pose)
emotion_kf = [[emotion_train_idx,emotion_test_idx] for emotion_train_idx,emotion_test_idx in kf_split_emotion]
pose_kf = [[pose_train_idx,pose_test_idx] for pose_train_idx,pose_test_idx in kf_split_pose]
# for emotion_train_idx,emotion_test_idx in kf_split_emotion:
# for gender_age_train_idx,gender_age_test_idx in kf_split_gender_age:
# print(emotion_train_idx,emotion_test_idx,gender_age_train_idx,gender_age_test_idx)
emotion_train_idx,emotion_test_idx = emotion_kf[0]
pose_train_idx,pose_test_idx = pose_kf[0]
print(len(emotion_train_idx),len(pose_train_idx))
print(len(emotion_test_idx),len(pose_test_idx))
train_emotion_paths = paths_emotion[emotion_train_idx]
train_emotion = emotion_label[emotion_train_idx]
test_emotion_paths = paths_emotion[emotion_test_idx]
test_emotion = emotion_label[emotion_test_idx]
train_pose_paths = paths_pose[pose_train_idx]
train_pose = pose_label[pose_train_idx]
test_pose_paths = paths_pose[pose_test_idx]
test_pose = pose_label[pose_test_idx]
model = None
if MODEL == 'vggFace':
print('based on vggface')
model = Net(MODEL,1,11,emotion_classes,pose_classes,age_classes,gender_classes,args.is_dropout,args.is_bn,args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL,1,11,emotion_classes,pose_classes,age_classes,gender_classes,args.is_dropout,args.is_bn,args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
def my_cross_loss(y_true, y_pred):
mask = K.all(K.equal(y_true, 0), axis=-1)
mask = 1 - K.cast(mask, K.floatx())
loss = K.categorical_crossentropy(y_true, y_pred) * mask
return K.sum(loss) / K.sum(mask)
def my_bin_loss(y_true, y_pred):
mask = K.all(K.equal(y_true, 0), axis=-1)
mask = 1 - K.cast(mask, K.floatx())
loss = K.binary_crossentropy(y_true, y_pred) * mask
return K.sum(loss) / K.sum(mask)
def my_mean_square(y_true,y_pred):
mask = K.all(K.equal(y_true, 0), axis=-1)
mask = 1 - K.cast(mask, K.floatx())
loss = keras.losses.mean_squared_error(y_true,y_pred)* mask
return K.sum(loss) / K.sum(mask)
def my_acc(y_true, y_pred):
mask = K.all(K.equal(y_true, 0), axis=-1)
mask = 1 - K.cast(mask, K.floatx())
acc = (K.cast(K.equal(K.argmax(y_true, axis=-1),K.argmax(y_pred, axis=-1)),K.floatx()))*mask
return K.sum(acc)/K.sum(mask)
# def pose_metric(y_true,y_pred):
# # y_true = K.dot(180/np.pi*K.ones(K.shape(y_true)),y_true)
# # y_pred = K.dot(180/np.pi*K.ones(K.shape(y_pred)),y_pred)
# y_true = 180/3.14*y_true
# y_pred = 180/3.14*y_pred
# comp = 15*K.ones(K.shape(y_pred))
# comp = K.cast(comp, K.floatx())
# sub = K.cast(K.abs(y_true-y_pred), K.floatx())
# diff = K.greater(comp,sub)
# diff = K.cast(diff, K.floatx())
# result = K.sum(diff,axis=0)/K.cast(K.shape(y_pred)[0], K.floatx())
# return result[0]
# def my_pose_metric(y_true,y_pred):
# mask = K.all(K.equal(y_true, 0), axis=-1)
# mask = 1 - K.cast(mask, K.floatx())
# acc = pose_metric(y_true,y_pred)*mask
# return K.sum(acc) / K.sum(mask)
loss_dict = {}
metrics_dict = {}
prediction = []
loss =[]
acc =[]
loss_weights = []
prediction.append('emotion_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.E_loss_weights)
prediction.append('pose_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.P_loss_weights)
loss_dict = dict(zip(prediction, loss))
metrics_dict = dict(zip(prediction, acc))
#pesudo label
if args.is_naive:
is_naive='naive_true'
print('this is naive method')
else:
is_naive='naive_false'
if args.is_distilled:
is_distilled='distilled_true'
print('this is distiiled method')
else:
is_distilled='distilled_false'
if args.is_pesudo:
is_pesudo='pesudo_true'
print('this is pesudo method')
else:
is_pesudo='pesudo_false'
if args.is_pesudo_density:
is_pesudo_density='pesudo_density_true'
else:
is_pesudo_density='pesudo_density_false'
if args.is_pesudo_distribution:
is_pesudo_distribution='pesudo_distribution_true'
else:
is_pesudo_distribution='pesudo_distribution_false'
if args.is_interpolation:
is_soft_weight = 'soft_weight'
print('this is overall soft weight')
else:
is_soft_weight = 'hard_weight'
print('this is overall hard weight')
if args.selection_threshold>0:
print('this is pesudo selection method')
#training trich
if args.is_augmentation:
augmentation='augmentation_true'
else:
augmentation='augmentation_false'
if args.is_dropout:
dropout='dropout_true'
else:
dropout='dropout_false'
if args.is_bn:
bn = 'bn_true'
else:
bn = 'bn_false'
#import parameter
if args.is_freezing:
freezing='freezing_true'
else:
freezing='freezing_false'
main_path='{}-{}/{}/'.format(EMOTION_DATASET,POSE_DATASET,MODEL)
appendex_path='net-{}_{}_{}_{}-pesudo-{}_{}_{}_{}_{}_{}_{}_threshold-{}_cluster_k-{}-lr-{}_{}_{}_{}'.format(augmentation,dropout,bn,args.weights_decay,
is_naive,is_distilled,is_soft_weight,is_pesudo,is_pesudo_density,is_pesudo_distribution,args.selection_threshold,args.interpolation_weights,args.cluster_k,
args.E_loss_weights,args.P_loss_weights,freezing,args.no_freezing_epoch)
weights_path = './train_weights/CONFUSION/'+main_path+appendex_path
logs_path = './train_log/CONFUSION/'+main_path+appendex_path
matrix_path = './train_log/CONFUSION/'+main_path+appendex_path+'/matrix/'
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
if not os.path.exists(matrix_path):
os.makedirs(matrix_path)
model_names = weights_path +'.{epoch:02d}-{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}.hdf5'
csv_name = logs_path + '.log'
checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True)
csvlogger=CSVLogger(filename=csv_name,append=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
patience=int(PATIENCE), verbose=1)
tensorboard = TensorBoard(log_dir=logs_path,batch_size=BATCH_SIZE)
callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard]
print('whether freezing:',args.is_freezing)
if args.is_pesudo or args.is_interpolation:
each_term_epoch = 4
else:
each_term_epoch = 1
if not args.is_freezing:
adam=keras.optimizers.Adam(lr=0.0001)
model = no_freeze_all(model)
model.compile(optimizer=adam, loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch==0 or args.is_naive:
model_pre = None
else:
model_pre = Model(inputs=model.input,outputs=model.output)
model_pre.set_weights(model.get_weights())
if args.is_pesudo or args.is_interpolation:
is_hard_weight = not args.is_interpolation
pesudo_emotion = assign_weights(model,False,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights)
pesudo_pose = assign_weights(model,True,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights)
else:
pesudo_emotion=None
pesudo_pose=None
model.fit_generator(
DataGenerator(model,model_pre,train_emotion_paths,train_pose_paths,train_emotion,train_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,args.is_augmentation,pesudo_emotion,pesudo_pose),
validation_data = DataGenerator(model, None,test_emotion_paths,test_pose_paths, test_emotion,test_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,False,pesudo_emotion,pesudo_pose),
epochs=each_term_epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
print('{}_train_finished'.format(epoch))
emotion_predict = model.predict_generator(emotion_DataGenerator(model,test_emotion_paths,test_emotion,args.batch_size))
emotion_reuslt=np.argmax(emotion_predict[0],axis=1)
emotion_matrix=confusion_matrix(test_emotion,emotion_reuslt)
pose_predict = model.predict_generator(pose_DataGenerator(model,test_pose_paths,test_pose,args.batch_size))
pose_reuslt=np.argmax(pose_predict[1],axis=1)
pose_matrix=confusion_matrix(test_pose,pose_reuslt)
print('emotion confusion matrix:',emotion_matrix)
print('pose confusion matrix:',pose_matrix)
np.savetxt(matrix_path+'{}_emotion_matrix.txt'.format(epoch),emotion_matrix)
np.savetxt(matrix_path+'{}_pose_matrix.txt'.format(epoch),pose_matrix)
else:
model.compile(optimizer='adam', loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch == 0 or args.is_naive:
model_pre = None
else:
model_pre = Model(inputs=model.input,outputs=model.output)
model_pre.set_weights(model.get_weights())
if args.is_pesudo or args.is_interpolation:
is_hard_weight = not args.is_interpolation
pesudo_emotion = assign_weights(model,False,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights)
pesudo_pose = assign_weights(model,True,train_emotion_paths,train_emotion,train_pose_paths,train_pose,args.cluster_k,args.is_augmentation,args.selection_threshold,args.is_pesudo_density,args.is_pesudo_distribution,is_hard_weight,args.interpolation_weights)
else:
pesudo_emotion=None
pesudo_pose=None
model.fit_generator(
DataGenerator(model,model_pre,train_emotion_paths,train_pose_paths,train_emotion,train_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,args.is_augmentation,pesudo_emotion,pesudo_pose),
validation_data = DataGenerator(model, None,test_emotion_paths,test_pose_paths, test_emotion,test_pose,args.batch_size,args.is_distilled,args.is_pesudo,args.is_interpolation,args.selection_threshold,args.interpolation_weights,False,pesudo_emotion,pesudo_pose),
epochs=each_term_epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
if epoch==args.no_freezing_epoch:
print('no freezing and start to changing learning rate')
model = no_freeze_all(model)
adam = keras.optimizers.Adam(lr=1*1e-4)
print('lr changing to:',1*1e-4)
model.compile(optimizer=adam, loss=loss_dict,loss_weights = loss_weights, metrics=metrics_dict)
print('epoch is{}'.format(epoch))
model.summary()
print('{}_train_finished'.format(epoch))
emotion_predict = model.predict_generator(emotion_DataGenerator(model,test_emotion_paths,test_emotion,args.batch_size))
emotion_reuslt=np.argmax(emotion_predict[0],axis=1)
emotion_matrix=confusion_matrix(test_emotion,emotion_reuslt)
pose_predict = model.predict_generator(pose_DataGenerator(model,test_pose_paths,test_pose,args.batch_size))
pose_reuslt=np.argmax(pose_predict[1],axis=1)
pose_matrix=confusion_matrix(test_pose,pose_reuslt)
print('emotion confusion matrix:',emotion_matrix)
print('pose confusion matrix:',pose_matrix)
np.savetxt(matrix_path+'{}_emotion_matrix.txt'.format(epoch),emotion_matrix)
np.savetxt(matrix_path+'{}_pose_matrix.txt'.format(epoch),pose_matrix)
del train_emotion_paths, train_pose_paths,train_emotion,train_pose
del test_emotion_paths, test_pose_paths,test_emotion, test_pose
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
EMOTION_DATASET = args.dataset_emotion
POSE_DATASET = args.dataset_pose
if EMOTION_DATASET == 'ferplus':
emotion_classes = 8
else:
emotion_classes = 7
gender_classes = 2
age_classes = 8
pose_classes = 5
# emotion_path='train_weights/EmotionNetVGGFace_vgg16/expw/2__01-0.64.hdf5'
# pose_path = 'train_weights/PoseNetVGGFace_vgg16/aflw/1__01-0.77.hdf5'
# age_path = 'train_weights/AgeNetVGGFace_vgg16/adience/1__01-0.67.hdf5'
paths_emotion, paths_pose, emotion_label, pose_label = load_data(
EMOTION_DATASET, POSE_DATASET)
kf = KFold(n_splits=100, shuffle=True, random_state=1)
kf_split_emotion = kf.split(paths_emotion)
kf_split_pose = kf.split(paths_pose)
emotion_kf = [[emotion_train_idx, emotion_test_idx]
for emotion_train_idx, emotion_test_idx in kf_split_emotion]
pose_kf = [[pose_train_idx, pose_test_idx]
for pose_train_idx, pose_test_idx in kf_split_pose]
# for emotion_train_idx,emotion_test_idx in kf_split_emotion:
# for gender_age_train_idx,gender_age_test_idx in kf_split_gender_age:
# print(emotion_train_idx,emotion_test_idx,gender_age_train_idx,gender_age_test_idx)
emotion_train_idx, emotion_test_idx = emotion_kf[0]
pose_train_idx, pose_test_idx = pose_kf[0]
print(len(emotion_train_idx), len(pose_train_idx))
print(len(emotion_test_idx), len(pose_test_idx))
train_emotion_paths = paths_emotion[emotion_train_idx]
train_emotion = emotion_label[emotion_train_idx]
test_emotion_paths = paths_emotion[emotion_test_idx]
test_emotion = emotion_label[emotion_test_idx]
train_pose_paths = paths_pose[pose_train_idx]
train_pose = pose_label[pose_train_idx]
test_pose_paths = paths_pose[pose_test_idx]
test_pose = pose_label[pose_test_idx]
model = None
if MODEL == 'vggFace':
print('based on vggface')
model = Net(MODEL, 1, 11, emotion_classes, pose_classes, age_classes,
gender_classes, args.is_dropout, args.is_bn,
args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL, 1, 11, emotion_classes, pose_classes, age_classes,
gender_classes, args.is_dropout, args.is_bn,
args.weights_decay)
MODEL = model.name
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
loss_dict = {}
metrics_dict = {}
prediction = []
loss = []
acc = []
loss_weights = []
prediction.append('emotion_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.E_loss_weights)
prediction.append('pose_prediction')
loss.append(my_cross_loss)
acc.append(my_acc)
loss_weights.append(args.P_loss_weights)
loss_dict = dict(zip(prediction, loss))
metrics_dict = dict(zip(prediction, acc))
#pesudo label
if args.is_naive:
is_naive = 'naive_true'
print('this is naive method')
else:
is_naive = 'naive_false'
if args.is_distilled:
is_distilled = 'distilled_true'
print('this is distiiled method')
else:
is_distilled = 'distilled_false'
if args.is_pesudo:
is_pesudo = 'pesudo_true'
print('this is pesudo method')
else:
is_pesudo = 'pesudo_false'
if args.is_pesudo_confidence:
is_pesudo_confidence = 'pesudo_confidence_true'
else:
is_pesudo_confidence = 'pesudo_confidence_false'
if args.is_pesudo_density:
is_pesudo_density = 'pesudo_density_true'
else:
is_pesudo_density = 'pesudo_density_false'
if args.is_pesudo_distribution:
is_pesudo_distribution = 'pesudo_distribution_true'
else:
is_pesudo_distribution = 'pesudo_distribution_false'
if args.is_interpolation:
is_soft_weight = 'soft_weight'
print('this is overall soft weight')
else:
is_soft_weight = 'hard_weight'
print('this is overall hard weight')
#training trich
if args.is_augmentation:
augmentation = 'augmentation_true'
else:
augmentation = 'augmentation_false'
if args.is_dropout:
dropout = 'dropout_true'
else:
dropout = 'dropout_false'
if args.is_bn:
bn = 'bn_true'
else:
bn = 'bn_false'
#import parameter
if args.is_freezing:
freezing = 'freezing_true'
else:
freezing = 'freezing_false'
main_path = '{}-{}/{}/'.format(EMOTION_DATASET, POSE_DATASET, MODEL)
# appendex_path='Net-{}_{}_{}_{}-Pesudo-{}_{}_{}_{}_{}_{}_{}_distill_t-{}_threshold_d-{}_interpolation_w-{}_cluster_k-{}_density_t-{}-LR-{}_{}_{}_{}'.format(augmentation,dropout,bn,args.weights_decay,
# is_naive,is_distilled,is_soft_weight,is_pesudo,is_pesudo_confidence,is_pesudo_density,is_pesudo_distribution,
# args.distill_t,args.selection_threshold,args.interpolation_weights,args.cluster_k,args.density_t,
# args.E_loss_weights,args.P_loss_weights,freezing,args.no_freezing_epoch)
appendex_path = 'Pesudo-{}_{}_{}_{}_{}_{}_{}_distill_t-{}_threshold_d-{}_interpolation_w-{}_cluster_k-{}_density_t-{}-LR-{}_{}_{}_{}'.format(
is_naive, is_distilled, is_soft_weight, is_pesudo,
is_pesudo_confidence, is_pesudo_density, is_pesudo_distribution,
args.distill_t, args.selection_threshold, args.interpolation_weights,
args.cluster_k, args.density_t, args.E_loss_weights,
args.P_loss_weights, freezing, args.no_freezing_epoch)
weights_path = './train_weights/ITERATION/' + main_path + appendex_path
logs_path = './train_log/ITERATION/' + main_path + appendex_path
matrix_path = './train_log/ITERATION/' + main_path + appendex_path + '/matrix/'
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
if not os.path.exists(matrix_path):
os.makedirs(matrix_path)
model_names = weights_path + '.{val_emotion_prediction_my_acc:.4f}-{val_pose_prediction_my_acc:.4f}.hdf5'
csv_name = logs_path + '.log'
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=True)
csvlogger = CSVLogger(filename=csv_name, append=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE),
verbose=1)
tensorboard = TensorBoard(log_dir=logs_path, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
print('whether freezing:', args.is_freezing)
# model_path = './train_weights/ITERATION/ferplus-aflw/EmotionPoseNetVGGFace_vgg16/Pesudo-naive_false_distilled_false_soft_weight_pesudo_true_pesudo_confidence_true_pesudo_density_true_pesudo_distribution_true_distill_t-2_threshold_d-0.0_interpolation_w-0.0_cluster_k-4_density_t-0.6-LR-1.0_1.0_freezing_false_0.0.7856-0.7687.hdf5'
# model.summary()
# model.load_weights(model_path)
if args.is_pesudo or args.is_interpolation or args.is_distilled:
each_term_epoch = 1
else:
each_term_epoch = 1
if not args.is_freezing:
adam = keras.optimizers.Adam(lr=0.0001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
# adam = keras.optimizers.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False)
model = no_freeze_all(model)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch == -1:
print('initializing')
model.fit_generator(Test_DataGenerator(
model, None, train_emotion_paths, train_pose_paths,
train_emotion, train_pose, args.batch_size,
args.is_distilled, args.is_pesudo, args.is_interpolation,
args.selection_threshold, args.interpolation_weights,
args.is_augmentation, None, None),
validation_data=Test_DataGenerator(
model, None, test_emotion_paths,
test_pose_paths, test_emotion,
test_pose, args.batch_size,
args.is_distilled, args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights, False,
None, None),
epochs=4,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
print('initializing finished')
else:
# model_pre = keras.models.clone_model(model)
# print('load:',model.get_weights()[3][:10])
model_pre = Model(inputs=model.input, outputs=model.output)
# model_pre.set_weights(model.get_weights())
# print('pre:',model_pre.get_weights()[3][:10])
if epoch % 2 == 0:
is_emotion = True
if args.is_naive:
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=[1, 0],
metrics=metrics_dict)
else:
is_emotion = False
if args.is_naive:
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=[0, 1],
metrics=metrics_dict)
if args.is_pesudo or args.is_interpolation or args.is_distilled:
is_hard_weight = not args.is_interpolation
pesudo_data = assign_weights(
model, model_pre, is_emotion, train_emotion_paths,
train_emotion, train_pose_paths, train_pose,
args.cluster_k, args.is_augmentation,
args.selection_threshold, args.is_distilled,
args.is_pesudo_confidence, args.is_pesudo_density,
args.is_pesudo_distribution, is_hard_weight,
args.interpolation_weights, args.distill_t,
args.density_t)
else:
pesudo_data = None
model.fit_generator(
DataGenerator(model, model_pre, train_emotion_paths,
train_pose_paths, train_emotion, train_pose,
args.batch_size, is_emotion,
args.is_distilled, args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
args.is_augmentation, pesudo_data),
validation_data=Test_DataGenerator(
model, None, test_emotion_paths, test_pose_paths,
test_emotion, test_pose, args.batch_size,
args.is_distilled, args.is_pesudo,
args.is_interpolation, args.selection_threshold,
args.interpolation_weights, False, None, None),
epochs=each_term_epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
# print('after:',model.get_weights()[3][:10])
if epoch % 2 == 0:
print('{}_emotion_train_finished'.format(epoch))
else:
print('{}_pose_train_finished'.format(epoch))
emotion_predict = model.predict_generator(
emotion_DataGenerator(model, test_emotion_paths, test_emotion,
args.batch_size))
emotion_reuslt = np.argmax(emotion_predict[0], axis=1)
emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt)
pose_predict = model.predict_generator(
pose_DataGenerator(model, test_pose_paths, test_pose,
args.batch_size))
pose_reuslt = np.argmax(pose_predict[1], axis=1)
pose_matrix = confusion_matrix(test_pose, pose_reuslt)
print('emotion confusion matrix:', emotion_matrix)
print('pose confusion matrix:', pose_matrix)
np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch),
emotion_matrix)
np.savetxt(matrix_path + '{}_pose_matrix.txt'.format(epoch),
pose_matrix)
else:
model.compile(optimizer='adam',
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
model.summary()
for epoch in range(EPOCH):
if epoch == 0:
print('initializing')
model.fit_generator(Test_DataGenerator(
model, None, train_emotion_paths, train_pose_paths,
train_emotion, train_pose, args.batch_size,
args.is_distilled, args.is_pesudo, args.is_interpolation,
args.selection_threshold, args.interpolation_weights,
args.is_augmentation, None, None),
validation_data=Test_DataGenerator(
model, None, test_emotion_paths,
test_pose_paths, test_emotion,
test_pose, args.batch_size,
args.is_distilled, args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights, False,
None, None),
epochs=3,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
print('initializing finished')
else:
# model_pre = keras.models.clone_model(model)
model_pre = Model(inputs=model.input, outputs=model.output)
model_pre.set_weights(model.get_weights())
if epoch % 2 == 0:
is_emotion = True
if args.is_naive:
if epoch < args.no_freezing_epoch:
model.compile(optimizer='adam',
loss=loss_dict,
loss_weights=[1, 0],
metrics=metrics_dict)
else:
adam = keras.optimizers.Adam(lr=1 * 1e-4)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=[1, 0],
metrics=metrics_dict)
else:
is_emotion = False
if args.is_naive:
if epoch < args.no_freezing_epoch:
model.compile(optimizer='adam',
loss=loss_dict,
loss_weights=[0, 1],
metrics=metrics_dict)
else:
adam = keras.optimizers.Adam(lr=1 * 1e-4)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=[0, 1],
metrics=metrics_dict)
if args.is_pesudo or args.is_interpolation or args.is_distilled:
is_hard_weight = not args.is_interpolation
pesudo_data = assign_weights(
model, model_pre, is_emotion, train_emotion_paths,
train_emotion, train_pose_paths, train_pose,
args.cluster_k, args.is_augmentation,
args.selection_threshold, args.is_distilled,
args.is_pesudo_confidence, args.is_pesudo_density,
args.is_pesudo_distribution, is_hard_weight,
args.interpolation_weights, args.distill_t,
args.density_t)
else:
pesudo_data = None
model.fit_generator(
DataGenerator(model, model_pre, train_emotion_paths,
train_pose_paths, train_emotion, train_pose,
args.batch_size, is_emotion,
args.is_distilled, args.is_pesudo,
args.is_interpolation,
args.selection_threshold,
args.interpolation_weights,
args.is_augmentation, pesudo_data),
validation_data=Test_DataGenerator(
model, None, test_emotion_paths, test_pose_paths,
test_emotion, test_pose, args.batch_size,
args.is_distilled, args.is_pesudo,
args.is_interpolation, args.selection_threshold,
args.interpolation_weights, False, None, None),
epochs=each_term_epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
if epoch % 2 == 0:
print('{}_emotion_train_finished'.format(epoch))
else:
print('{}_pose_train_finished'.format(epoch))
if epoch == args.no_freezing_epoch:
print('no freezing and start to changing learning rate')
model = no_freeze_all(model)
adam = keras.optimizers.Adam(lr=0.00001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
print('lr changing to:', 1 * 1e-5)
model.compile(optimizer=adam,
loss=loss_dict,
loss_weights=loss_weights,
metrics=metrics_dict)
print('epoch is{}'.format(epoch))
model.summary()
emotion_predict = model.predict_generator(
emotion_DataGenerator(model, test_emotion_paths, test_emotion,
args.batch_size))
emotion_reuslt = np.argmax(emotion_predict[0], axis=1)
emotion_matrix = confusion_matrix(test_emotion, emotion_reuslt)
pose_predict = model.predict_generator(
pose_DataGenerator(model, test_pose_paths, test_pose,
args.batch_size))
pose_reuslt = np.argmax(pose_predict[1], axis=1)
pose_matrix = confusion_matrix(test_pose, pose_reuslt)
print('emotion confusion matrix:', emotion_matrix)
print('pose confusion matrix:', pose_matrix)
np.savetxt(matrix_path + '{}_emotion_matrix.txt'.format(epoch),
emotion_matrix)
np.savetxt(matrix_path + '{}_pose_matrix.txt'.format(epoch),
pose_matrix)
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
if args.task_type == 0:
task_name = 'emotion'
elif args.task_type ==1:
task_name = 'age'
elif args.task_type == 5:
task_name = 'pose'
elif args.task_type == 10:
task_name = 'gender'
paths, task_label = load_data(DATASET,task_name)
print('[K-FOLD] Started...')
kf = KFold(n_splits=5, shuffle=True, random_state=1)
kf_split_task = kf.split(task_label)
task_kf = [[task_train_idx,task_test_idx] for task_train_idx,task_test_idx in kf_split_task]
task_train_idx,task_test_idx = task_kf[0]
train_paths = paths[task_train_idx]
train_task = task_label[task_train_idx]
test_paths = paths[task_test_idx]
test_task = task_label[task_test_idx]
model = None
if MODEL == 'vggFace':
if args.dataset=='ferplus':
model = Net(MODEL,1,args.task_type,8,5,8,2,args.is_dropout,args.is_bn,args.weights_decay)
else:
model = Net(MODEL,1,args.task_type,7,5,8,2,args.is_dropout,args.is_bn,args.weights_decay)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
if args.dataset=='ferplus':
model = Net(MODEL,1,args.task_type,8,5,8,2,args.is_dropout,args.is_bn,args.weights_decay)
else:
model = Net(MODEL,1,args.task_type,7,5,8,2,args.is_dropout,args.is_bn,args.weights_decay)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
if args.task_type == 0:
task_classes = model.emotion_classes
losses = {"emotion_prediction": 'categorical_crossentropy' }
metrics = { "emotion_prediction": 'acc' }
elif args.task_type ==1:
task_classes = model.age_classes
losses = {"age_prediction": 'categorical_crossentropy' }
metrics = { "age_prediction": 'acc' }
elif args.task_type == 5:
task_classes = model.pose_classes
losses = {"pose_prediction": 'categorical_crossentropy' }
metrics = { "pose_prediction": 'acc' }
elif args.task_type == 10:
task_classes = model.gender_classes
losses = {"gender_prediction": 'categorical_crossentropy' }
metrics = { "gender_prediction": 'acc' }
#training trick
if args.is_augmentation:
augmentation='augmentation_true'
else:
augmentation='augmentation_false'
if args.is_dropout:
dropout='dropout_true'
else:
dropout='dropout_false'
if args.is_bn:
bn = 'bn_true'
else:
bn = 'bn_false'
# key parameter
if args.is_freezing:
freezing='freezing_true'
else:
freezing='freezing_false'
weights_path = './train_weights/{}/{}/net-{}_{}_{}_{}-lr-{}_{}'.format(model.name,DATASET,augmentation,dropout,bn,args.weights_decay,freezing,args.no_freezing_epoch)
logs_path = './train_log/{}/{}/net-{}_{}_{}_{}-lr-{}_{}'.format(model.name,DATASET,augmentation,dropout,bn,args.weights_decay,freezing,args.no_freezing_epoch)
matrix_path = logs_path+'/matrix/'
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
if not os.path.exists(matrix_path):
os.makedirs(matrix_path)
model_names = weights_path + '{epoch:02d}-{val_acc:.4f}.hdf5'
csv_name = logs_path + '.log'
board_name = logs_path
checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True)
csvlogger=CSVLogger(filename=csv_name,separator=',',append=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
patience=int(PATIENCE), verbose=1)
tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE)
callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard]
if args.is_freezing:
model.compile(optimizer='adam', loss=losses, metrics=metrics)
model.summary()
model.fit_generator(
DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation),
validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False),
epochs=args.no_freezing_epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks
)
if args.epoch>args.no_freezing_epoch:
adam = keras.optimizers.Adam(lr=1*1e-5)
print('lr changing to:',1*1e-5)
model = no_freeze_all(model)
model.compile(optimizer=adam, loss=losses, metrics=metrics)
model.summary()
model.fit_generator(
DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation),
validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False),
epochs=args.epoch-args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
predicted_result = model.predict_generator(DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False))
task_reuslt=np.argmax(np.array(predicted_result),axis=1)
matrix=confusion_matrix(test_task,task_reuslt)
print('confusion matrix:',matrix)
np.savetxt(matrix_path+'.txt',matrix)
else:
adam = keras.optimizers.Adam(lr=0.00001)
model.compile(optimizer=adam, loss=losses, metrics=metrics)
model.summary()
model.fit_generator(
DataGenerator(model, train_paths, train_task, task_classes, BATCH_SIZE,is_augmentation=args.is_augmentation),
validation_data=DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False),
epochs=args.epoch,
verbose=2,
workers=1,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
predicted_result = model.predict_generator(DataGenerator(model, test_paths, test_task, task_classes,BATCH_SIZE,is_augmentation=False))
task_reuslt=np.argmax(np.array(predicted_result),axis=1)
matrix=confusion_matrix(test_task,task_reuslt)
print('confusion matrix:',matrix)
np.savetxt(matrix_path+'.txt',matrix)
del train_paths, train_task
del test_paths, test_task
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
paths, pose_label = load_data(DATASET)
n_fold = 1
losses = {"pose_prediction": custom_mse_pose}
metrics = {"pose_prediction": pose_metric}
print('[K-FOLD] Started...')
kf = KFold(n_splits=10, shuffle=True, random_state=1)
kf_split = kf.split(pose_label)
for train_idx, test_idx in kf_split:
model = None
if MODEL == 'vggFace':
model = Net(MODEL,1,5,8,2,8)
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL,1,5,8,2,8)
train_paths = paths[train_idx]
train_label = pose_label[train_idx]
test_paths = paths[test_idx]
test_label = pose_label[test_idx]
print(len(train_paths),len(test_paths))
model.summary()
weights_path = './train_weights/pose/{}/{}/'.format(DATASET,model.name)
logs_path = './train_log/pose/{}/{}/'.format(DATASET,model.name)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
model_names = weights_path + '.{epoch:02d}-{val_pose_metric: 0.4f}.hdf5'
csv_name = logs_path + '{}.log'.format(n_fold)
board_name = logs_path + '{}'.format(n_fold)
checkpoint = ModelCheckpoint(model_names, verbose=1,save_weights_only = True,save_best_only=True)
csvlogger=CSVLogger(csv_name)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss', factor=0.1,
patience=int(PATIENCE/2), verbose=1)
tensorboard = TensorBoard(log_dir=board_name,batch_size=BATCH_SIZE)
callbacks = [checkpoint,csvlogger,early_stop,reduce_lr,tensorboard]
if MODEL == 'ssrnet':
callbacks = [
# ModelCheckpoint(
# "train_weight/{}-{val_gender_prediction_binary_accuracy:.4f}-{val_age_prediction_mean_absolute_error:.4f}.h5".format(
# MODEL),
# verbose=1, save_best_only=True, save_weights_only=True),
ModelCheckpoint(MODEL, 'val_loss', verbose=1,save_best_only=True),
CSVLogger('train_log/{}-{}.log'.format(MODEL, n_fold)),
DecayLearningRate([30, 60])]
model.compile(optimizer='adam', loss=losses, metrics = metrics)
model.fit_generator(
DataGenerator(model, train_paths, train_label, BATCH_SIZE),
validation_data=DataGenerator(model, test_paths, test_label, BATCH_SIZE),
epochs=EPOCH,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks
)
n_fold += 1
del train_paths, train_roll,train_pictch,train_yaw
del test_paths, test_roll,test_pitch,test_yaw
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
if DATASET == 'fgnet':
age_classes = 70
losses = {"age_prediction": mae}
metrics = {"age_prediction": mae}
elif DATASET == 'megaage_asian':
age_classes = 71
losses = {"age_prediction": mae}
metrics = {"age_prediction": mae}
else:
gender_classes = 2
age_classes = 8
losses = {"age_prediction": 'categorical_crossentropy'}
metrics = {"age_prediction": 'acc'}
paths, age_label = load_data(DATASET)
n_fold = 1
print('[K-FOLD] Started...')
kf = KFold(n_splits=10, shuffle=True, random_state=1)
kf_split = kf.split(age_label)
for train_idx, test_idx in kf_split:
model = None
if MODEL == 'vggFace':
model = Net(MODEL, 1, 1, 8, 2, age_classes)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
model = Net(MODEL, 1, 1, 8, 2, age_classes)
train_paths = paths[train_idx]
train_age = age_label[train_idx]
test_paths = paths[test_idx]
test_age = age_label[test_idx]
if MODEL == 'ssrnet':
losses = {
"age_prediction": "mae",
"gender_prediction": "mae",
}
metrics = {
"age_prediction": "mae",
"gender_prediction": "binary_accuracy",
}
weights_path = './train_weights/age/{}/{}/'.format(DATASET, model.name)
logs_path = './train_log/age/{}/{}/'.format(DATASET, model.name)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
model_names = weights_path + '{}__'.format(
n_fold) + '{epoch:02d}-{val_acc:.2f}.hdf5'
csv_name = logs_path + '{}.log'.format(n_fold)
board_name = logs_path + '{}'.format(n_fold)
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE / 2),
verbose=1)
tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
model.compile(optimizer='adam', loss=losses, metrics=metrics)
# emotion_model = Net(MODEL,1,1,8,2,age_classes)
# emotion_model = updateTargetmodel(model,emotion_model)
# emotion_model = keras.models.clone_model(model)
# emotion_model.set_weights(model.get_weights())
# emotion_model = freeze_all(emotion_model)
# emotion_model.compile(optimizer='adam', loss=losses, metrics=metrics)
print('freezing model')
model.summary()
model.fit_generator(DataGenerator(model, train_paths, train_age,
age_classes, BATCH_SIZE),
validation_data=DataGenerator(
model, test_paths, test_age, age_classes,
BATCH_SIZE),
epochs=args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
model = no_freeze_all(model)
print('no freezing')
model.summary()
model.fit_generator(DataGenerator(model, train_paths, train_age,
age_classes, BATCH_SIZE),
validation_data=DataGenerator(
model, test_paths, test_age, age_classes,
BATCH_SIZE),
epochs=EPOCH - args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
n_fold += 1
print(
np.array_equal(model.get_weights()[0][0][0][0],
emotion_model.get_weights()[0][0][0][0]))
if np.array_equal(model.get_weights()[0],
emotion_model.get_weights()[0]):
print('please check code!')
del train_paths, train_age
del test_paths, test_age
def main():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.tensorflow_backend.set_session(sess)
args = parser.parse_args()
DATASET = args.dataset
MODEL = args.model
EPOCH = args.epoch
PATIENCE = args.patience
BATCH_SIZE = args.batch_size
NUM_WORKER = args.num_worker
if args.task_type == 0:
task_name = 'emotion'
elif args.task_type == 1:
task_name = 'age'
elif args.task_type == 5:
task_name = 'pose'
elif args.task_type == 10:
task_name = 'gender'
paths, task_label = load_data(DATASET, task_name)
print('[K-FOLD] Started...')
kf = KFold(n_splits=5, shuffle=True, random_state=1)
kf_split_task = kf.split(task_label)
task_kf = [[task_train_idx, task_test_idx]
for task_train_idx, task_test_idx in kf_split_task]
task_train_idx, task_test_idx = task_kf[0]
train_paths = paths[task_train_idx]
train_task = task_label[task_train_idx]
test_paths = paths[task_test_idx]
test_task = task_label[task_test_idx]
model = None
if MODEL == 'vggFace':
if args.dataset == 'ferplus':
model = Net(MODEL, 1, args.task_type, args.is_dropout, 8, 5, 8, 2)
else:
model = Net(MODEL, 1, args.task_type, args.is_dropout, 7, 5, 8, 2)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
else:
if args.dataset == 'ferplus':
model = Net(MODEL, 1, args.task_type, args.is_dropout, 8, 5, 8, 2)
else:
model = Net(MODEL, 1, args.task_type, args.is_dropout, 7, 5, 8, 2)
if args.is_freezing:
model = freeze_all_but_mid_and_top(model)
if args.task_type == 0:
task_classes = model.emotion_classes
losses = {"emotion_prediction": 'categorical_crossentropy'}
metrics = {"emotion_prediction": 'acc'}
elif args.task_type == 1:
task_classes = model.age_classes
losses = {"age_prediction": 'categorical_crossentropy'}
metrics = {"age_prediction": 'acc'}
elif args.task_type == 5:
task_classes = model.pose_classes
losses = {"pose_prediction": 'categorical_crossentropy'}
metrics = {"pose_prediction": 'acc'}
elif args.task_type == 10:
task_classes = model.gender_classes
losses = {"gender_prediction": 'categorical_crossentropy'}
metrics = {"gender_prediction": 'acc'}
if args.is_freezing:
freezing = 'freezing_true'
else:
freezing = 'freezing_false'
if args.is_dropout:
dropout = 'dropout_true'
else:
dropout = 'drouout_false'
weights_path = './train_weights/{}/{}/{}-{}_'.format(
model.name, DATASET, freezing, dropout)
logs_path = './train_log/{}/{}/{}-{}_'.format(model.name, DATASET,
freezing, dropout)
if not os.path.exists(weights_path):
os.makedirs(weights_path)
if not os.path.exists(logs_path):
os.makedirs(logs_path)
model_names = weights_path + '{epoch:02d}-{val_acc:.2f}.hdf5'
csv_name = logs_path + '.log'
board_name = logs_path
checkpoint = ModelCheckpoint(model_names,
verbose=1,
save_weights_only=True,
save_best_only=True)
csvlogger = CSVLogger(filename=csv_name, separator=',', append=True)
early_stop = EarlyStopping('val_loss', patience=PATIENCE)
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE / 2),
verbose=1)
tensorboard = TensorBoard(log_dir=board_name, batch_size=BATCH_SIZE)
callbacks = [checkpoint, csvlogger, early_stop, reduce_lr, tensorboard]
adam = keras.optimizer.adam(lr=0.0001)
model.compile(optimizer=adam, loss=losses, metrics=metrics)
print('starting model')
model.summary()
model.fit_generator(DataGenerator(model, train_paths, train_task,
task_classes, BATCH_SIZE),
validation_data=DataGenerator(model, test_paths,
test_task, task_classes,
BATCH_SIZE),
epochs=args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks)
if args.is_freezing:
reduce_lr = LearningRateScheduler(scheduler)
else:
reduce_lr = ReduceLROnPlateau('val_loss',
factor=0.1,
patience=int(PATIENCE / 2),
verbose=1)
if args.is_freezing:
before_freezing_weights = model.get_weights()[1]
model = no_freeze_all(model)
model.compile(optimizer=adam, loss=losses, metrics=metrics)
callbacks_new = [
checkpoint, csvlogger, early_stop, reduce_lr, tensorboard
]
no_freezing_weights = model.get_weights()[1]
print('combile whether change',
np.equal(before_freezing_weights, no_freezing_weights))
model.summary()
model.fit_generator(DataGenerator(model, train_paths, train_task,
task_classes, BATCH_SIZE),
validation_data=DataGenerator(model, test_paths,
test_task, task_classes,
BATCH_SIZE),
epochs=EPOCH - args.no_freezing_epoch,
verbose=2,
workers=NUM_WORKER,
use_multiprocessing=False,
max_queue_size=int(BATCH_SIZE * 2),
callbacks=callbacks_new)
train_no_freezing_weights = model.get_weights()[1]
print('second stage training change',
np.equal(no_freezing_weights, train_no_freezing_weights))
del train_paths, train_task
del test_paths, test_task