def get_prediction(x):
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = reshape_to_cnn(x)
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
return prediction
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TEST_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# define model graph
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# Computing MSE and Concordance values, and adding them to summary
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'eval/mse_valence':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 0],
label_batch[:, :, 0]),
'eval/mse_arousal':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 1],
label_batch[:, :, 1]),
})
summary_ops = []
conc_total = 0
mse_total = 0
for i, name in enumerate(['valence', 'arousal']):
with tf.name_scope(name) as scope:
concordance_cc2, values, updates = metrics.concordance_cc2(
tf.reshape(prediction[:, :, i], [-1]),
tf.reshape(label_batch[:, :, i], [-1]))
for n, v in updates.items():
names_to_updates[n + '/' + name] = v
op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2)
op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name)
summary_ops.append(op)
mse_eval = 'eval/mse_' + name
op = tf.summary.scalar(mse_eval, names_to_values[mse_eval])
op = tf.Print(op, [names_to_values[mse_eval]], mse_eval)
summary_ops.append(op)
mse_total += names_to_values[mse_eval]
conc_total += concordance_cc2
conc_total = conc_total / 2
mse_total = mse_total / 2
op = tf.summary.scalar('eval/concordance_total', conc_total)
op = tf.Print(op, [conc_total], 'eval/concordance_total')
summary_ops.append(op)
op = tf.summary.scalar('eval/mse_total', mse_total)
op = tf.Print(op, [mse_total], 'eval/mse_total')
summary_ops.append(op)
num_batches = int(NUM_BATCHES)
loggingTF.set_verbosity(1)
if not os.path.exists(SUMMARY_PATH):
os.makedirs(SUMMARY_PATH)
slim.evaluation.evaluate_once(
'',
MODEL_PATH,
SUMMARY_PATH,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
summary_op=tf.summary.merge(summary_ops),
)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = reshape_to_cnn(image)
VGGFace_network.setup(image_batch, trainable=False)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction,
[BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
compute_loss(prediction, label_batch)
total_loss = slim.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
restore_names = VGGFace_network.get_restore_vars()
variables_to_restore = tf.contrib.framework.get_variables_to_restore(
include=restore_names)
init_fn = slim.assign_from_checkpoint_fn(VGG_RESTORE_PATH,
variables_to_restore,
ignore_missing_vars=False)
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
summarize_gradients=
True # Whether or not add summaries for each gradient.
)
loggingTF.set_verbosity(1)
# keep 10000 ckpts
saver = tf.train.Saver(max_to_keep=10000)
# including initialize local and global variables
slim.learning.train(
train_op,
TRAIN_DIR,
init_fn=init_fn,
save_summaries_secs=60 *
15, # How often, in seconds, to save summaries.
log_every_n_steps=
500, # The frequency, in terms of global steps, that the loss and global step are logged.
save_interval_secs=60 *
15, # How often, in seconds, to save the model to `logdir`.
saver=saver)
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(VALID_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# define model graph
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch, trainable=False
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# Computing MSE and Concordance values, and adding them to summary
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'eval/mse_valence':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 0],
label_batch[:, :, 0]),
'eval/mse_arousal':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 1],
label_batch[:, :, 1]),
})
summary_ops = []
conc_total = 0
mse_total = 0
for i, name in enumerate(['valence', 'arousal']):
with tf.name_scope(name) as scope:
concordance_cc2, values, updates = metrics.concordance_cc2(
tf.reshape(prediction[:, :, i], [-1]),
tf.reshape(label_batch[:, :, i], [-1]))
for n, v in updates.items():
names_to_updates[n + '/' + name] = v
op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2)
op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name)
summary_ops.append(op)
mse_eval = 'eval/mse_' + name
op = tf.summary.scalar(mse_eval, names_to_values[mse_eval])
op = tf.Print(op, [names_to_values[mse_eval]], mse_eval)
summary_ops.append(op)
mse_total += names_to_values[mse_eval]
conc_total += concordance_cc2
conc_total = conc_total / 2
mse_total = mse_total / 2
op = tf.summary.scalar('eval/concordance_total', conc_total)
op = tf.Print(op, [conc_total], 'eval/concordance_total')
summary_ops.append(op)
op = tf.summary.scalar('eval/mse_total', mse_total)
op = tf.Print(op, [mse_total], 'eval/mse_total')
summary_ops.append(op)
num_batches = int(NUM_BATCHES)
loggingTF.set_verbosity(1)
if not os.path.exists(SUMMARY_PATH):
os.makedirs(SUMMARY_PATH)
# always check latest ckpt and wait for next.
# slim.evaluation.evaluation_loop(
# '',
# CHECK_POINT_DIR,
# SUMMARY_PATH,
# num_evals=num_batches,
# eval_op=list(names_to_updates.values()),
# summary_op=tf.summary.merge(summary_ops),
# eval_interval_secs=EVAL_INTERVAL_SECS,
# )
# iterate all ckpts and evaluate
ckpt = tf.train.get_checkpoint_state(CHECK_POINT_DIR)
for model_path in ckpt.all_model_checkpoint_paths:
# step = utils.get_global_step(model_path)
# if step < 105000 or step > 241500:
# continue
slim.evaluation.evaluate_once(
'',
model_path,
SUMMARY_PATH,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
summary_op=tf.summary.merge(summary_ops),
)
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(VALID_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction(rnn_in)
prediction = tf.reshape(prediction, [-1, 2])
label_batch = tf.reshape(label, [-1, 2])
with tf.Session(graph=g) as sess:
# if not os.path.exists(SUMMARY_PATH):
# os.makedirs(SUMMARY_PATH)
# eval_summary_writer = tf.summary.FileWriter(SUMMARY_PATH, graph=g)
restore_variables(sess)
total_ccc_v = 0
total_ccc_a = 0
total_ccc = 0
total_mse_v = 0
total_mse_a = 0
total_mse = 0
for i in range(NUM_BATCHES):
try:
pred, lab = sess.run([prediction, label_batch])
except tf.errors.OutOfRangeError:
break
print 'prediction batch : ' + str(i)
print pred
conc_arousal = concordance_cc2(pred[:, 1], lab[:, 1])
conc_valence = concordance_cc2(pred[:, 0], lab[:, 0])
mse_arousal = sum((pred[:, 1] - lab[:, 1])**2) / len(lab[:, 1])
mse_valence = sum((pred[:, 0] - lab[:, 0])**2) / len(lab[:, 0])
total_ccc_v += conc_valence
total_ccc_a += conc_arousal
total_ccc += ((conc_valence + conc_valence) / 2.0)
total_mse_v += (mse_valence)
total_mse_a += (mse_arousal)
total_mse += ((mse_arousal + mse_valence) / 2.0)
print 'Finish read data'
# add summary
num_batches = float(NUM_BATCHES)
# summary = tf.Summary()
# summary.value.add(tag='eval/conc_valence', simple_value=float(total_ccc_v/num_batches))
# summary.value.add(tag='eval/conc_arousal', simple_value=float(total_ccc_a/num_batches))
# summary.value.add(tag='eval/conc_total', simple_value=float(total_ccc/num_batches))
# summary.value.add(tag='eval/mse_arousal', simple_value=float(total_mse_a/num_batches))
# summary.value.add(tag='eval/mse_valence', simple_value=float(total_mse_v/num_batches))
#
# eval_summary_writer.add_summary(summary, get_global_step(model_path))
print '#####################Summary#######################'
# print 'Evaluate model {}'.format(model_path)
print 'Concordance on valence : {}'.format(
float(total_ccc_v / num_batches))
print 'Concordance on arousal : {}'.format(
float(total_ccc_a / num_batches))
print 'Concordance on total : {}'.format(
float(total_ccc / num_batches))
print 'MSE Arousal : {}'.format(float(total_mse_a / num_batches))
print 'MSE Valence : {}'.format(float(total_mse_v / num_batches))
print 'MSE TOTAL : {}'.format(float(total_mse / num_batches))