def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TEST_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# define model graph
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in, attn_length=30)
prediction = tf.reshape(prediction, [-1, 2])
label_batch = tf.reshape(label, [-1, 2])
with tf.Session(graph=g) as sess:
restore_variables(sess)
evaluated_predictions = []
evaluated_labels = []
while True:
try:
pred, lab = sess.run([prediction, label_batch])
evaluated_predictions.append(pred)
evaluated_labels.append(lab)
except tf.errors.OutOfRangeError:
break
print 'Finish read data'
predictions = np.reshape(evaluated_predictions, (-1, 2))
labels = np.reshape(evaluated_labels, (-1, 2))
for i, name in enumerate(['valence', 'arousal']):
result = np.stack((predictions[:, i], labels[:, i]), axis=-1)
np.savetxt(SUMMARY_PATH + name + '_test.txt',
result,
fmt="%.3f")
conc_arousal = concordance_cc2(predictions[:, 1], labels[:, 1])
conc_valence = concordance_cc2(predictions[:, 0], labels[:, 0])
mse_arousal = sum(
(predictions[:, 1] - labels[:, 1])**2) / len(labels[:, 1])
mse_valence = sum(
(predictions[:, 0] - labels[:, 0])**2) / len(labels[:, 0])
print '#####################Summary#######################'
print 'Concordance on valence : {}'.format(conc_valence)
print 'Concordance on arousal : {}'.format(conc_arousal)
print 'Concordance on total : {}'.format(
(conc_arousal + conc_valence) / 2)
print 'MSE Arousal : {}'.format(mse_arousal)
print 'MSE Valence : {}'.format(mse_valence)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = reshape_to_cnn(image)
VGGFace_network.setup(image_batch, trainable=False)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_lstm(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
compute_loss(prediction, label_batch)
total_loss = slim.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
restore_names = VGGFace_network.get_restore_vars()
variables_to_restore = tf.contrib.framework.get_variables_to_restore(include=restore_names)
init_fn = slim.assign_from_checkpoint_fn(VGG_RESTORE_PATH, variables_to_restore, ignore_missing_vars=False)
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
train_op = slim.learning.create_train_op(total_loss,
optimizer,
summarize_gradients=True # Whether or not add summaries for each gradient.
)
loggingTF.set_verbosity(1)
# keep 10000 ckpts
saver = tf.train.Saver(max_to_keep=10000)
# including initialize local and global variables
slim.learning.train(train_op,
TRAIN_DIR,
init_fn=init_fn,
save_summaries_secs=60 * 15, # How often, in seconds, to save summaries.
log_every_n_steps=500, # The frequency, in terms of global steps, that the loss and global step are logged.
save_interval_secs=60 * 15, # How often, in seconds, to save the model to `logdir`.
saver=saver
)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# Construct and return the model
image_batch = reshape_to_cnn(image)
network_fn = nets_factory.get_network_fn(
name='densenet121',
num_classes=None,
weight_decay=0.00004,
data_format='NHWC',
is_training=True
)
face_output, _ = network_fn(image_batch)
# print face_output.get_shape().as_list()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
# compute_loss(prediction, label_batch)
# total_loss = slim.losses.get_total_loss()
# optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
# when the network only contains DenseNet all variables can be restored,
# so here we restore global variables with ignore_missing_vars = True
variables_to_restore = tf.global_variables()
init_fn = slim.assign_from_checkpoint_fn(DenseNet_RESTORE_PATH, variables_to_restore,
ignore_missing_vars=True)
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
trainable_names = ['attention-layer', 'fcatten']
variables_to_train = tf.contrib.framework.get_variables_to_restore(include=trainable_names)
print_var(variables_to_train)
print 'global vars'
print_var(tf.global_variables())
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TEST_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# define model graph
image_batch = tf.reshape(image, [-1, 96, 96, 3])
# Construct and return the model
network_fn = nets_factory.get_network_fn(name='densenet161',
num_classes=None,
weight_decay=0.00004,
data_format='NHWC',
is_training=False)
face_output, _ = network_fn(image_batch)
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# Computing MSE and Concordance values, and adding them to summary
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'eval/mse_valence':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 0],
label_batch[:, :, 0]),
'eval/mse_arousal':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 1],
label_batch[:, :, 1]),
})
summary_ops = []
conc_total = 0
mse_total = 0
for i, name in enumerate(['valence', 'arousal']):
with tf.name_scope(name) as scope:
concordance_cc2, values, updates = metrics.concordance_cc2(
tf.reshape(prediction[:, :, i], [-1]),
tf.reshape(label_batch[:, :, i], [-1]))
for n, v in updates.items():
names_to_updates[n + '/' + name] = v
op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2)
op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name)
summary_ops.append(op)
mse_eval = 'eval/mse_' + name
op = tf.summary.scalar(mse_eval, names_to_values[mse_eval])
op = tf.Print(op, [names_to_values[mse_eval]], mse_eval)
summary_ops.append(op)
mse_total += names_to_values[mse_eval]
conc_total += concordance_cc2
conc_total = conc_total / 2
mse_total = mse_total / 2
op = tf.summary.scalar('eval/concordance_total', conc_total)
op = tf.Print(op, [conc_total], 'eval/concordance_total')
summary_ops.append(op)
op = tf.summary.scalar('eval/mse_total', mse_total)
op = tf.Print(op, [mse_total], 'eval/mse_total')
summary_ops.append(op)
num_batches = int(NUM_BATCHES)
loggingTF.set_verbosity(1)
if not os.path.exists(SUMMARY_PATH):
os.makedirs(SUMMARY_PATH)
slim.evaluation.evaluate_once(
'',
MODEL_PATH,
SUMMARY_PATH,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
summary_op=tf.summary.merge(summary_ops),
)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE, NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# Construct and return the model
image_batch = reshape_to_cnn(image)
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
face_output, _ = resnet_v1.resnet_v1_50(
inputs=image_batch,
num_classes=None,
is_training=True,
global_pool=True,
)
# print face_output.get_shape().as_list()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in, attn_length=30)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
compute_loss(prediction, label_batch)
total_loss = slim.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
# when the network only contains DenseNet all variables can be restored,
# so here we restore global variables with ignore_missing_vars = True
variables_to_restore = tf.global_variables()
init_fn = slim.assign_from_checkpoint_fn(ResNet_RESTORE_PATH, variables_to_restore,
ignore_missing_vars=True)
# init_fn(sess)
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
trainable_names = ['attention-layer', 'fcatten']
variables_to_train = tf.contrib.framework.get_variables_to_restore(include=trainable_names)
# print_var(variables_to_train)
# print_var(tf.global_variables())
train_op = slim.learning.create_train_op(total_loss,
optimizer,
variables_to_train=variables_to_train,
summarize_gradients=True
# Whether or not add summaries for each gradient.
)
loggingTF.set_verbosity(1)
# keep 10000 ckpts
saver = tf.train.Saver(max_to_keep=10000)
# including initialize local and global variables
slim.learning.train(train_op,
TRAIN_DIR,
init_fn=init_fn,
save_summaries_secs=60 * 15, # How often, in seconds, to save summaries.
log_every_n_steps=500,
# The frequency, in terms of global steps, that the loss and global step are logged.
save_interval_secs=60 * 15, # How often, in seconds, to save the model to `logdir`.
saver=saver
)
def evaluate():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(VALID_TFR_PATH, False)
frameNo, image, label = iterator.get_next()
# define model graph
# VGG FACE network
VGGFace_network = vgg_face.VGGFace(SEQUENCE_LENGTH * BATCH_SIZE)
image_batch = tf.reshape(image, [-1, 96, 96, 3])
VGGFace_network.setup(
image_batch
) # image_batch is a tensor of shape (batch_size*seq_length,image_dim,image_dim,3)
face_output = VGGFace_network.get_face_fc0()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_indRNN(rnn_in)
prediction = tf.reshape(prediction, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# Computing MSE and Concordance values, and adding them to summary
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'eval/mse_valence':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 0],
label_batch[:, :, 0]),
'eval/mse_arousal':
slim.metrics.streaming_mean_squared_error(prediction[:, :, 1],
label_batch[:, :, 1]),
})
summary_ops = []
conc_total = 0
mse_total = 0
for i, name in enumerate(['valence', 'arousal']):
with tf.name_scope(name) as scope:
concordance_cc2, values, updates = metrics.concordance_cc2(
tf.reshape(prediction[:, :, i], [-1]),
tf.reshape(label_batch[:, :, i], [-1]))
for n, v in updates.items():
names_to_updates[n + '/' + name] = v
op = tf.summary.scalar('eval/concordance_' + name, concordance_cc2)
op = tf.Print(op, [concordance_cc2], 'eval/concordance_' + name)
summary_ops.append(op)
mse_eval = 'eval/mse_' + name
op = tf.summary.scalar(mse_eval, names_to_values[mse_eval])
op = tf.Print(op, [names_to_values[mse_eval]], mse_eval)
summary_ops.append(op)
mse_total += names_to_values[mse_eval]
conc_total += concordance_cc2
conc_total = conc_total / 2
mse_total = mse_total / 2
op = tf.summary.scalar('eval/concordance_total', conc_total)
op = tf.Print(op, [conc_total], 'eval/concordance_total')
summary_ops.append(op)
op = tf.summary.scalar('eval/mse_total', mse_total)
op = tf.Print(op, [mse_total], 'eval/mse_total')
summary_ops.append(op)
num_batches = int(NUM_BATCHES)
loggingTF.set_verbosity(1)
if not os.path.exists(SUMMARY_PATH):
os.makedirs(SUMMARY_PATH)
# always check latest ckpt and wait for next.
slim.evaluation.evaluation_loop(
'',
CHECK_POINT_DIR,
SUMMARY_PATH,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
summary_op=tf.summary.merge(summary_ops),
eval_interval_secs=EVAL_INTERVAL_SECS,
)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# Construct and return the model
image_batch = reshape_to_cnn(image)
# 'face2_gru_atten' : trainable=False, 'face2_gru_atten_trainCNN' : trainable=True,
resnet = ResNet50({'data': image_batch}, trainable=False)
face_output = resnet.get_output()
# print face_output.get_shape().as_list()
# resnet.load(VF2_RESTORE_PATH, sess)
# saver_model = tf.train.Saver()
# MODEL_PATH = '/vol/gpudata/ml9915/caffe_model/resnet50_scratch_caffe/model.ckpt-0'
# saver_model.save(sess, MODEL_PATH, global_step=0)
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in)
prediction = tf.reshape(prediction,
[BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
compute_loss(prediction, label_batch)
total_loss = slim.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
# restore_names = get_restore_variable()
# variables_to_restore = tf.contrib.framework.get_variables_to_restore(include=restore_names)
# print_var(variables_to_restore)
init_fn = slim.assign_from_checkpoint_fn(ResNet50_RESTORE_PATH,
tf.global_variables(),
ignore_missing_vars=True)
# init_fn(sess)
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
trainable_names = ['attention-layer', 'fcatten']
variables_to_train = tf.contrib.framework.get_variables_to_restore(
include=trainable_names)
# print_var(tf.trainable_variables())
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
variables_to_train=variables_to_train,
summarize_gradients=True
# Whether or not add summaries for each gradient.
)
loggingTF.set_verbosity(1)
# keep 10000 ckpts
saver = tf.train.Saver(max_to_keep=10000)
# including initialize local and global variables
slim.learning.train(
train_op,
TRAIN_DIR,
init_fn=init_fn,
save_summaries_secs=60 *
15, # How often, in seconds, to save summaries.
log_every_n_steps=500,
# The frequency, in terms of global steps, that the loss and global step are logged.
save_interval_secs=60 *
15, # How often, in seconds, to save the model to `logdir`.
saver=saver)
def train():
g = tf.Graph()
with g.as_default():
# load data get iterator
data_loader = data_utils_mean.DataLoader(SEQUENCE_LENGTH, BATCH_SIZE,
NUM_EPOCHS)
iterator = data_loader.load_data(TRAIN_TFR_PATH, True)
with tf.Session(graph=g) as sess:
frameNo, image, label = iterator.get_next()
# Construct and return the model
image_batch = reshape_to_cnn(image)
network_fn = nets_factory.get_network_fn(name='densenet121',
num_classes=None,
weight_decay=0.00004,
data_format='NHWC',
is_training=True)
face_output, _ = network_fn(image_batch)
# print face_output.get_shape().as_list()
# RNN part
rnn_in = reshape_to_rnn(face_output)
prediction = models.get_prediction_atten(rnn_in, attn_length=30)
prediction = tf.reshape(prediction,
[BATCH_SIZE, SEQUENCE_LENGTH, 2])
label_batch = tf.reshape(label, [BATCH_SIZE, SEQUENCE_LENGTH, 2])
# compute losses using slim
compute_loss(prediction, label_batch)
total_loss = slim.losses.get_total_loss()
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# restore VGG-FACE model at the beginning
# when the network only contains DenseNet all variables can be restored,
# so here we restore global variables with ignore_missing_vars = True
flow_init_assign_op, flow_init_feed_dict = slim.assign_from_checkpoint(
RNN_RESTORE_PATH,
get_RNN_variables(),
ignore_missing_vars=False)
resnet_init_assign_op, resnet_init_feed_dict = slim.assign_from_checkpoint(
DenseNet_RESTORE_PATH,
get_CNN_variables(),
ignore_missing_vars=False)
def init_fn(sess):
sess.run(flow_init_assign_op, flow_init_feed_dict)
sess.run(resnet_init_assign_op, resnet_init_feed_dict)
# print_var(tf.trainable_variables())
# print_var(get_RNN_variables())
# print_var(get_CNN_variables())
# summarize_gradients : Whether or not add summaries for each gradient.
# variables_to_train: an optional list of variables to train. If None, it will default to all tf.trainable_variables().
train_op = slim.learning.create_train_op(
total_loss,
optimizer,
summarize_gradients=True
# Whether or not add summaries for each gradient.
)
loggingTF.set_verbosity(1)
# keep 10000 ckpts
saver = tf.train.Saver(max_to_keep=10000)
# including initialize local and global variables
slim.learning.train(
train_op,
TRAIN_DIR,
init_fn=init_fn,
save_summaries_secs=60 *
15, # How often, in seconds, to save summaries.
log_every_n_steps=500,
# The frequency, in terms of global steps, that the loss and global step are logged.
save_interval_secs=60 *
15, # How often, in seconds, to save the model to `logdir`.
saver=saver)
