def train():
md = Model(is_training=True)
print('---Read Data...')
image_batch, label_batch, mask_beta_batch, mask_lambda_batch = datasets.get_train_batch(
train_tfrecord_name_path, md.batch_size)
# print(image_batch)
print('---Training Model...')
init_fn = tf.contrib.slim.assign_from_checkpoint_fn(
pretrain_vgg_19_ckpt_path,
tf.contrib.slim.get_model_variables('vgg_19')) # 'vgg_19'
saver = tf.train.Saver(max_to_keep=501)
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(summary_path, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
init_fn(sess)
coord = tf.train.Coordinator() # queue manage
threads = tf.train.start_queue_runners(coord=coord)
iter = 0
loss_list = []
for epoch in range(num_epochs):
for _ in range(train_val_num / md.batch_size):
images, labels, mask_betas, mask_lambdas = sess.run([
image_batch, label_batch, mask_beta_batch,
mask_lambda_batch
])
feed_dict = {
md.images: images,
md.labels: labels,
md.mask_beta: mask_betas,
md.mask_lambda: mask_lambdas
}
_, _summary, _global_step, _loss, = sess.run(
[md.step_op, md.summary, md.global_step, md.loss],
feed_dict=feed_dict)
train_writer.add_summary(_summary, _global_step)
loss_list.append(_loss)
iter += 1
if iter % 1000 == 0:
print('epoch = %s, iter = %s, loss = %s' %
(epoch, iter, np.mean(loss_list)))
loss_list = []
if iter % 1000 == 0:
saver.save(sess, model_path_save, global_step=iter)
coord.request_stop()
coord.join(threads)
print('Train end.')
def test(tfrecord_list_path, data_nums, model_path):
md = Model(is_training=False) # Test model
print('---Read Data...')
image_batch, sentence_batch, mask_batch, image_id_batch = datasets.get_train_batch(
tfrecord_list_path, md.batch_size)
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver.restore(sess, model_path)
coord = tf.train.Coordinator() # queue manage
threads = tf.train.start_queue_runners(coord=coord)
loss_list = []
acc_list = []
predictions_list = []
image_id_list = []
for _ in range(data_nums / md.batch_size):
images, sentences, masks, image_ids = sess.run(
[image_batch, sentence_batch, mask_batch, image_id_batch])
feed_dict = {
md.images: images,
md.sentences: sentences,
md.masks: masks
}
_loss, _acc, _predictions, = sess.run(
[md.loss, md.accuracy, md.predictions], feed_dict=feed_dict)
loss_list.append(_loss)
acc_list.append(_acc)
predictions_list.append(_predictions)
image_id_list.append(image_ids)
bleu, meteor, rouge, cider = metrics.coco_caption_metrics(
predictions_list,
image_id_list,
batch_size=md.batch_size,
is_training=md.is_training)
print(
'loss = %s, acc = %s, belu = %s, meteor = %s, rouge = %s, cider = %s'
% (np.mean(loss_list), np.mean(acc_list), bleu, meteor, rouge,
cider))
coord.request_stop()
coord.join(threads)
def train():
c = Config()
md = Model(is_training=True, config=c, batch_size=c.batch_size)
mt = Model(is_training=False, config=c, batch_size=3)
print('Read Data...')
image_frontal_batch, image_lateral_batch, sentence_batch, mask_batch, image_id_batch = datasets.get_train_batch(
c.train_tfrecord_path, c, md.batch_size)
image_frontal_batch2, image_lateral_batch2, sentence_batch2, mask_batch2, image_id_batch2 = datasets.get_train_batch(
c.test_tfrecord_path, c, mt.batch_size)
init_fn_frontal = slim.assign_from_checkpoint_fn(
c.pretrain_cnn_model_frontal,
slim.get_model_variables('FrontalInceptionV3'))
init_fn_lateral = slim.assign_from_checkpoint_fn(
c.pretrain_cnn_model_lateral,
slim.get_model_variables('LateralInceptionV3'))
saver = tf.train.Saver(max_to_keep=100)
print('Train Model...')
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(c.summary_path, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
init_fn_frontal(sess)
init_fn_lateral(sess)
coord = tf.train.Coordinator() # queue manage
threads = tf.train.start_queue_runners(coord=coord)
iter = 0
# loss_list, acc_list, predicts_list, sentences_list, image_id_list = [], [], [], [], []
for epoch in range(c.epoch_num):
for _ in range(c.train_num / md.batch_size):
images_frontal, images_lateral, sentences, masks, image_ids = sess.run(
[
image_frontal_batch, image_lateral_batch,
sentence_batch, mask_batch, image_id_batch
])
feed_dict = {
md.images_frontal: images_frontal,
md.images_lateral: images_lateral,
md.sentences: sentences,
md.masks: masks
}
_, _summary, _global_step, _loss, _acc, _predicts, = sess.run(
[
md.step_op, md.summary, md.global_step, md.loss,
md.accuracy, md.predicts
],
feed_dict=feed_dict)
train_writer.add_summary(_summary, _global_step)
# loss_list.append(_loss)
# acc_list.append(_acc)
# predicts_list.append(_predicts)
# sentences_list.append(sentences)
# image_id_list.append(image_ids)
iter += 1
if iter % 100 == 0:
# train test
# bleu, meteor, rouge, cider = metrics.coco_caption_metrics_hier(predicts_list,
# sentences_list,
# image_id_list,
# config=c,
# batch_size=md.batch_size,
# is_training=md.is_training)
# print('iter = %s, loss = %.4f, acc = %.4f, bleu = %s, meteor = %s, rouge = %s, cider = %s' %
# (iter, np.mean(loss_list), np.mean(acc_list), bleu, meteor, rouge, cider))
# test test
loss_list, acc_list, predicts_list, sentences_list, image_id_list = [], [], [], [], []
for _ in range(c.test_num / mt.batch_size):
images_frontal, images_lateral, sentences, masks, image_ids = sess.run(
[
image_frontal_batch2, image_lateral_batch2,
sentence_batch2, mask_batch2, image_id_batch2
])
feed_dict = {
mt.images_frontal: images_frontal,
mt.images_lateral: images_lateral,
mt.sentences: sentences,
mt.masks: masks
}
_loss, _acc, _predicts = sess.run(
[mt.loss, mt.accuracy, mt.predicts],
feed_dict=feed_dict)
loss_list.append(_loss)
acc_list.append(_acc)
predicts_list.append(_predicts)
sentences_list.append(sentences)
image_id_list.append(image_ids)
bleu, meteor, rouge, cider = metrics.coco_caption_metrics_hier(
predicts_list,
sentences_list,
image_id_list,
config=c,
batch_size=mt.batch_size,
is_training=mt.is_training)
print(
'---------iter = %s, loss = %.4f, acc = %.4f, bleu = %s, meteor = %s, rouge = %s, cider = %s'
% (iter, np.mean(loss_list), np.mean(acc_list), bleu,
meteor, rouge, cider))
# loss_list, acc_list, predicts_list, sentences_list, image_id_list = [], [], [], [], []
saver.save(sess, c.model_path, global_step=iter)
coord.request_stop()
coord.join(threads)
def train():
md = Model(is_training=True) # Train model
# md_val = Model(is_training=True) # Val model
# md_test = Model(is_training=False) # Test model
print('---Read Data...')
image_batch, sentence_batch, mask_batch, image_id_batch = datasets.get_train_batch(
train_tfrecord_name_path, md.batch_size)
# image_list_v, sentence_list_v, mask_list_v, image_id_list_v = datasets.get_inputs(imgs_path, captions_path, val_split_path, vocabulary_path)
# image_list_t, sentence_list_t, mask_list_t, image_id_list_t = datasets.get_inputs(imgs_path, captions_path, test_split_path, vocabulary_path)
print('---Training Model...')
init_fn = slim.assign_from_checkpoint_fn(
pretrain_inception_v3_ckpt_path,
slim.get_model_variables('InceptionV3')) # 'InceptionV3'
saver = tf.train.Saver(max_to_keep=301)
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(summary_path, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
init_fn(sess)
coord = tf.train.Coordinator() # queue manage
threads = tf.train.start_queue_runners(coord=coord)
iter = 1
loss_list = []
acc_list = []
predictions_list = []
image_id_list = []
for epoch in range(num_epochs):
for _ in range(train_num / md.batch_size):
# for efficiency, it is no need tensor to numpy, the numpy to tensor, so no need feed_dict, but for model's simplicity, I make the unneccessary move.
images, sentences, masks, image_ids = sess.run(
[image_batch, sentence_batch, mask_batch, image_id_batch])
feed_dict = {
md.images: images,
md.sentences: sentences,
md.masks: masks
}
_, _summary, _global_step, _loss, _acc, _predictions, = sess.run(
[
md.step_op, md.summary, md.global_step, md.loss,
md.accuracy, md.predictions
],
feed_dict=feed_dict)
train_writer.add_summary(_summary, _global_step)
loss_list.append(_loss)
acc_list.append(_acc)
predictions_list.append(_predictions)
image_id_list.append(image_ids)
if iter % 1000 == 0:
saver.save(sess, model_path_save, global_step=iter)
bleu, meteor, rouge, cider = metrics.coco_caption_metrics(
predictions_list,
image_id_list,
batch_size=md.batch_size)
print(
'epoch = %s, iter = %s, loss = %.4f, acc = %.4f, bleu = %s, meteor = %s, rouge = %s, cider = %s'
% (epoch, iter, np.mean(loss_list), np.mean(acc_list),
bleu, meteor, rouge, cider))
# val model
# loss_list, acc_list, predictions_list, image_id_list = eval(sess, md_val, image_list_v, sentence_list_v, mask_list_v, image_id_list_v)
# bleu, meteor, rouge, cider = metrics.coco_caption_metrics(predictions_list, image_id_list, batch_size=md_val.batch_size)
# loss_val = round(np.mean(loss_list), 4)
# acc_val = round(np.mean(acc_list), 4)
# print('------epoch = %s, iter = %s, loss = %s, acc = %s, bleu = %s, meteor = %s, rouge = %s, cider = %s' %
# (epoch, iter, tc.colored(loss_val, 'red'), tc.colored(acc_val, 'red'), tc.colored(bleu, 'red'),
# tc.colored(meteor, 'red'), tc.colored(rouge, 'red'), tc.colored(cider, 'red')))
# # test model
# loss_list, acc_list, predictions_list, image_id_list = eval(sess, md_test, image_list_t, sentence_list_t, mask_list_t, image_id_list_t)
# bleu, meteor, rouge, cider = metrics.coco_caption_metrics(predictions_list, image_id_list, batch_size=md_test.batch_size)
# loss_test = round(np.mean(loss_list), 4)
# acc_test = round(np.mean(acc_list), 4)
# print('------epoch = %s, iter = %s, loss = %s, acc = %s, bleu = %s, meteor = %s, rouge = %s, cider = %s' %
# (epoch, iter, tc.colored(loss_test, 'blue'), tc.colored(acc_test, 'blue'), tc.colored(bleu, 'blue'),
# tc.colored(meteor, 'blue'), tc.colored(rouge, 'blue'), tc.colored(cider, 'blue')))
loss_list = []
acc_list = []
predictions_list = []
image_id_list = []
iter += 1
coord.request_stop()
coord.join(threads)
def eval(model_path):
md = Model(is_training=False)
print('---Read Data...')
image_batch, label_batch, mask_beta_batch, mask_lambda_batch = datasets.get_train_batch(
test_tfrecord_name_path, md.batch_size)
temp_label_tp = tf.placeholder(shape=[None], dtype=tf.float32)
predictions_tp = tf.placeholder(shape=[None], dtype=tf.float32)
_, auc_op = tf.metrics.auc(temp_label_tp, predictions_tp)
print('---Test Model...')
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
saver.restore(sess, model_path)
coord = tf.train.Coordinator() # queue manage
threads = tf.train.start_queue_runners(coord=coord)
loss_list = []
iter = 0
epochs = test_num / md.batch_size
auc_predict = np.zeros([epochs * md.batch_size, md.label_num])
auc_label = np.zeros([epochs * md.batch_size, md.label_num])
threshold = 0.5 * np.ones([md.label_num])
# threshold = [0.0991, 0.0318, 0.0165, 0.0203, 0.0261, 0.097, 0.0338, 0.0171, 0.0104, 0.055, 0.0483, 0.1592, 0.0017, 0.5841, 0.0138]
true_positive_count = np.zeros([md.label_num])
precision_positive_count = np.zeros([md.label_num])
actual_positive_count = np.zeros([md.label_num])
for _ in range(epochs):
images, labels, mask_betas, mask_lambdas = sess.run(
[image_batch, label_batch, mask_beta_batch, mask_lambda_batch])
feed_dict = {
md.images: images,
md.labels: labels,
md.mask_beta: mask_betas,
md.mask_lambda: mask_lambdas
}
_loss, _predictions, _auc = sess.run(
[md.loss, md.predictions, md.auc], feed_dict=feed_dict)
loss_list.append(_loss)
for i in range(md.batch_size):
auc_predict[iter] = _predictions[i]
auc_label[iter] = labels[i]
iter += 1
# print(_predictions[i])
# print(labels[i])
# print('*' * 100)
for j in range(md.label_num):
if labels[i][j] == 1 and _predictions[i][j] >= threshold[j]:
true_positive_count[j] += 1
if labels[i][j] == 1:
actual_positive_count[j] += 1
if _predictions[i][j] >= threshold[j]:
precision_positive_count[j] += 1
coord.request_stop()
coord.join(threads)
# 1. loss and auc
auc_list = []
for i in range(md.label_num):
temp_auc = sess.run(auc_op,
feed_dict={
temp_label_tp: auc_label[:, i],
predictions_tp: auc_predict[:, i]
})
auc_list.append(temp_auc)
# print(auc_list)
print('mean loss = %s, mean auc = %s, %s' %
(np.mean(loss_list), np.mean(auc_list), auc_list))
# 2. recall and prediction
# recall = true_positive_count / actual_positive_count
# for j in range(md.label_num):
# recall[j] = round(recall[j], 4)
# print('mean recall = %s, %s' % (np.mean(recall), recall.tolist()))
# precision = true_positive_count / precision_positive_count
# for j in range(md.label_num):
# precision[j] = round(precision[j], 4)
# print('mean precision = %s, %s' % (np.mean(precision), precision.tolist()))
# f1_score = 2 * recall * precision / (recall + precision)
# for j in range(md.label_num):
# f1_score[j] = round(f1_score[j], 4)
# print('mean F1-score = %s, %s' % (np.mean(f1_score), f1_score.tolist()))
# 3. to find good threshold for 15 label
# 3.1 use mean to get threshold
threshold_mean = np.mean(auc_predict, axis=0)
for j in range(md.label_num):
threshold_mean[j] = round(threshold_mean[j], 4)
print('mean throshold = %s' % threshold_mean.tolist())
# 3.2 from train label nums to get threshold
# label_count = [8659, 2637, 1378, 1707, 2242, 8208, 2852, 1423, 876, 4708, 4034, 13782, 141, 50500, 1251]
# threshold = np.zeros([md.label_num])
# for j in range(md.label_num):
# temp_array = np.array(sorted(auc_predict.tolist(), key=lambda auc_predict: auc_predict[j]))
# threshold[j] = temp_array[label_count[j], j]
# threshold[j] = round(threshold[j], 4)
# print('throshold = %s' % threshold.tolist())
print('Test end.')