def train():
model_config = ModelConfig()
training_config = TrainConfig()
model_config.batch_size = 32
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % (FLAGS.model_type, model_config.feat_type)
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.u
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=training_config.decay_step,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# create reader
reader = AttentionDataReader(batch_size=model_config.batch_size,
subset='trainval',
feat_type=model_config.feat_type)
# Run training.
training_util.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
reader=reader,
feed_fn=model.fill_feed_dict)
def train():
model_config = ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
training_config = TrainConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# Create training directory.
train_dir = FLAGS.train_dir % FLAGS.model_type
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'train')
model.build()
# Set up the learning rate.
learning_rate = tf.constant(training_config.initial_learning_rate)
def _learning_rate_decay_fn(learn_rate, global_step):
return tf.train.exponential_decay(
learn_rate,
global_step,
decay_steps=3,
decay_rate=training_config.decay_factor)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up gradient clipping function
# def _clip_gradient_by_value(gvs):
# return [(tf.clip_by_value(grad, -training_config.clip_gradients,
# training_config.clip_gradients), var) for grad, var in gvs]
# grad_proc_fn = _clip_gradient_by_value
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver)
def vaq_multiple_choices(checkpoint_path=None, subset='kpval'):
# subset = 'kptest'
need_attr = True
need_im_feat = False
use_answer_type = False
# feat_type = 'semantic'
feat_type = 'res152'
model_config = ModelConfig()
model_config.cell_option = 4
# Get model
print(FLAGS.model_type)
model_fn = get_model_creation_fn(FLAGS.model_type)
mc_ctx = MultipleChoiceEvaluater(subset='val',
need_im_feat=need_im_feat,
need_attr=need_attr,
feat_type=feat_type,
use_ans_type=use_answer_type)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir % FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, phase='condition')
model.build()
losses = model.losses
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' % os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
num_batches = mc_ctx.num_samples
print('Running multiple choices...')
predictions, answer_ids = [], []
for i in range(num_batches):
if i % 1000 == 0:
print('Running multiple choices: %d/%d' % (i, num_batches))
outputs = mc_ctx.get_task_data()
feed_dict = model.fill_feed_dict(outputs[:-2])
np_losses = sess.run(losses, feed_dict=feed_dict)
scores = -np_losses.sum(axis=1)[np.newaxis, :]
predictions.append(scores)
answer_ids.append(outputs[-2])
predictions = np.concatenate(predictions, axis=0)
answer_ids = np.array(answer_ids)
# evaluate
mc_ctx.evaluate_results(answer_ids, predictions,
model_type=FLAGS.model_type)
def test(checkpoint_path=None):
batch_size = 1
config = ModelConfig()
# Get model function
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = AttentionFetcher(batch_size=batch_size, subset=TEST_SET)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(
FLAGS.checkpoint_dir % (FLAGS.model_type, config.feat_type))
checkpoint_path = ckpt.model_checkpoint_path
print(checkpoint_path)
# build and restore model
model = model_fn(config, phase='test')
model.build()
g_prob = model.prob
g_att_map = model.attention_map
# sess = tf.Session()
sess = tf.Session(graph=tf.get_default_graph())
tf.logging.info('Restore from model %s' %
os.path.basename(checkpoint_path))
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
# Create the vocabulary.
visualiser = PredictionVisualiser(FLAGS.model_type, do_plot=True)
ans_ids = []
quest_ids = []
print('Running inference on split %s...' % TEST_SET)
for i in range(reader.num_batches):
if i % 100 == 0:
update_progress(i / float(reader.num_batches))
outputs = reader.get_test_batch()
if i < 100:
continue
generated_ans, att_map = sess.run([g_prob, g_att_map],
feed_dict=model.fill_feed_dict(
outputs[:-2]))
# process attention map
att_map = att_map.reshape([batch_size, 14, 14, -1])
att_map = np.transpose(att_map, [0, 3, 1, 2])
generated_ans[:, -1] = 0
top_ans = np.argmax(generated_ans, axis=1)
gt_ans = outputs[3]
ans_ids.append(top_ans)
quest_id = outputs[-2]
quest_ids.append(quest_id)
if np.random.rand() > 0.05:
visualiser.plot(quest_id, generated_ans, att_map)
def vaq_condition(checkpoint_path=None):
subset = 'dev'
model_config = ModelConfig()
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = Reader(batch_size=1, subset=subset, output_attr=True, output_im=False,
output_qa=True, output_capt=False)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir % FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'condition')
model.build()
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' % os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
fetch_op = model.losses
num_batches = reader.num_batches
save_file = 'data/%s_vaq_cond_score1000-2000_%s.hdf5' % ((FLAGS.model_type).lower(), subset)
print('Save File: %s' % save_file)
print('Running conditioning...')
nlls, quest_ids = [], []
for i in range(num_batches):
update_progress(i / float(num_batches))
outputs = reader.get_test_batch()
im_feed, quest, _, ans_feed, quest_id, image_id = outputs
losses = sess.run(fetch_op, feed_dict=model.fill_feed_dict(outputs[:-2]))
scores = losses[:, :-1].mean(axis=1)
scores = scores[np.newaxis, ::]
nlls.append(scores)
quest_ids.append(quest_id)
nlls = np.concatenate(nlls, axis=0)
quest_ids = np.concatenate(quest_ids, axis=0)
print('\nSaving result files: %s...' % save_file)
save_hdf5(save_file, {'nll': nlls, 'quest_ids': quest_ids})
def build_model(self, model_config):
model_creator = get_model_creation_fn(model_config.model_type)
model = model_creator(model_config, phase=model_config.phase)
model.build()
return model
def vaq_condition(checkpoint_path=None):
subset = 'kpval'
model_config = ModelConfig()
model_config.cell_option = 4
# Get model
model_fn = get_model_creation_fn(FLAGS.model_type)
# build data reader
reader = Reader(batch_size=1, subset=subset, output_attr=True, output_im=True,
output_qa=True, output_capt=False, output_ans_seq=False)
ans_prop_ctx = AnswerProposals(top_k=5)
if checkpoint_path is None:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir % FLAGS.model_type)
checkpoint_path = ckpt.model_checkpoint_path
g = tf.Graph()
with g.as_default():
# Build the model.
model = model_fn(model_config, 'evaluate')
model.build()
saver = tf.train.Saver()
sess = tf.Session()
tf.logging.info('Restore from model %s' % os.path.basename(checkpoint_path))
saver.restore(sess, checkpoint_path)
fetch_op = model.losses
num_batches = reader.num_batches
save_file = 'data/%s_vqa_vaq_rerank_%s.hdf5' % ((FLAGS.model_type).lower(), subset)
print('Save File: %s' % save_file)
print('Running conditioning...')
vaq_scores, quest_ids = [], []
vqa_scores, vqa_pred_labels = [], []
for i in range(num_batches):
update_progress(i / float(num_batches))
outputs = reader.get_test_batch()
im_feed, attr_feed, quest, _, _, quest_id, image_id = outputs
quest_id = int(quest_id)
outputs, vqa_score, answer_ids = pre_process_input_data(outputs, ans_prop_ctx)
losses = sess.run(fetch_op, feed_dict=model.fill_feed_dict(outputs))
vaq_score = losses[:, :-1].mean(axis=1)
vaq_score = vaq_score[np.newaxis, ::]
vaq_scores.append(vaq_score)
quest_ids.append(quest_id)
vqa_scores.append(vqa_score[np.newaxis, :])
vqa_pred_labels.append(answer_ids[np.newaxis, :])
vaq_scores = np.concatenate(vaq_scores, axis=0)
vqa_scores = np.concatenate(vqa_scores, axis=0)
vqa_pred_labels = np.concatenate(vqa_pred_labels, axis=0)
quest_ids = np.array(quest_ids, dtype=np.int32)
print('\nSaving result files: %s...' % save_file)
save_hdf5(save_file, {'vaq_scores': vaq_scores,
'vqa_scores': vqa_scores,
'vqa_pred_labels': vqa_pred_labels,
'quest_ids': quest_ids})