def testCallModelFnWithPlaceholders(self):
with _reset_for_test() as session:
config = configuration.ModelConfig()
model = show_and_tell_model.ShowAndTellModel(config, mode='train')
def model_fn(images, input_seq, target_seq, input_mask):
model.build_model_for_tpu(images, input_seq, target_seq, input_mask)
return model.total_loss
images = tf.placeholder(tf.float32, shape=(1, 224, 224, 3))
input_seq = tf.placeholder(tf.int32, shape=(1, 128))
target_seq = tf.placeholder(tf.int32, shape=(1, 128))
input_mask = tf.placeholder(tf.int32, shape=(1, 128))
tpu_model_fn = tpu.rewrite(model_fn,
[images, input_seq, target_seq, input_mask])
caption = np.random.randint(low=0, high=1000, size=128).reshape((1, 128))
session.run(tpu.initialize_system())
session.run(tf.global_variables_initializer())
inputs = {
images: np.random.randn(1, 224, 224, 3),
input_seq: caption,
target_seq: caption,
input_mask: np.random.random_integers(0, 1, size=128).reshape(1, 128),
}
session.run(tpu_model_fn, inputs)
session.run(tpu.shutdown_system())
def run():
eval_dir=FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s",eval_dir)
tf.gfile.MakeDirs(eval_dir)
g=tf.Graph()
with g.as_default():
model_config=configuration.ModelConfig()
model_config.input_file_pattern=FLAGS.input_file_pattern
model=show_and_tell_model.ShowAndTellModel(model_config,mode="eval")
model.build()
saver=tf.train.Saver()
summary_op=tf.summary.merge_all()
summary_writer=tf.summary.FileWriter(eval_dir)
g.finalize()
i=-1
while True:
start=time.time()
tf.logging.info("Starting evaluation at "+time.strftime("%Y-%m-%d-%H:%M:%S",time.localtime()))
current_filenames=os.listdir(FLAGS.checkpoint_dir)
nums=[]
for x in current_filenames:
if x[-5:]=='index':
nums.append(int(re.findall(r'\d+',x)[0]))
nums.sort()
for x in nums:
if x>i:
run_once(model,saver,os.path.join(FLAGS.checkpoint_dir,'model.ckpt-'+str(x)),summary_writer,summary_op)
i=x
break
time_to_next_eval=start+FLAGS.eval_interval_secs-time.time()
if time_to_next_eval>0:
time.sleep(time_to_next_eval)
def run():
"""Runs evaluation in a loop, and logs summaries to TensorBoard."""
# Create the evaluation directory if it doesn't exist.
eval_dir = FLAGS.eval_dir
if not tf.gfile.IsDirectory(eval_dir):
tf.logging.info("Creating eval directory: %s", eval_dir)
tf.gfile.MakeDirs(eval_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = show_and_tell_model.ShowAndTellModel(model_config, mode="eval")
model.build()
# Create the Saver to restore model Variables.
saver = tf.train.Saver()
# Create the summary operation and the summary writer.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir)
g.finalize()
# Run a new evaluation run every eval_interval_secs.
while True:
start = time.time()
tf.logging.info("Starting evaluation at " + time.strftime(
"%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op)
time_to_next_eval = start + FLAGS.eval_interval_secs - time.time()
if time_to_next_eval > 0:
time.sleep(time_to_next_eval)
def main(unused_argv):
assert FLAGS.input_file_pattern,"--input_file_pattern is required"
assert FLAGS.train_dir,"--train_dir is required"
model_config=configuration.ModelConfig()
model_config.input_file_pattern=FLAGS.input_file_pattern
model_config.inception_checkpoint_file=FLAGS.inception_checkpoint_file
training_config=configuration.TrainingConfig()
train_dir=FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s",train_dir)
tf.gfile.MakeDirs(train_dir)
g=tf.Graph()
with g.as_default():
model=show_and_tell_model.ShowAndTellModel(model_config,mode="train",train_inception=FLAGS.train_inception)
model.build()
learning_rate_decay_fn=None
if FLAGS.train_inception:
learning_rate=tf.constant(training_config.train_inception_learning_rate)
else:
learning_rate=tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor>0:
num_batches_per_epoch=(training_config.num_examples_per_epoch/model_config.batch_size)
decay_steps=int(num_batches_per_epoch*training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate,global_step):
return tf.train.exponential_decay(learning_rate,global_step,decay_steps=decay_steps,decay_rate=training_config.learning_rate_decay_factor,staircase=True)
learning_rate_decay_fn=_learning_rate_decay_fn
train_op=tf.contrib.layers.optimize_loss(loss=model.total_loss,global_step=model.global_step,learning_rate=learning_rate,optimizer=training_config.optimizer,clip_gradients=training_config.clip_gradients,learning_rate_decay_fn=learning_rate_decay_fn)
saver=tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep)
tf.contrib.slim.learning.train(train_op,train_dir,log_every_n_steps=FLAGS.log_every_n_steps,graph=g,global_step=model.global_step,number_of_steps=FLAGS.number_of_steps,init_fn=model.init_fn,saver=saver)
def model_fn(features, labels, mode, params):
im_mode = MODEKEY_TO_MODE[mode]
model_config = configuration.ModelConfig()
training_config = configuration.TrainingConfig()
model = show_and_tell_model.ShowAndTellModel(
model_config, mode=im_mode, train_inception=FLAGS.train_inception)
model.build_model_for_tpu(images=features["images"],
input_seqs=features["input_seqs"],
target_seqs=features["target_seqs"],
input_mask=features["input_mask"])
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=training_config.initial_learning_rate)
optimizer = tf.contrib.estimator.clip_gradients_by_norm(
optimizer, training_config.clip_gradients)
if FLAGS.use_tpu:
optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
train_op = optimizer.minimize(
model.total_loss, global_step=tf.train.get_or_create_global_step())
def scaffold_fn():
"""Load pretrained Inception checkpoint at initialization time."""
return tf.train.Scaffold(init_fn=model.init_fn)
return tf.contrib.tpu.TPUEstimatorSpec(mode=mode,
loss=model.total_loss,
train_op=train_op,
scaffold_fn=scaffold_fn)
def input_fn(params):
model_config = configuration.ModelConfig()
model_config.input_file_pattern = params["input_file_pattern"]
model_config.batch_size = params["batch_size"]
model_config.mode = params["mode"]
model = show_and_tell_model.ShowAndTellModel(model_config, mode="train")
model.build_inputs()
return {
"images": model.images,
"input_seqs": model.input_seqs,
"target_seqs": model.target_seqs,
"input_mask": model.input_mask
}
def main(_):
model = show_and_tell_model.ShowAndTellModel( FLAGS.model_path )
vocabi = vocab.Vocabulary(FLAGS.vocab_file)
filenames = _load_filenames()
generator = caption_generator.CaptionGenerator (model, vocabi)
for filename in filenames:
with tf.gfile.GFile(filename, "rb") as f:
image = f.read()
captions = generator.beam_search(image)
print("Captions for image %s:" % os.path.basename(filename))
for i, caption in enumerate(captions):
# Ignore begin and end tokens <S> and </S>.
sentence = [ vocab.Vocabulary.id_to_token(vocabi,w) for w in caption.sentence[1:-1]]
sentence = " ".join(sentence)
print(" %d) %s (p=%f)" % (i, sentence, math.exp(caption.logprob)))
def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
training_config = configuration.TrainingConfig()
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = show_and_tell_model.ShowAndTellModel(model_config,
mode="train")
model.build()
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=None,
optimizer=tf.train.AdamOptimizer())
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
save_interval_secs=300)
def run():
inference_dir = FLAGS.inference_dir
if not tf.gfile.IsDirectory(inference_dir):
tf.logging.info("Creating inference directory: %s", inference_dir)
tf.gfile.MakeDirs(inference_dir)
g = tf.Graph()
with g.as_default():
# Build the model for evaluation.
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model = show_and_tell_model.ShowAndTellModel(model_config,
mode="inference")
model.build()
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(inference_dir)
g.finalize()
tf.logging.info("Starting inference at " +
time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()))
run_once(model, saver, summary_writer, summary_op)
tf.logging.info("Competed inference")
model_config = configuration.ModelConfig()
training_config = configuration.TrainingConfig()
conf = configuration.MyConfig()
iter = read_data.DataIterator(encoded_image_path=conf.feat_path,
caption_vector_path=conf.original_train_vec_path,
image_size=conf.label_image_size)
# gpu_config = tf.ConfigProto(device_count = {'GPU': 1})
# sess = tf.InteractiveSession(config=gpu_config)
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
sess = tf.InteractiveSession()
model = show_and_tell_model.ShowAndTellModel(model_config, mode="train")
model.build()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=1)
loss_stored = []
# step = epoch * train_data_set / batch_size
# with tf.device('/gpu:1'):
for i in range(conf.original_train_steps):
images, in_seqs, tar_seqs, masks = iter.next_batch(model_config.batch_size)
loss = model.run_batch(sess, images, in_seqs, tar_seqs, masks)
#every 100 steps print loss value
def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file
training_config = configuration.TrainingConfig()
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = show_and_tell_model.ShowAndTellModel(
model_config, mode="train", train_inception=FLAGS.train_inception)
model.build()
# model.images: [batch_size, 299, 299, 3] image scaled to [-1, 1]
# model.input_seqs: [batch_size, 20] numpy array of int64, padding to 20 with 0s
# Call visualize_input(model) to visualize input for debug purposes
# Set up the learning rate.
learning_rate_decay_fn = None
if FLAGS.train_inception:
learning_rate = tf.constant(
training_config.train_inception_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (
training_config.num_examples_per_epoch /
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver)
def main(unused_argv):
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file
model_config.vgg19_checkpoint_file = FLAGS.vgg19_checkpoint_file
model_config.word_embedding_file = FLAGS.word_embedding_file
training_config = configuration.TrainingConfig()
if cnn_model == 'InceptionV3':
trained_models_dir = inception_trained_models_dir
elif cnn_model == 'VGG19':
trained_models_dir = vgg_trained_models_dir
else:
print('Unknown cnn model {0}'.format(cnn_model))
exit(0)
if not tf.gfile.IsDirectory(trained_models_dir):
tf.logging.info("Creating training directory: %s", trained_models_dir)
tf.gfile.MakeDirs(trained_models_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model, train from scratch
model = show_and_tell_model.ShowAndTellModel(
model_config,
mode="train",
cnn_model = FLAGS.cnn_model,
train_cnn_model=FLAGS.train_cnn_model,
custom_word_embedding=FLAGS.custom_word_embedding)
model.build()
# Set up the learning rate.
learning_rate_decay_fn = None
if FLAGS.train_cnn_model:
learning_rate = tf.constant(
training_config.train_inception_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (training_config.num_examples_per_epoch / model_config.batch_size)
decay_steps = int(num_batches_per_epoch * training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.
tf.contrib.slim.learning.train(
train_op,
trained_models_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver)
import tensorflow as tf
import configuration
import show_and_tell_model
import read_data
model_config = configuration.ModelConfig()
training_config = configuration.TrainingConfig()
iter = read_data.DataIterator(
encoded_image_path="data/image_vgg19_fc1_feature.h5",
caption_vector_path="data/train_vector.txt")
sess = tf.InteractiveSession()
model = show_and_tell_model.ShowAndTellModel(model_config,
mode="train",
train_inception=False)
model.build()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
loss_stored = []
# step = epoch * train_data_set / batch_size
for i in range(1000000):
images, in_seqs, tar_seqs, masks = iter.next_batch(model_config.batch_size)
loss = model.run_batch(sess, images, in_seqs, tar_seqs, masks)
#every 100 steps print loss value
if (i + 1) % 100 == 0:
def build_model(self, model_config):
model = show_and_tell_model.ShowAndTellModel(
model_config, mode="inference")
model.build()
return model
def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
# Create training directory.
train_dir = FLAGS.train_dir
filename_saved_model = os.path.join(FLAGS.train_dir, 'im2txt')
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
save_flags(os.path.join(FLAGS.train_dir, 'flags.txt'))
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file
training_config = configuration.TrainingConfig()
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model (teacher-forcing mode).
model = show_and_tell_model.ShowAndTellModel(
model_config, mode="train", train_inception=FLAGS.train_inception)
model.build()
# Build the model (free-running mode).
model_free = show_and_tell_model.ShowAndTellModel(
model_config,
mode="free",
train_inception=FLAGS.train_inception,
vocab=vocab,
reuse=True)
model_free.build([
model.images, model.input_seqs, model.target_seqs, model.input_mask
])
# Build the model for validation with variable sharing
model_valid = show_and_tell_model.ShowAndTellModel(model_config,
mode="inference",
reuse=True)
model_valid.build()
# get teacher behavior
teacher_outputs, [teacher_state_c, teacher_state_h] = model.behavior
teacher_state_c = tf.expand_dims(teacher_state_c, axis=1)
teacher_state_h = tf.expand_dims(teacher_state_h, axis=1)
# get free behavior
free_outputs, [free_state_c, free_state_h] = model_free.behavior
free_state_c = tf.expand_dims(free_state_c, axis=1)
free_state_h = tf.expand_dims(free_state_h, axis=1)
# get free sentence
free_sentence = model_free.free_sentence
# prepare behavior to be LSTM's input
teacher_behavior = tf.concat(
[teacher_outputs, teacher_state_c, teacher_state_h], axis=1)
free_behavior = tf.concat([free_outputs, free_state_c, free_state_h],
axis=1)
d_lstm_cell = tf.contrib.rnn.BasicLSTMCell(model_config.num_lstm_units)
d_lstm_cell = tf.contrib.rnn.DropoutWrapper(
d_lstm_cell,
input_keep_prob=model_config.lstm_dropout_keep_prob,
output_keep_prob=model_config.lstm_dropout_keep_prob)
with tf.variable_scope("discriminator") as scope_disc:
teacher_lengths = tf.reduce_sum(model.input_mask, 1) + 2
free_lengths = tf.ones_like(teacher_lengths) * (30 + 2)
# teacher-behavior into Discriminator-LSTM
d_outputs_teacher, _ = tf.nn.dynamic_rnn(
cell=d_lstm_cell,
inputs=teacher_behavior,
sequence_length=teacher_lengths,
dtype=tf.float32,
scope=scope_disc)
# gather last outputs (deals with variable length of captions)
teacher_lengths = tf.expand_dims(teacher_lengths, 1)
batch_range = tf.expand_dims(
tf.constant(np.array(range(model_config.batch_size)),
dtype=tf.int32), 1)
gather_idx = tf.concat([batch_range, teacher_lengths - 1], axis=1)
d_last_output_teacher = tf.gather_nd(d_outputs_teacher, gather_idx)
# concat inception feature
d_teacher_concat = tf.concat(
[d_last_output_teacher, model.image_embeddings], 1)
# FC to get T/F logits
# d_logits_teacher = tf.contrib.layers.fully_connected( inputs = d_last_output_teacher, #d_teacher_concat,
d_logits_teacher = tf.contrib.layers.fully_connected(
inputs=d_teacher_concat,
num_outputs=2,
activation_fn=None,
weights_initializer=model.initializer,
scope=scope_disc)
scope_disc.reuse_variables()
# teacher with wrong image
wrong_image_embeddings = tf.concat(
[model.image_embeddings[1:], model.image_embeddings[0:1]], 0)
d_teacher_wrong_concat = tf.concat(
[d_last_output_teacher, wrong_image_embeddings], 1)
d_logits_teacher_wrong = tf.contrib.layers.fully_connected(
inputs=d_teacher_wrong_concat,
num_outputs=2,
activation_fn=None,
weights_initializer=model.initializer,
scope=scope_disc)
# free-behavior into Discriminator-LSTM
d_outputs_free, _ = tf.nn.dynamic_rnn(cell=d_lstm_cell,
inputs=free_behavior,
sequence_length=free_lengths,
dtype=tf.float32,
scope=scope_disc)
d_last_output_free = d_outputs_free[:, -1, :]
d_free_concat = tf.concat(
[d_last_output_free, model.image_embeddings], 1)
# d_logits_free = tf.contrib.layers.fully_connected( inputs = d_last_output_free, #d_free_concat,
d_logits_free = tf.contrib.layers.fully_connected(
inputs=d_free_concat,
num_outputs=2,
activation_fn=None,
weights_initializer=model.initializer,
scope=scope_disc)
d_accuracy_teacher = tf.reduce_mean(
tf.cast(tf.argmax(d_logits_teacher, axis=1), tf.float32))
d_accuracy_teacher_wrong = tf.reduce_mean(
tf.cast(1 - tf.argmax(d_logits_teacher_wrong, axis=1),
tf.float32))
d_accuracy_free = tf.reduce_mean(
tf.cast(1 - tf.argmax(d_logits_free, axis=1), tf.float32))
d_accuracy = (d_accuracy_teacher + d_accuracy_teacher_wrong +
d_accuracy_free) / 3
NLL_loss = model.total_loss
d_loss_teacher = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
name='d_loss_teacher',
logits=d_logits_teacher,
labels=tf.ones_like(d_logits_teacher)))
d_loss_teacher_wrong = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
name='d_loss_teacher_wrong',
logits=d_logits_teacher_wrong,
labels=tf.zeros_like(d_logits_teacher_wrong)))
d_loss_free = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
name='d_loss_free',
logits=d_logits_free,
labels=tf.zeros_like(d_logits_free)))
d_loss = d_loss_teacher + d_loss_teacher_wrong + d_loss_free
g_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
name='g_loss',
logits=d_logits_free,
labels=tf.ones_like(d_logits_free)))
g_and_NLL_loss = g_loss + NLL_loss
# tf.summary
summary_NLL_loss = tf.summary.scalar('NLL_loss', NLL_loss)
summary_temp_list = [
tf.summary.scalar('d_loss', d_loss),
tf.summary.scalar('d_loss_teacher', d_loss_teacher),
tf.summary.scalar('d_loss_teacher_wrong', d_loss_teacher_wrong),
tf.summary.scalar('d_loss_free', d_loss_free),
tf.summary.scalar('d_accuracy', d_accuracy),
tf.summary.scalar('d_accuracy_teacher', d_accuracy_teacher),
tf.summary.scalar('d_accuracy_teacher_wrong',
d_accuracy_teacher_wrong),
tf.summary.scalar('d_accuracy_free', d_accuracy_free)
]
summary_disc = tf.summary.merge(summary_temp_list)
summary_temp_list = [
tf.summary.scalar('g_and_NLL_loss', g_and_NLL_loss),
summary_NLL_loss,
tf.summary.scalar('g_loss', g_loss),
tf.summary.scalar('d_accuracy_free', d_accuracy_free)
]
summary_gen = tf.summary.merge(summary_temp_list)
# Set up the learning rate for training ops
learning_rate_decay_fn = None
if FLAGS.train_inception:
learning_rate = tf.constant(
training_config.train_inception_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (
training_config.num_examples_per_epoch //
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Collect trainable variables
vars_all = [
v for v in tf.trainable_variables()
if v not in model.inception_variables
]
d_vars = [v for v in vars_all if 'discr' in v.name]
g_vars = [v for v in vars_all if 'discr' not in v.name]
# Set up the training ops.
train_op_NLL = tf.contrib.layers.optimize_loss(
loss=NLL_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn,
variables=g_vars,
name='optimize_NLL_loss')
train_op_disc = tf.contrib.layers.optimize_loss(
loss=d_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn,
variables=d_vars,
name='optimize_disc_loss')
train_op_gen = tf.contrib.layers.optimize_loss(
loss=NLL_loss + g_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn,
variables=g_vars,
name='optimize_gen_loss')
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
with tf.Session() as sess:
# initialize all variables
tf.global_variables_initializer().run()
# load inception variables
model.init_fn(sess)
# Set up the training ops
nBatches = num_batches_per_epoch
summaryWriter = tf.summary.FileWriter(train_dir, sess.graph)
# start input enqueue threads
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
counter = 0
start_time = time.time()
could_load, checkpoint_counter = load(sess, saver, train_dir)
if could_load:
counter = checkpoint_counter
generator = caption_generator.CaptionGenerator(model_valid,
vocab,
beam_size=1)
try:
# for validation
f_valid_text = open(os.path.join(train_dir, 'valid.txt'), 'a')
filenames = os.listdir('testimgs')
filenames.sort()
valid_images = []
print('validation image filenames')
for filename in filenames:
with tf.gfile.GFile(os.path.join('testimgs', filename),
'r') as f:
valid_images.append(f.read())
print(filename)
# # run inference for not-trained model
# for i, valid_image in enumerate(valid_images):
# captions = generator.beam_search( sess, valid_image )
# f_valid_text.write( 'initial caption (beam) {}\n'.format( str(datetime.datetime.now().time())[:-7] ) )
# for j, caption in enumerate(captions):
# sentence = [vocab.id_to_word(w) for w in caption.sentence[1:-1]]
# sentence = " ".join(sentence)
# sentence = " {}-{}) {} (p={:.8f})".format(i+1,j+1, sentence, math.exp(caption.logprob))
# print( sentence )
# f_valid_text.write( sentence +'\n' )
# f_valid_text.flush()
# run training loop
lossnames_to_print = [
'NLL_loss', 'g_loss', 'd_loss', 'd_acc', 'g_acc'
]
val_NLL_loss = float('Inf')
val_g_loss = float('Inf')
val_d_loss = float('Inf')
val_d_acc = 0
val_g_acc = 0
for epoch in range(FLAGS.number_of_steps):
for batch_idx in range(nBatches):
counter += 1
is_disc_trained = False
is_gen_trained = False
# train NLL loss only (for im2txt sanity check)
# _, val_NLL_loss, smry, val_free_sentence, val_teacher_sentence = \
# sess.run(
# [train_op_NLL, NLL_loss, summary_NLL_loss,free_sentence, model.input_seqs] )
# summaryWriter.add_summary(smry, counter)
if val_NLL_loss > 3:
_, val_NLL_loss, smry, val_free_sentence, val_teacher_sentence = sess.run(
[
train_op_NLL, NLL_loss, summary_NLL_loss,
free_sentence, model.input_seqs
])
summaryWriter.add_summary(smry, counter)
else:
# train discriminator
_, val_d_loss, val_d_acc, smry = sess.run([
train_op_disc, d_loss, d_accuracy, summary_disc
])
summaryWriter.add_summary(smry, counter)
# train generator twice
_, val_g_loss, val_NLL_loss, val_g_acc, smry, val_free_sentence, val_teacher_sentence = sess.run(
[
train_op_gen, g_loss, NLL_loss, d_accuracy,
summary_gen, free_sentence,
model.input_seqs
])
summaryWriter.add_summary(smry, counter)
_, val_g_loss, val_NLL_loss, val_g_acc, smry, val_free_sentence, val_teacher_sentence = sess.run(
[
train_op_gen, g_loss, NLL_loss, d_accuracy,
summary_gen, free_sentence,
model.input_seqs
])
summaryWriter.add_summary(smry, counter)
if counter % FLAGS.log_every_n_steps == 0:
elapsed = time.time() - start_time
log(epoch, batch_idx, nBatches, lossnames_to_print,
[
val_NLL_loss, val_g_loss, val_d_loss,
val_d_acc, val_g_acc
], elapsed, counter)
if counter % 500 == 1 or \
(epoch==FLAGS.number_of_steps-1 and batch_idx==nBatches-1) :
saver.save(sess,
filename_saved_model,
global_step=counter)
# if True:
if (batch_idx +
1) % (nBatches //
10) == 0 or batch_idx == nBatches - 1:
# run test after every epoch
f_valid_text.write( 'count {} epoch {} batch {}/{} ({})\n'.format( \
counter, epoch, batch_idx, nBatches, str(datetime.datetime.now().time())[:-7] ) )
for i, valid_image in enumerate(valid_images):
captions = generator.beam_search(
sess, valid_image)
for j, caption in enumerate(captions):
sentence = [
vocab.id_to_word(w)
for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
sentence = " {}-{}) {} (p={:.8f})".format(
i + 1, j + 1, sentence,
math.exp(caption.logprob))
print(sentence)
f_valid_text.write(sentence + '\n')
# free sentence check
for i, caption in enumerate(val_free_sentence):
if i > 9: break
sentence = [
vocab.id_to_word(w) for w in caption
]
sentence = " ".join(sentence)
sentence = " free %d) %s" % (i + 1, sentence)
print(sentence)
f_valid_text.write(sentence + '\n')
sentence = [
vocab.id_to_word(w)
for w in val_teacher_sentence[i, 1:]
]
sentence = " ".join(sentence)
sentence = " teacher %d) %s\n" % (i + 1,
sentence)
print(sentence)
f_valid_text.write(sentence + '\n')
f_valid_text.flush()
except tf.errors.OutOfRangeError:
print('Finished training: epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
def main(unused_argv):
assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"
model_config = configuration.ModelConfig()
model_config.input_file_pattern = FLAGS.input_file_pattern
model_config.inception_checkpoint_file = FLAGS.inception_checkpoint_file
training_config = configuration.TrainingConfig()
# Create training directory.
train_dir = FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = show_and_tell_model.ShowAndTellModel(
model_config, mode="train", train_inception=FLAGS.train_inception)
model.build()
# Set up the learning rate.
learning_rate_decay_fn = None
if FLAGS.train_inception:
learning_rate = tf.constant(
training_config.train_inception_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (
training_config.num_examples_per_epoch /
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
saver = tf.train.Saver(
max_to_keep=training_config.max_checkpoints_to_keep)
# Run training.automatic initialize the threads?
summary_op = tf.summary.merge_all()
save_summaries_secs = 10
summary_writer = tf.summary.FileWriter('./log_train')
tf.contrib.slim.learning.train(train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=model.init_fn,
saver=saver,
summary_op=summary_op,
save_summaries_secs=save_summaries_secs,
summary_writer=summary_writer)
def main(unused_argv):
'''assert FLAGS.input_file_pattern, "--input_file_pattern is required"
assert FLAGS.train_dir, "--train_dir is required"'''
model_config = configuration.ModelConfig()
model_config.input_file_pattern = base_folder + "/mscoco/train-?????-of-00256" #FLAGS.input_file_pattern
model_config.inception_checkpoint_file = base_folder + "/model/inception_v3.ckpt" #FLAGS.inception_checkpoint_file
training_config = configuration.TrainingConfig()
# Create training directory.
train_dir = base_folder + "/model/3m_wa_n_all/train" #FLAGS.train_dir
if not tf.gfile.IsDirectory(train_dir):
tf.logging.info("Creating training directory: %s", train_dir)
tf.gfile.MakeDirs(train_dir)
# Build the TensorFlow graph.
g = tf.Graph()
with g.as_default():
# Build the model.
model = show_and_tell_model.ShowAndTellModel(
model_config, mode="train", train_inception=FLAGS.train_inception)
model.build()
# Set up the learning rate.
learning_rate_decay_fn = None
if FLAGS.train_inception:
learning_rate = tf.constant(
training_config.train_inception_learning_rate)
else:
learning_rate = tf.constant(training_config.initial_learning_rate)
if training_config.learning_rate_decay_factor > 0:
num_batches_per_epoch = (
training_config.num_examples_per_epoch /
model_config.batch_size)
decay_steps = int(num_batches_per_epoch *
training_config.num_epochs_per_decay)
def _learning_rate_decay_fn(learning_rate, global_step):
return tf.train.exponential_decay(
learning_rate,
global_step,
decay_steps=decay_steps,
decay_rate=training_config.learning_rate_decay_factor,
staircase=True)
learning_rate_decay_fn = _learning_rate_decay_fn
# Set up the training ops.
train_op = tf.contrib.layers.optimize_loss(
loss=model.total_loss,
global_step=model.global_step,
learning_rate=learning_rate,
optimizer=training_config.optimizer,
clip_gradients=training_config.clip_gradients,
learning_rate_decay_fn=learning_rate_decay_fn)
# Set up the Saver for saving and restoring model checkpoints.
var_list = tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES) #, scope = the_scope)
print("VL ", var_list)
saver = tf.train.Saver(
var_list=var_list,
max_to_keep=training_config.max_checkpoints_to_keep,
save_relative_paths=True)
var_list2 = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='(?!attention_var)')
print("test", var_list2)
saver2 = tf.train.Saver(var_list=var_list2)
attVar = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="attention_var")
local_init_op = tf.variables_initializer(attVar)
def init_fn_2(sess):
tf.logging.info("Restoring the original weight %s",
model_config.inception_checkpoint_file)
checkpoint_path2 = model_config.base_folder + "/model/3m/train/model.ckpt-3003677"
saver2.restore(sess, checkpoint_path2)
sess.run(local_init_op)
config = tf.ConfigProto()
config.gpu_options.visible_device_list = "0"
gpu_fract = .75
config.gpu_options.per_process_gpu_memory_fraction = gpu_fract
# Run training.
tf.contrib.slim.learning.train(
train_op,
train_dir,
log_every_n_steps=FLAGS.log_every_n_steps,
graph=g,
global_step=model.global_step,
number_of_steps=FLAGS.number_of_steps,
init_fn=init_fn_2,
saver=saver,
#local_init_op = local_init_op,
session_config=config)