def build_graph(self, image_feature, text, neg_text=None):
image_emb = tf.nn.xw_plus_b(image_feature, self.encode_img_W,
self.encode_img_b)
pos_loss = self.compute_seq_loss(image_emb, text)
if neg_text is not None and FLAGS.use_neg:
neg_losses = []
num_negs = neg_text.get_shape()[1]
for i in xrange(num_negs):
tf.get_variable_scope().reuse_variables()
neg_text_i = neg_text[:, i, :]
neg_loss = self.compute_seq_loss(image_emb, neg_text_i)
neg_losses.append(neg_loss)
neg_losses = tf.concat(1, neg_losses)
#[batch_size, num_neg_text] @NOTICE input should be pos_score and
#neg_score which is the larger the better
loss = melt.hinge_loss(-pos_loss, -neg_losses, FLAGS.margin)
scores = tf.concat(1, [pos_loss, neg_losses])
else:
#loss = tf.reduce_mean(pos_loss)
#use melt.reduce_mean if input is not [batch_size, 1] but [batch_size * num_steps, 1]
loss = melt.reduce_mean(pos_loss)
scores = tf.concat(1, [pos_loss, pos_loss])
self.scores = scores
return loss
def build_graph(self,
image_feature,
text,
neg_text,
lookup_negs_once=False):
"""
Args:
image_feature: [batch_size, IMAGE_FEATURE_LEN]
text: [batch_size, MAX_TEXT_LEN]
neg_text: [batch_size, num_negs, MAXT_TEXT_LEN]
"""
with tf.variable_scope("image_text_sim"):
#-------------get image feature
#[batch_size, hidden_size] <= [batch_size, IMAGE_FEATURE_LEN]
normed_image_feature = self.forward_image_feature(image_feature)
#--------------get image text sim as pos score
#[batch_size, emb_dim] -> [batch_size, text_MAX_WORDS, emb_dim] -> [batch_size, emb_dim]
text_feature = self.gen_text_feature(text, self.emb)
tf.add_to_collection('text_feature', text_feature)
pos_score = self.compute_image_text_sim(normed_image_feature,
text_feature)
#--------------get image neg texts sim as neg scores
#[batch_size, num_negs, text_MAX_WORDS, emb_dim] -> [batch_size, num_negs, emb_dim]
tf.get_variable_scope().reuse_variables()
if lookup_negs_once:
neg_text_feature = self.gen_text_feature(neg_text, self.emb)
neg_scores_list = []
num_negs = neg_text.get_shape()[1]
for i in xrange(num_negs):
if lookup_negs_once:
neg_text_feature_i = neg_text_feature[:, i, :]
else:
neg_text_feature_i = self.gen_text_feature(
neg_text[:, i, :], self.emb)
neg_scores_i = self.compute_image_text_sim(
normed_image_feature, neg_text_feature_i)
neg_scores_list.append(neg_scores_i)
#[batch_size, num_negs]
neg_scores = tf.concat(neg_scores_list, 1)
#---------------rank loss
#[batch_size, 1 + num_negs]
scores = tf.concat([pos_score, neg_scores], 1)
#may be turn to prob is and show is
#probs = tf.sigmoid(scores)
if FLAGS.rank_loss == 'hinge':
loss = melt.hinge_loss(pos_score, neg_scores, FLAGS.margin)
elif FLAGS.rank_loss == 'cross':
loss = melt.cross_entropy_loss(scores, num_negs)
else:
loss = melt.hinge_cross_loss(pos_score, neg_scores)
tf.add_to_collection('scores', scores)
return loss
def build_graph(self,
image_feature,
text,
neg_text,
lookup_negs_once=True):
"""
Args:
image_feature: [batch_size, IMAGE_FEATURE_LEN]
text: [batch_size, MAX_TEXT_LEN]
neg_text: [batch_size, num_negs, MAXT_TEXT_LEN]
"""
with tf.variable_scope("image_text_sim"):
#-------------get image feature
#[batch_size, hidden_size] <= [batch_size, IMAGE_FEATURE_LEN]
normed_image_feature = self.forward_image_feature(image_feature)
#--------------get image text sim as pos score
#[batch_size, emb_dim] -> [batch_size, text_MAX_WORDS, emb_dim] -> [batch_size, emb_dim]
text_feature = self.gen_text_feature(text)
pos_score = self.compute_image_text_sim(normed_image_feature,
text_feature)
#--------------get image neg texts sim as neg scores
#[batch_size, num_negs, text_MAX_WORDS, emb_dim] -> [batch_size, num_negs, emb_dim]
tf.get_variable_scope().reuse_variables()
if lookup_negs_once:
neg_text_feature = self.gen_text_feature(neg_text)
neg_scores_list = []
num_negs = neg_text.get_shape()[1]
for i in xrange(num_negs):
if lookup_negs_once:
neg_text_feature_i = neg_text_feature[:, i, :]
else:
neg_text_feature_i = self.gen_text_feature(neg_text[:,
i, :])
neg_scores_i = self.compute_image_text_sim(
normed_image_feature, neg_text_feature_i)
neg_scores_list.append(neg_scores_i)
#[batch_size, num_negs]
neg_scores = tf.concat(1, neg_scores_list)
#---------------rank loss
#[batch_size, 1 + num_negs]
scores = tf.concat(1, [pos_score, neg_scores])
loss = melt.hinge_loss(pos_score, neg_scores, FLAGS.margin)
self.scores = scores
return loss
def build_graph(self,
image_feature,
text,
neg_text=None,
exact_loss=False):
image_emb = self.build_image_embeddings(image_feature)
pos_loss = self.decoder.sequence_loss(text,
input=image_emb,
exact_loss=exact_loss)
loss = None
scores = None
if neg_text is not None and (FLAGS.use_neg or FLAGS.show_neg):
neg_losses = []
num_negs = neg_text.get_shape()[1]
for i in xrange(num_negs):
tf.get_variable_scope().reuse_variables()
neg_text_i = neg_text[:, i, :]
neg_loss = self.decoder.sequence_loss(image_emb,
neg_text_i,
exact_loss=exact_loss)
neg_losses.append(neg_loss)
neg_losses = tf.concat(neg_losses, 1)
if FLAGS.use_neg:
#neg_losses [batch_size, num_neg_text]
loss = melt.hinge_loss(-pos_loss, -neg_losses, FLAGS.margin)
scores = tf.concat([pos_loss, neg_losses], 1)
if loss is None:
if not self.is_predict:
loss = tf.reduce_mean(pos_loss)
else:
loss = pos_loss
if scores is None:
if neg_text is not None:
scores = tf.concat([pos_loss, pos_loss], 1)
else:
scores = pos_loss
if not self.is_training and not self.is_predict: #evaluate mode
tf.add_to_collection('scores', scores)
return loss