def build_evaluate_image_word_graph(self, image_feature):
with tf.variable_scope("image_text_sim"):
image_feature = self.forward_image_feature(image_feature)
#no need for embedding lookup
word_feature = self.forward_word_feature()
score = melt.cosine(image_feature, word_feature, nonorm=True)
return score
def build_text_words_emb_sim_graph(self, text_max_words=TEXT_MAX_WORDS):
with tf.variable_scope(self.scope):
text_feature = self.gen_text_feature(
self.get_text_feed(text_max_words), self.emb)
word_feature = self.gen_word_feature()
score = melt.cosine(text_feature, word_feature)
return score
def dot(x, y):
if FLAGS.dist_normalize or get_dist_type(FLAGS.loss) == DistType.cosine:
#has already normalized
return melt.dot(x, y)
else:
#TODO ...
#return -tf.sqrt(pairwise_distance(x, y))
##not normalized before but for contrastive will it be better to use pairwise_distance like in pytorch ?
return melt.cosine(x, y)
def build_text_emb_sim_graph(self, text, text2):
"""
for bow is emb sim
for rnn is sim after rnn encoding
"""
with tf.variable_scope(self.scope):
text_feature = self.gen_text_feature(text, self.emb)
text_feature2 = self.gen_text_feature(text2, self.emb)
score = melt.cosine(text_feature, text_feature2)
return score
def build_fixed_text_graph(self, text_feature_npy):
"""
text features directly load to graph,
used in evaluate.py for both fixed text and fixed words
"""
with tf.variable_scope("image_text_sim"):
image_feature = self.forward_image_feature(self.image_feature_place)
text_feature = melt.constant(self.sess, text_feature_npy)
score = melt.cosine(image_feature, text_feature, nonorm=True)
return score
def build_embsim_graph(self, ltext, rtext):
"""
for bow is emb sim
for rnn is sim after rnn encoding
"""
#TODO imporve speed by concat pos and all negs to one batch
with tf.variable_scope(self.scope):
ltext_feature = self.encode(ltext)
rtext_feature = self.encode(rtext)
score = melt.cosine(ltext_feature, rtext_feature)
return score
def build_fixed_text_feature_graph(self, text_feature_npy):
"""
text features directly load to graph, @NOTICE text_feature_npy all vector must of same length
used in evaluate.py for both fixed text and fixed words
@FIXME dump text feature should change api
"""
with tf.variable_scope("image_text_sim"):
image_feature = self.forward_image_feature(self.image_feature_feed)
text_feature = melt.load_constant(self.sess, text_feature_npy)
score = melt.cosine(image_feature, text_feature, nonorm=True)
return score
def dot(x, y):
if FLAGS.concat_sim:
return concat_sim_score(x, y)
if FLAGS.dist_normalize or get_dist_type(FLAGS.loss) == DistType.cosine:
#has already normalized
return melt.dot(x, y)
else:
#TODO ...
#return -tf.sqrt(pairwise_distance(x, y))
##not normalized before but for contrastive will it be better to use pairwise_distance like in pytorch ?
#distance_L1 = tf.reduce_sum(tf.abs(tf.subtract(vocab, tf.expand_dims(batch,1))), axis=2)
return melt.cosine(x, y)
def build_textsim_graph(self, text, text2):
with tf.variable_scope("image_text_sim"):
text_feature = self.forward_text(text)
text_feature2 = self.forward_text(text2)
score = melt.cosine(text_feature, text_feature2, nonorm=True)
return score
def build_graph(self, image_feature, text):
with tf.variable_scope("image_text_sim"):
image_feature = self.forward_image_feature(image_feature)
text_feature = self.forward_text(text)
score = melt.cosine(image_feature, text_feature, nonorm=True)
return score
