def build_evaluate_image_word_graph(self, image_feature):
with tf.variable_scope(self.scope):
image_feature = self.lforward(image_feature)
#no need for embedding lookup
word_feature = self.forward_word_feature()
score = dot(image_feature, word_feature)
return score
def build_rsim_graph(self, ltext, rtext):
with tf.variable_scope(self.scope):
with tf.variable_scope('encode') as encode_scope:
ltext_feature = self.rforward(ltext)
rtext_feature = self.rforward(rtext)
score = dot(ltext_feature, rtext_feature)
return score
def build_fixed_text_feature_graph(self, text_feature_npy):
"""
text features directly load to graph, @NOTICE text_feature_npy all vector must of same length
used in evaluate.py for both fixed text and fixed words
@FIXME dump text feature should change api
"""
with tf.variable_scope(self.scope):
image_feature = self.forward_image_feature(self.image_feature_feed)
text_feature = melt.load_constant(text_feature_npy, self.sess)
score = dot(image_feature, text_feature)
return score
def build_graph(self, ltext, rtext):
with tf.variable_scope(self.scope):
ltext_feature = self.lforward(ltext)
#make to cpu ? for mem issue of cnn? if not perf hurt much?
#reidctor.bulk_predict duration: 125.557517052
#cpu is slow evaluate_scores duration: 135.078355074
#if self.encoder_type != 'cnn':
rtext_feature = self.rforward(rtext)
#else:
#with tf.device('/cpu:0'):
#rtext_feature = self.rforward(rtext)
score = dot(ltext_feature, rtext_feature)
return score
def _build_words_importance_graph(self, text, forward_fn):
with tf.variable_scope(self.scope):
text2 = text
# TODO hack here for rnn with start pad <S> as 2!
if FLAGS.encode_start_mark:
start_pad = tf.zeros([1, 1], dtype=text.dtype) + 2
text2 = tf.concat([start_pad, text], 1)
if FLAGS.encode_end_mark:
end_pad = tf.zeros([1, 1], dtype=text.dtype) + 1
text2 = tf.concat([text2, end_pad], 1)
# text batch_size must be 1! currently [1, seq_len] -> [seq_len, 1]
words = tf.transpose(text2, [1, 0])
#[seq_len, 1] -> [seq_len, emb_dim]
word_feature = forward_fn(words)
#[batch_size, seq_len] -> [batch_size, emb_dim] [1, emb_dim]
text_feature = forward_fn(text)
#[1, seq_len]
score = dot(text_feature, word_feature)
return score
def build_lsim_graph(self, ltext, rtext):
with tf.variable_scope(self.scope):
ltext_feature = self.lforward(ltext)
rtext_feature = self.lforward(rtext)
score = dot(ltext_feature, rtext_feature)
return score
def build_text_words_sim_graph(self, text_max_words=TEXT_MAX_WORDS):
with tf.variable_scope(self.scope):
text_feature = self.forward_text(self.get_text_feed(text_max_words))
word_feature = self.forward_word_feature()
score = dot(text_feature, word_feature)
return score
def build_image_words_sim_graph(self):
with tf.variable_scope(self.scope):
image_feature = self.forward_image_feature(self.get_image_feature_feed())
word_feature = self.forward_word_feature()
score = dot(image_feature, word_feature)
return score
def build_textsim_graph(self, text, text2):
with tf.variable_scope(self.scope):
text_feature = self.forward_text(text)
text_feature2 = self.forward_text(text2)
score = dot(text_feature, text_feature2)
return score
def build_graph(self, image_feature, text):
with tf.variable_scope(self.scope):
image_feature = self.forward_image_feature(image_feature)
text_feature = self.forward_text(text)
score = dot(image_feature, text_feature)
return score
