def lstm_net(text_seq_batch, num_vocab, embed_dim, lstm_dim): # embedding matrix with each row containing the embedding vector of a word # this has to be done on CPU currently with tf.variable_scope('word_embedding'), tf.device("/cpu:0"): embedding_mat = tf.get_variable("embedding", [num_vocab, embed_dim]) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) lstm_top = lstm('lstm_lang', embedded_seq, None, output_dim=lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=False, concat_output=False)[-1] return lstm_top
def lstm_encoder(text_seq_batch, name, num_vocab, embed_dim, lstm_dim, apply_dropout, reuse=None): with tf.variable_scope(name, reuse=reuse): embedding_mat = tf.get_variable("embedding_mat", [num_vocab, embed_dim]) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) # Take the output at the final timestep of LSTM. lstm_top = lstm("lstm_lang", embedded_seq, None, output_dim=lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=apply_dropout, concat_output=False)[-1] return lstm_top
def lstm_net(text_seq_batch, num_vocab, embed_dim, lstm_dim): # embedding matrix with each row containing the embedding vector of a word # this has to be done on CPU currently with tf.variable_scope('word_embedding'), tf.device("/cpu:0"): embedding_mat = tf.get_variable("embedding", [num_vocab, embed_dim]) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) lstm_top = lstm('lstm_lang', embedded_seq, None, output_dim=lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=False, concat_output=False)[-1] return lstm_top
def lstm_net_glove(text_seq_batch, embedding, lstm_dim): # Initialize embedding layer with tf.variable_scope('word_embedding'), tf.device("/cpu:0"): embedding_mat = tf.get_variable("embedding", initializer=embedding, trainable=False) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) lstm_top = lstm('lstm_lang', embedded_seq, None, output_dim=lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=False, concat_output=False)[-1] return lstm_top
def model_structure(self, sen_data, vis_data, batch_size, is_train, dropout=None): if dropout == None: dropout = self.dropout text_seq_batch = tf.transpose(sen_data, [1, 0]) # input data is [num_steps, batch_size] with tf.variable_scope('word_embedding'), tf.device("/cpu:0"): embedding_mat = tf.get_variable("embedding", [self.vocab_size, self.lstm_dim], tf.float32, initializer=tf.contrib.layers.xavier_initializer(uniform=True)) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) # we encode phrase based on the last step of hidden states _, states = lstm('lstm_lang', embedded_seq, None, output_dim=self.lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=False,concat_output=False, initializer=tf.random_uniform_initializer(minval=-0.08, maxval=0.08)) # batch normalization for visual and language part sen_raw = states[-1].h vis_raw = tf.reshape(vis_data, [self.batch_size*self.num_prop, self.img_feat_size]) sen_bn = bn(sen_raw, is_train, "SEN_BN", 0.9) vis_bn = bn(vis_raw, is_train, "VIS_BN", 0.9) sen_output = tf.reshape(sen_bn, [self.batch_size, 1, 1, self.lstm_dim]) vis_output = tf.reshape(vis_bn, [self.batch_size, self.num_prop, 1, self.img_feat_size]) sen_tile = tf.tile(sen_output, [1, self.num_prop, 1, 1]) feat_concat = tf.concat([sen_tile, vis_output], 3) feat_proj_init = msr_init([1, 1, self.lstm_dim+self.img_feat_size, self.hidden_size]) feat_proj = conv("feat_proj", feat_concat, 1, 1, self.hidden_size, weights_initializer=feat_proj_init) feat_relu = tf.nn.relu(feat_proj) att_conv_init = msr_init([1, 1, self.hidden_size, 1]) att_conv = conv("att_conv", feat_relu, 1, 1, 1, weights_initializer=att_conv_init) att_scores = tf.reshape(att_conv, [self.batch_size, self.num_prop]) return att_scores
def model_structure(self, sen_data, enc_data, dec_data, msk_data, vis_data, batch_size, is_train, dropout=None): def set_drop_test(): return tf.cast(1.0, tf.float32) def set_drop_train(): return tf.cast(self.dropout, tf.float32) dropout = tf.cond(is_train, set_drop_train, set_drop_test) seq_length = tf.reduce_sum(msk_data, 1) text_seq_batch = sen_data with tf.variable_scope('word_embedding'), tf.device("/cpu:0"): embedding_mat = tf.get_variable( "embedding", [self.vocab_size, self.lstm_dim], tf.float32, initializer=tf.contrib.layers.xavier_initializer(uniform=True)) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_seq = tf.nn.embedding_lookup(embedding_mat, text_seq_batch) # we encode phrase based on the last step of hidden states outputs, states = lstm('enc_lstm', embedded_seq, None, seq_length, output_dim=self.lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=True, keep_prob=dropout, concat_output=False, initializer=tf.random_uniform_initializer( minval=-0.08, maxval=0.08)) sen_raw = states[-1].h sen_raw = tf.nn.l2_normalize(sen_raw, dim=1) # print sen_raw.get_shape() vis_raw = tf.reshape( vis_data, [self.batch_size * self.num_prop, self.img_feat_size]) sen_output = tf.reshape(sen_raw, [self.batch_size, 1, 1, self.lstm_dim]) vis_output = tf.reshape( vis_raw, [self.batch_size, self.num_prop, 1, self.img_feat_size]) sen_tile = tf.tile(sen_output, [1, self.num_prop, 1, 1]) feat_concat = tf.concat([sen_tile, vis_output], 3) feat_proj_init = msr_init( [1, 1, self.lstm_dim + self.img_feat_size, self.hidden_size]) feat_proj = conv("feat_proj", feat_concat, 1, 1, self.hidden_size, weights_initializer=feat_proj_init) feat_relu = tf.nn.relu(feat_proj) att_conv_init = msr_init([1, 1, self.hidden_size, 1]) att_conv = conv("att_conv", feat_relu, 1, 1, 1, weights_initializer=att_conv_init) #Generate the visual attention feature att_scores_t = tf.reshape(att_conv, [self.batch_size, self.num_prop]) # att_prob = tf.nn.softmax(att_scores_t) att_prob = tf.nn.relu(att_scores_t) att_scores = tf.reshape(att_prob, [self.batch_size, self.num_prop, 1]) vis_att_feat = tf.reduce_sum( tf.multiply(vis_data, tf.tile(att_scores, [1, 1, self.img_feat_size])), 1) vis_att_featFC = fc_relu( "vis_enc", vis_att_feat, self.lstm_dim, weights_initializer=tf.random_uniform_initializer(minval=-0.002, maxval=0.002)) vis_att_tile = tf.reshape(vis_att_featFC, [self.batch_size, 1, self.lstm_dim]) text_enc_batch = enc_data # embedded_enc: batch_size x phrase_len x lstm_dim with tf.variable_scope('enc_embedding'), tf.device("/cpu:0"): embedding_enc = tf.get_variable( "embedding", [self.vocab_size, self.lstm_dim], tf.float32, initializer=tf.contrib.layers.xavier_initializer(uniform=True)) # text_seq has shape [T, N] and embedded_seq has shape [T, N, D]. embedded_enc = tf.nn.embedding_lookup(embedding_enc, text_enc_batch) # dec_vis_embed = batch_size x phrase_len x (2*lstm_dim) dec_vis_embed = tf.concat([ embedded_enc, tf.concat([ vis_att_tile, tf.zeros((self.batch_size, self.phrase_len - 1, self.lstm_dim)) ], 1) ], 2) # dec_outputs: batch_size x phrase_len x lstm_dim dec_outs, _ = lstm('dec_lstm', dec_vis_embed, None, seq_length, output_dim=self.lstm_dim, num_layers=1, forget_bias=1.0, apply_dropout=True, keep_prob=dropout, concat_output=True, initializer=tf.random_uniform_initializer( minval=-0.08, maxval=0.08)) dec_outs = tf.reshape( dec_outs, [self.batch_size * self.phrase_len, self.lstm_dim]) # dec_logits: (batch_size*phrase_len) x vocab_size dec_logits = fc( 'dec_logits', dec_outs, self.vocab_size, weights_initializer=tf.contrib.layers.xavier_initializer( uniform=True)) return att_scores_t, dec_logits, vis_data