def ian(inputs,
sen_len,
target,
sen_len_tr,
keep_prob1,
keep_prob2,
_id='all'):
cell = tf.contrib.rnn.LSTMCell
# sentence hidden
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
hiddens_s = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sen' + _id, 'all')
pool_s = reduce_mean_with_len(hiddens_s, sen_len)
# target hidden
target = tf.nn.dropout(target, keep_prob=keep_prob1)
hiddens_t = bi_dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr,
FLAGS.max_sentence_len, 't' + _id, 'all')
pool_t = reduce_mean_with_len(hiddens_t, sen_len_tr)
# attention sentence
att_s = bilinear_attention_layer(hiddens_s, pool_t, sen_len,
2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 'sen')
outputs_s = tf.squeeze(tf.matmul(att_s, hiddens_s))
# attention target
att_t = bilinear_attention_layer(hiddens_t, pool_s, sen_len_tr,
2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 't')
outputs_t = tf.squeeze(tf.matmul(att_t, hiddens_t))
outputs = tf.concat([outputs_s, outputs_t], 1)
prob = softmax_layer(outputs, 4 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return prob, att_s, att_t
def hn_inter_att(inputs_o, sen_len_o, doc_len_o, inputs_r, sen_len_r, doc_len_r, keep_prob1, keep_prob2, _id='0'):
print 'I am hn-inter-att!'
cell = tf.contrib.rnn.LSTMCell
# Original Part
inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1)
sen_len_o = tf.reshape(sen_len_o, [-1])
hiddens_sen_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o, FLAGS.max_sentence_len, 'sentence_o', 'all')
alpha_sen_o = mlp_attention_layer(hiddens_sen_o, sen_len_o, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sentence_o')
outputs_sen_o = tf.reshape(tf.matmul(alpha_sen_o, hiddens_sen_o), [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden])
hiddens_doc_o = bi_dynamic_rnn(cell, outputs_sen_o, FLAGS.n_hidden, doc_len_o, FLAGS.max_doc_len, 'doc_o', 'all')
# Reverse Part
inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1)
sen_len_r = tf.reshape(sen_len_r, [-1])
hiddens_sen_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r, FLAGS.max_sentence_len, 'sentence_r', 'all')
alpha_sen_r = mlp_attention_layer(hiddens_sen_r, sen_len_r, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sentence_r')
outputs_sen_r = tf.reshape(tf.matmul(alpha_sen_r, hiddens_sen_r), [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden])
hiddens_doc_r = bi_dynamic_rnn(cell, outputs_sen_r, FLAGS.n_hidden, doc_len_r, FLAGS.max_doc_len, 'doc_r', 'all')
# Combined Part
hiddens_doc = tf.concat([hiddens_doc_o, hiddens_doc_r], 2) # batch_size * max_doc_len * 4n_hidden
alpha_doc = mlp_attention_layer(hiddens_doc, doc_len_o, 4 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'doc')
outputs_doc = tf.reshape(tf.matmul(alpha_doc, hiddens_doc), [-1, 4 * FLAGS.n_hidden])
logits = softmax_layer(outputs_doc, 4 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return logits
def hn_att(inputs, sen_len, doc_len, keep_prob1, keep_prob2, _id='1'):
print 'I am HAN!'
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
cell = tf.contrib.rnn.LSTMCell
sen_len = tf.reshape(sen_len, [-1])
hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence' + _id,
'all')
alpha_sen = mlp_attention_layer(hiddens_sen, sen_len, 2 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base,
'sentence' + _id)
outputs_sen = tf.reshape(tf.matmul(alpha_sen, hiddens_sen),
[-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden])
# sen_len = tf.reshape(sen_len, [-1])
# sen_len = tf.cast(sen_len, tf.float32)
# alpha = 1.0 - tf.cast(sen_len / (tf.reduce_sum(sen_len, 1, keep_dims=True) + 1), tf.float32)
# alpha = tf.reshape(softmax_with_len(alpha, doc_len, FLAGS.max_doc_len), [-1, FLAGS.max_doc_len, 1])
# outputs_sen *= alpha
hiddens_doc = bi_dynamic_rnn(cell, outputs_sen, FLAGS.n_hidden, doc_len,
FLAGS.max_doc_len, 'doc' + _id, 'all')
alpha_doc = mlp_attention_layer(hiddens_doc, doc_len, 2 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base,
'doc' + _id)
outputs_doc = tf.reshape(tf.matmul(alpha_doc, hiddens_doc),
[-1, 2 * FLAGS.n_hidden])
return outputs_doc
def hn(inputs, sen_len, doc_len, keep_prob1, keep_prob2, _id='1'):
print 'I am HN!'
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
cell = tf.contrib.rnn.LSTMCell
sen_len = tf.reshape(sen_len, [-1])
hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence' + _id,
FLAGS.t1)
hiddens_sen = tf.reshape(hiddens_sen,
[-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden])
hidden_doc = bi_dynamic_rnn(cell, hiddens_sen, FLAGS.n_hidden, doc_len,
FLAGS.max_doc_len, 'doc' + _id, FLAGS.t2)
return hidden_doc
def dual_sen(inputs_o, sen_len_o, inputs_r, sen_len_r, keep_prob1, keep_prob2):
print 'I am dual-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1)
hiddens_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o,
FLAGS.max_sentence_len, 'sentence_o', 'last')
inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1)
hiddens_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r,
FLAGS.max_sentence_len, 'sentence_r', 'last')
hiddens = tf.concat([hiddens_o, hiddens_r], 1)
logits = softmax_layer(hiddens, 4 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return logits
def hn(inputs, sen_len, doc_len, keep_prob1, keep_prob2, id_=1):
print 'I am HN!'
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
cell = tf.contrib.rnn.LSTMCell
sen_len = tf.reshape(sen_len, [-1])
hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence' + str(id_),
FLAGS.t1)
hiddens_sen = tf.reshape(hiddens_sen,
[-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden])
hidden_doc = bi_dynamic_rnn(cell, hiddens_sen, FLAGS.n_hidden, doc_len,
FLAGS.max_doc_len, 'doc' + str(id_), FLAGS.t2)
return softmax_layer(hidden_doc, 2 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
def lstm(inputs, sen_len, keep_prob1, keep_prob2, _id='1'):
print 'I am dual-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, _id, 'last')
return hiddens
def lstm_sen(inputs, sen_len, keep_prob1, keep_prob2):
print 'I am lstm-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence_o', 'last')
logits = softmax_layer(hiddens, 2 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return logits
def bi_rnn(inputs, sen_len, keep_prob1, keep_prob2, _id='1'):
print 'I am bi-rnn.'
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
cell = tf.contrib.rnn.LSTMCell
hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence' + _id,
FLAGS.t1)
return softmax_layer(hiddens, 2 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class, _id)
def dual_att_sen(inputs_o, sen_len_o, inputs_r, sen_len_r, keep_prob1,
keep_prob2):
print 'I am dual-att-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1)
hiddens_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o,
FLAGS.max_sentence_len, 'sentence_o', 'all')
inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1)
hiddens_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r,
FLAGS.max_sentence_len, 'sentence_r', 'all')
hiddens = tf.concat([hiddens_o, hiddens_r], 2)
alpha = mlp_attention_layer(hiddens, sen_len_o, 4 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base, 'sen')
outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 4 * FLAGS.n_hidden])
logits = softmax_layer(outputs, 4 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return logits
def lcr_rot(input_fw, input_bw, sen_len_fw, sen_len_bw, target, sen_len_tr, keep_prob1, keep_prob2, _id='all'):
print('I am lcr_rot.')
cell = tf.contrib.rnn.LSTMCell
# left hidden
input_fw = tf.nn.dropout(input_fw, keep_prob=keep_prob1)
hiddens_l = bi_dynamic_rnn(cell, input_fw, FLAGS.n_hidden, sen_len_fw, FLAGS.max_sentence_len, 'l' + _id, 'all')
# hiddens_l = dynamic_rnn(cell, input_fw, FLAGS.n_hidden, sen_len_fw, FLAGS.max_sentence_len, 'l' + _id, 'all')
pool_l = reduce_mean_with_len(hiddens_l, sen_len_fw)
# right hidden
input_bw = tf.nn.dropout(input_bw, keep_prob=keep_prob1)
hiddens_r = bi_dynamic_rnn(cell, input_bw, FLAGS.n_hidden, sen_len_bw, FLAGS.max_sentence_len, 'r' + _id, 'all')
# hiddens_r = dynamic_rnn(cell, input_bw, FLAGS.n_hidden, sen_len_bw, FLAGS.max_sentence_len, 'r' + _id, 'all')
pool_r = reduce_mean_with_len(hiddens_r, sen_len_bw)
# target hidden
target = tf.nn.dropout(target, keep_prob=keep_prob1)
hiddens_t = bi_dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr, FLAGS.max_sentence_len, 't' + _id, 'all')
# hiddens_t = dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr, FLAGS.max_sentence_len, 't' + _id, 'all')
pool_t = reduce_mean_with_len(hiddens_t, sen_len_tr)
# pool_t = tf.concat(1, [target, target])
# attention target
att_t_l = bilinear_attention_layer(hiddens_t, pool_l, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 'tl')
# att_t_l = bilinear_attention_layer(hiddens_t, pool_l, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tl')
outputs_t_l = tf.squeeze(tf.matmul(att_t_l, hiddens_t))
att_t_r = bilinear_attention_layer(hiddens_t, pool_r, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 'tr')
# att_t_r = bilinear_attention_layer(hiddens_t, pool_r, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tr')
outputs_t_r = tf.squeeze(tf.matmul(att_t_r, hiddens_t))
# attention left
att_l = bilinear_attention_layer(hiddens_l, outputs_t_l, sen_len_fw, 2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 'l')
outputs_l = tf.squeeze(tf.matmul(att_l, hiddens_l))
# # attention right
att_r = bilinear_attention_layer(hiddens_r, outputs_t_r, sen_len_bw, 2 * FLAGS.n_hidden, FLAGS.l2_reg,
FLAGS.random_base, 'r')
outputs_r = tf.squeeze(tf.matmul(att_r, hiddens_r))
outputs = tf.concat([outputs_l, outputs_r, outputs_t_l, outputs_t_r], 1)
# outputs = (outputs_l + outputs_r + outputs_t_l + outputs_t_r) / 4.0
prob = softmax_layer(outputs, 8 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return prob, att_l, att_r, att_t_l, att_t_r
def add_bilstm_layer(self, inputs, seq_len, max_len, scope_name='bilstm'):
inputs = tf.nn.dropout(inputs, keep_prob=self.keep_prob1)
cell = tf.contrib.rnn.LSTMCell
seq_len = tf.reshape(seq_len, [-1])
hiddens = bi_dynamic_rnn(cell, inputs, self.config.n_hidden, seq_len,
max_len, scope_name, 'all')
alpha = mlp_attention_layer(hiddens, seq_len, 2 * self.config.n_hidden,
self.config.l2_reg,
self.config.random_base, scope_name)
outputs = tf.squeeze(tf.matmul(alpha, hiddens))
return outputs
def bi_rnn_att(inputs, sen_len, keep_prob1, keep_prob2, _id='1'):
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
cell = tf.contrib.rnn.LSTMCell
hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, _id, 'all')
alpha_sen = mlp_attention_layer(hiddens_sen, sen_len, 2 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base, _id)
outputs_sen = tf.squeeze(tf.matmul(alpha_sen, hiddens_sen))
return softmax_layer(outputs_sen, 2 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class, _id)
def lstm_att(inputs, sen_len, keep_prob1, keep_prob2, _id='1'):
print 'I am dual-att-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, _id, 'all')
alpha = mlp_attention_layer(hiddens, sen_len, 2 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base, _id)
outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 2 * FLAGS.n_hidden])
return outputs
def create_model(self, inputs):
inputs = tf.reshape(
inputs,
[-1, self.config.max_sentence_len, self.config.embedding_dim])
inputs = tf.nn.dropout(inputs, keep_prob=self.keep_prob1)
cell = tf.contrib.rnn.LSTMCell
outputs_dim = 2 * self.config.n_hidden
outputs_sen = bi_dynamic_rnn(cell, inputs, self.config.n_hidden,
tf.reshape(self.sen_len, [-1]),
self.config.max_sentence_len, 'sen',
'last')
outputs_sen = tf.reshape(outputs_sen,
[-1, self.config.max_doc_len, outputs_dim])
outputs_doc = bi_dynamic_rnn(cell, outputs_sen, self.config.n_hidden,
self.doc_len, self.config.max_doc_len,
'doc', 'last')
doc_logits = softmax_layer(outputs_doc, outputs_dim,
self.config.random_base, self.keep_prob2,
self.config.l2_reg, self.config.n_class,
'doc')
return doc_logits
def lstm_att_sen(inputs, sen_len, keep_prob1, keep_prob2):
print 'I am lstm-att-sen!'
cell = tf.contrib.rnn.LSTMCell
inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1)
hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len,
FLAGS.max_sentence_len, 'sentence_o', 'all')
alpha = mlp_attention_layer(hiddens, sen_len, 2 * FLAGS.n_hidden,
FLAGS.l2_reg, FLAGS.random_base, 'sen')
outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 2 * FLAGS.n_hidden])
logits = softmax_layer(outputs, 2 * FLAGS.n_hidden, FLAGS.random_base,
keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
return logits
