def ian(inputs, sen_len, target, sen_len_tr, keep_prob1, keep_prob2, _id='all'): cell = tf.contrib.rnn.LSTMCell # sentence hidden inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) hiddens_s = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sen' + _id, 'all') pool_s = reduce_mean_with_len(hiddens_s, sen_len) # target hidden target = tf.nn.dropout(target, keep_prob=keep_prob1) hiddens_t = bi_dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr, FLAGS.max_sentence_len, 't' + _id, 'all') pool_t = reduce_mean_with_len(hiddens_t, sen_len_tr) # attention sentence att_s = bilinear_attention_layer(hiddens_s, pool_t, sen_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sen') outputs_s = tf.squeeze(tf.matmul(att_s, hiddens_s)) # attention target att_t = bilinear_attention_layer(hiddens_t, pool_s, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 't') outputs_t = tf.squeeze(tf.matmul(att_t, hiddens_t)) outputs = tf.concat([outputs_s, outputs_t], 1) prob = softmax_layer(outputs, 4 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return prob, att_s, att_t
def hn_inter_att(inputs_o, sen_len_o, doc_len_o, inputs_r, sen_len_r, doc_len_r, keep_prob1, keep_prob2, _id='0'): print 'I am hn-inter-att!' cell = tf.contrib.rnn.LSTMCell # Original Part inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1) sen_len_o = tf.reshape(sen_len_o, [-1]) hiddens_sen_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o, FLAGS.max_sentence_len, 'sentence_o', 'all') alpha_sen_o = mlp_attention_layer(hiddens_sen_o, sen_len_o, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sentence_o') outputs_sen_o = tf.reshape(tf.matmul(alpha_sen_o, hiddens_sen_o), [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden]) hiddens_doc_o = bi_dynamic_rnn(cell, outputs_sen_o, FLAGS.n_hidden, doc_len_o, FLAGS.max_doc_len, 'doc_o', 'all') # Reverse Part inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1) sen_len_r = tf.reshape(sen_len_r, [-1]) hiddens_sen_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r, FLAGS.max_sentence_len, 'sentence_r', 'all') alpha_sen_r = mlp_attention_layer(hiddens_sen_r, sen_len_r, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sentence_r') outputs_sen_r = tf.reshape(tf.matmul(alpha_sen_r, hiddens_sen_r), [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden]) hiddens_doc_r = bi_dynamic_rnn(cell, outputs_sen_r, FLAGS.n_hidden, doc_len_r, FLAGS.max_doc_len, 'doc_r', 'all') # Combined Part hiddens_doc = tf.concat([hiddens_doc_o, hiddens_doc_r], 2) # batch_size * max_doc_len * 4n_hidden alpha_doc = mlp_attention_layer(hiddens_doc, doc_len_o, 4 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'doc') outputs_doc = tf.reshape(tf.matmul(alpha_doc, hiddens_doc), [-1, 4 * FLAGS.n_hidden]) logits = softmax_layer(outputs_doc, 4 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return logits
def hn_att(inputs, sen_len, doc_len, keep_prob1, keep_prob2, _id='1'): print 'I am HAN!' inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) cell = tf.contrib.rnn.LSTMCell sen_len = tf.reshape(sen_len, [-1]) hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence' + _id, 'all') alpha_sen = mlp_attention_layer(hiddens_sen, sen_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sentence' + _id) outputs_sen = tf.reshape(tf.matmul(alpha_sen, hiddens_sen), [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden]) # sen_len = tf.reshape(sen_len, [-1]) # sen_len = tf.cast(sen_len, tf.float32) # alpha = 1.0 - tf.cast(sen_len / (tf.reduce_sum(sen_len, 1, keep_dims=True) + 1), tf.float32) # alpha = tf.reshape(softmax_with_len(alpha, doc_len, FLAGS.max_doc_len), [-1, FLAGS.max_doc_len, 1]) # outputs_sen *= alpha hiddens_doc = bi_dynamic_rnn(cell, outputs_sen, FLAGS.n_hidden, doc_len, FLAGS.max_doc_len, 'doc' + _id, 'all') alpha_doc = mlp_attention_layer(hiddens_doc, doc_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'doc' + _id) outputs_doc = tf.reshape(tf.matmul(alpha_doc, hiddens_doc), [-1, 2 * FLAGS.n_hidden]) return outputs_doc
def hn(inputs, sen_len, doc_len, keep_prob1, keep_prob2, _id='1'): print 'I am HN!' inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) cell = tf.contrib.rnn.LSTMCell sen_len = tf.reshape(sen_len, [-1]) hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence' + _id, FLAGS.t1) hiddens_sen = tf.reshape(hiddens_sen, [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden]) hidden_doc = bi_dynamic_rnn(cell, hiddens_sen, FLAGS.n_hidden, doc_len, FLAGS.max_doc_len, 'doc' + _id, FLAGS.t2) return hidden_doc
def dual_sen(inputs_o, sen_len_o, inputs_r, sen_len_r, keep_prob1, keep_prob2): print 'I am dual-sen!' cell = tf.contrib.rnn.LSTMCell inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1) hiddens_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o, FLAGS.max_sentence_len, 'sentence_o', 'last') inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1) hiddens_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r, FLAGS.max_sentence_len, 'sentence_r', 'last') hiddens = tf.concat([hiddens_o, hiddens_r], 1) logits = softmax_layer(hiddens, 4 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return logits
def hn(inputs, sen_len, doc_len, keep_prob1, keep_prob2, id_=1): print 'I am HN!' inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) cell = tf.contrib.rnn.LSTMCell sen_len = tf.reshape(sen_len, [-1]) hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence' + str(id_), FLAGS.t1) hiddens_sen = tf.reshape(hiddens_sen, [-1, FLAGS.max_doc_len, 2 * FLAGS.n_hidden]) hidden_doc = bi_dynamic_rnn(cell, hiddens_sen, FLAGS.n_hidden, doc_len, FLAGS.max_doc_len, 'doc' + str(id_), FLAGS.t2) return softmax_layer(hidden_doc, 2 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class)
def lstm(inputs, sen_len, keep_prob1, keep_prob2, _id='1'): print 'I am dual-sen!' cell = tf.contrib.rnn.LSTMCell inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, _id, 'last') return hiddens
def lstm_sen(inputs, sen_len, keep_prob1, keep_prob2): print 'I am lstm-sen!' cell = tf.contrib.rnn.LSTMCell inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence_o', 'last') logits = softmax_layer(hiddens, 2 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return logits
def bi_rnn(inputs, sen_len, keep_prob1, keep_prob2, _id='1'): print 'I am bi-rnn.' inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) cell = tf.contrib.rnn.LSTMCell hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence' + _id, FLAGS.t1) return softmax_layer(hiddens, 2 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class, _id)
def dual_att_sen(inputs_o, sen_len_o, inputs_r, sen_len_r, keep_prob1, keep_prob2): print 'I am dual-att-sen!' cell = tf.contrib.rnn.LSTMCell inputs_o = tf.nn.dropout(inputs_o, keep_prob=keep_prob1) hiddens_o = bi_dynamic_rnn(cell, inputs_o, FLAGS.n_hidden, sen_len_o, FLAGS.max_sentence_len, 'sentence_o', 'all') inputs_r = tf.nn.dropout(inputs_r, keep_prob=keep_prob1) hiddens_r = bi_dynamic_rnn(cell, inputs_r, FLAGS.n_hidden, sen_len_r, FLAGS.max_sentence_len, 'sentence_r', 'all') hiddens = tf.concat([hiddens_o, hiddens_r], 2) alpha = mlp_attention_layer(hiddens, sen_len_o, 4 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sen') outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 4 * FLAGS.n_hidden]) logits = softmax_layer(outputs, 4 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return logits
def lcr_rot(input_fw, input_bw, sen_len_fw, sen_len_bw, target, sen_len_tr, keep_prob1, keep_prob2, _id='all'): print('I am lcr_rot.') cell = tf.contrib.rnn.LSTMCell # left hidden input_fw = tf.nn.dropout(input_fw, keep_prob=keep_prob1) hiddens_l = bi_dynamic_rnn(cell, input_fw, FLAGS.n_hidden, sen_len_fw, FLAGS.max_sentence_len, 'l' + _id, 'all') # hiddens_l = dynamic_rnn(cell, input_fw, FLAGS.n_hidden, sen_len_fw, FLAGS.max_sentence_len, 'l' + _id, 'all') pool_l = reduce_mean_with_len(hiddens_l, sen_len_fw) # right hidden input_bw = tf.nn.dropout(input_bw, keep_prob=keep_prob1) hiddens_r = bi_dynamic_rnn(cell, input_bw, FLAGS.n_hidden, sen_len_bw, FLAGS.max_sentence_len, 'r' + _id, 'all') # hiddens_r = dynamic_rnn(cell, input_bw, FLAGS.n_hidden, sen_len_bw, FLAGS.max_sentence_len, 'r' + _id, 'all') pool_r = reduce_mean_with_len(hiddens_r, sen_len_bw) # target hidden target = tf.nn.dropout(target, keep_prob=keep_prob1) hiddens_t = bi_dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr, FLAGS.max_sentence_len, 't' + _id, 'all') # hiddens_t = dynamic_rnn(cell, target, FLAGS.n_hidden, sen_len_tr, FLAGS.max_sentence_len, 't' + _id, 'all') pool_t = reduce_mean_with_len(hiddens_t, sen_len_tr) # pool_t = tf.concat(1, [target, target]) # attention target att_t_l = bilinear_attention_layer(hiddens_t, pool_l, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tl') # att_t_l = bilinear_attention_layer(hiddens_t, pool_l, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tl') outputs_t_l = tf.squeeze(tf.matmul(att_t_l, hiddens_t)) att_t_r = bilinear_attention_layer(hiddens_t, pool_r, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tr') # att_t_r = bilinear_attention_layer(hiddens_t, pool_r, sen_len_tr, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'tr') outputs_t_r = tf.squeeze(tf.matmul(att_t_r, hiddens_t)) # attention left att_l = bilinear_attention_layer(hiddens_l, outputs_t_l, sen_len_fw, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'l') outputs_l = tf.squeeze(tf.matmul(att_l, hiddens_l)) # # attention right att_r = bilinear_attention_layer(hiddens_r, outputs_t_r, sen_len_bw, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'r') outputs_r = tf.squeeze(tf.matmul(att_r, hiddens_r)) outputs = tf.concat([outputs_l, outputs_r, outputs_t_l, outputs_t_r], 1) # outputs = (outputs_l + outputs_r + outputs_t_l + outputs_t_r) / 4.0 prob = softmax_layer(outputs, 8 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return prob, att_l, att_r, att_t_l, att_t_r
def add_bilstm_layer(self, inputs, seq_len, max_len, scope_name='bilstm'): inputs = tf.nn.dropout(inputs, keep_prob=self.keep_prob1) cell = tf.contrib.rnn.LSTMCell seq_len = tf.reshape(seq_len, [-1]) hiddens = bi_dynamic_rnn(cell, inputs, self.config.n_hidden, seq_len, max_len, scope_name, 'all') alpha = mlp_attention_layer(hiddens, seq_len, 2 * self.config.n_hidden, self.config.l2_reg, self.config.random_base, scope_name) outputs = tf.squeeze(tf.matmul(alpha, hiddens)) return outputs
def bi_rnn_att(inputs, sen_len, keep_prob1, keep_prob2, _id='1'): inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) cell = tf.contrib.rnn.LSTMCell hiddens_sen = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, _id, 'all') alpha_sen = mlp_attention_layer(hiddens_sen, sen_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, _id) outputs_sen = tf.squeeze(tf.matmul(alpha_sen, hiddens_sen)) return softmax_layer(outputs_sen, 2 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class, _id)
def lstm_att(inputs, sen_len, keep_prob1, keep_prob2, _id='1'): print 'I am dual-att-sen!' cell = tf.contrib.rnn.LSTMCell inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, _id, 'all') alpha = mlp_attention_layer(hiddens, sen_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, _id) outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 2 * FLAGS.n_hidden]) return outputs
def create_model(self, inputs): inputs = tf.reshape( inputs, [-1, self.config.max_sentence_len, self.config.embedding_dim]) inputs = tf.nn.dropout(inputs, keep_prob=self.keep_prob1) cell = tf.contrib.rnn.LSTMCell outputs_dim = 2 * self.config.n_hidden outputs_sen = bi_dynamic_rnn(cell, inputs, self.config.n_hidden, tf.reshape(self.sen_len, [-1]), self.config.max_sentence_len, 'sen', 'last') outputs_sen = tf.reshape(outputs_sen, [-1, self.config.max_doc_len, outputs_dim]) outputs_doc = bi_dynamic_rnn(cell, outputs_sen, self.config.n_hidden, self.doc_len, self.config.max_doc_len, 'doc', 'last') doc_logits = softmax_layer(outputs_doc, outputs_dim, self.config.random_base, self.keep_prob2, self.config.l2_reg, self.config.n_class, 'doc') return doc_logits
def lstm_att_sen(inputs, sen_len, keep_prob1, keep_prob2): print 'I am lstm-att-sen!' cell = tf.contrib.rnn.LSTMCell inputs = tf.nn.dropout(inputs, keep_prob=keep_prob1) hiddens = bi_dynamic_rnn(cell, inputs, FLAGS.n_hidden, sen_len, FLAGS.max_sentence_len, 'sentence_o', 'all') alpha = mlp_attention_layer(hiddens, sen_len, 2 * FLAGS.n_hidden, FLAGS.l2_reg, FLAGS.random_base, 'sen') outputs = tf.reshape(tf.matmul(alpha, hiddens), [-1, 2 * FLAGS.n_hidden]) logits = softmax_layer(outputs, 2 * FLAGS.n_hidden, FLAGS.random_base, keep_prob2, FLAGS.l2_reg, FLAGS.n_class) return logits