def build_decoder_cell(self, encoder_outputs, encoder_state): """构建解码器所有层""" sequence_length = self.encoder_inputs_length if self.mode == 'decode': # 如果使用beam_search,则需要将encoder的输出进行tile_batch,其实就是复制beam_size份 encoder_outputs = tf.contrib.seq2seq.tile_batch(encoder_outputs, multiplier=pm.beam_width) encoder_state = tf.contrib.seq2seq.tile_batch(encoder_state, multiplier=pm.beam_width) sequence_length = tf.contrib.seq2seq.tile_batch(sequence_length, multiplier=pm.beam_width) cell = tf.contrib.rnn.LSTMCell(pm.hidden_size * 2) if self.mode == 'train': cell = DropoutWrapper(cell, dtype=tf.float32, output_keep_prob=pm.keep_prob) else: cell = DropoutWrapper(cell, dtype=tf.float32, output_keep_prob=1.0) # 使用attention机制 self.attention_mechanism = BahdanauAttention(num_units=pm.hidden_size, memory=encoder_outputs, memory_sequence_length=sequence_length) def cell_input_fn(inputs, attention): attn_projection = layers.Dense(pm.hidden_size * 2, dtype=tf.float32, use_bias=False, name='attention_cell_input_fn') return attn_projection(array_ops.concat([inputs, attention], -1)) cell = AttentionWrapper( cell=cell, # rnn cell实例,可以是单个cell,也可以是多个cell stack后的mutli layer rnn attention_mechanism=self.attention_mechanism, # attention mechanism的实例,此处为BahdanauAttention attention_layer_size=pm.hidden_size, # 用来控制我们最后生成的attention是怎么得来;如果不是None,则在调用_compute_attention方法时,得到的加权和向量还会与output进行concat,然后再经过一个线性映射,变成维度为attention_layer_size的向量 cell_input_fn=cell_input_fn, # input送入decoder cell的方式,默认是会将input和上一步计算得到的attention拼接起来送入decoder cell, name='Attention_Wrapper') if self.mode == 'decode': decoder_initial_state = cell.zero_state(batch_size=self.batch_size * pm.beam_width, dtype=tf.float32).clone(cell_state=encoder_state) else: decoder_initial_state = cell.zero_state(batch_size=self.batch_size, dtype=tf.float32).clone(cell_state=encoder_state) return cell, decoder_initial_state
class Encoder(object): """ Object representing an RNN encoder. """ def __init__(self, cell_factory, input_size, hidden_size, input_dropout=None, output_dropout=None): """ :param cell_factory: :param input_size: :param hidden_size: :return: """ self.cell_factory = cell_factory self.input_size = input_size self.hidden_size = hidden_size self.cell = self.cell_factory(self.hidden_size) if input_dropout is not None or output_dropout is not None: self.cell = DropoutWrapper(self.cell, 1 - (input_dropout or 0.0), 1 - (output_dropout or 0.0)) self.state_size = self.cell.state_size def __call__(self, inputs, start_state, scope=None): """Run this RNN cell on inputs, starting from the given state. Args: inputs: list of 2D Tensors with shape [batch_size x self.input_size]. start_state: 2D Tensor with shape [batch_size x self.state_size]. scope: VariableScope for the created subgraph; defaults to class name. Returns: A pair containing: - Outputs: list of 2D Tensors with shape [batch_size x self.output_size] - States: list of 2D Tensors with shape [batch_size x self.state_size]. """ with vs.variable_scope(scope or "Encoder"): return rnn_encoder_factory(self.cell, inputs, start_state) def zero_state(self, batch_size): return self.cell.zero_state(batch_size, tf.float32)
##### RNN Units """ # Create a single GRU cell encoder_fw_cell = tf.nn.rnn_cell.GRUCell(128) # Add dropout : Dropout is applied to the hidden state output at every time step encoder_fw_cell = DropoutWrapper(encoder_fw_cell, output_keep_prob=keep_prob) encoder_bw_cell = tf.nn.rnn_cell.GRUCell(128) encoder_bw_cell = DropoutWrapper(encoder_bw_cell, output_keep_prob=keep_prob) # Unrolling of time-sequence # Apply the encoder cell on input sequence and unroll computation upto # max sequence length enc_outputs, enc_state= tf.nn.bidirectional_dynamic_rnn( encoder_fw_cell,encoder_bw_cell, input_emb,sequence_length=input_lens,initial_state_fw=encoder_fw_cell.zero_state(BATCH_SIZE, dtype=tf.float32),initial_state_bw=encoder_bw_cell.zero_state(BATCH_SIZE, dtype=tf.float32),scope='blstm5') output = tf.concat(enc_outputs, 2) # Create a single GRU cell encoder_fw_cell1 = tf.nn.rnn_cell.GRUCell(128) # Add dropout : Dropout is applied to the hidden state output at every time step encoder_fw_cell1 = DropoutWrapper(encoder_fw_cell1, output_keep_prob=keep_prob) encoder_bw_cell1 = tf.nn.rnn_cell.GRUCell(128) encoder_bw_cell1 = DropoutWrapper(encoder_bw_cell1, output_keep_prob=keep_prob) enc_outputs, enc_state= tf.nn.bidirectional_dynamic_rnn( encoder_fw_cell1,encoder_bw_cell1,output,sequence_length=input_lens,dtype=tf.float32,scope='blstm6') enc_state = tf.concat(enc_state, 1)
def add_encoder(self, inputs, type_layer): '''Construction of the RNN model with LSTM cells. Arguments: - type_layer: should be 'Context' or 'Questions' ''' with tf.variable_scope( 'Encoding-Layer', initializer=tf.contrib.layers.xavier_initializer()) as scope: reuse = type_layer == "Questions" initializer = tf.random_uniform_initializer(-1, 1) cell_fw, cell_bw = self.create_cells(self.config.nb_hidden_layers, reuse) if type_layer == "Context": batch_size = self.config.len_context sequence_length = self.context_len_placeholder elif type_layer == "Questions": batch_size = self.config.len_questions sequence_length = self.questions_len_placeholder cell_fw = DropoutWrapper(cell_fw, output_keep_prob=self.dropout_placeholder) initial_state_fw = cell_fw.zero_state(batch_size, tf.float32) if self.config.hidden_bidirectional: cell_bw = DropoutWrapper( cell_bw, output_keep_prob=self.dropout_placeholder) initial_state_bw = cell_bw.zero_state(batch_size, tf.float32) outputs, hidden_states = tf.nn.bidirectional_dynamic_rnn( cell_fw, cell_bw, inputs, initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw, sequence_length=sequence_length) else: outputs, hidden_states = tf.nn.dynamic_rnn( cell_fw, inputs, initial_state=initial_state_fw, sequence_length=sequence_length) if self.config.output_type == "output": encoding_outputs = tf.transpose(outputs, [1, 0, 2]) encoding_outputs = tf.gather(encoding_outputs, self.config.len_questions - 1) elif self.config.output_type == "hs": if self.config.hidden_bidirectional: encoding_outputs = (hidden_states[0], hidden_states[1]) if self.config.nb_hidden_layers > 1: encoding_outputs = (encoding_outputs[0][-1], encoding_outputs[1][-1]) if self.config.type_cell == "LSTM": encoding_outputs = (encoding_outputs[0].h, encoding_outputs[1].h) else: encoding_outputs = hidden_states if self.config.nb_hidden_layers > 1: encoding_outputs = encoding_outputs[-1] if self.config.type_cell == "LSTM": encoding_outputs = encoding_outputs.h return encoding_outputs