def decode_training_set(encoder_state, decoder_cell, decoder_embedded_input, sequence_length, decoding_scope, output_function, keep_prob, batch_size): """Decode the training set.""" attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size]) (attention_keys, attentions_values, attention_score_function, attention_construct_function) = prepare_attention( attention_states, attention_option='bahdanau', num_units=decoder_cell.output_size) training_decoder_function = attention_decoder_fn_train( encoder_state[0], attention_keys, attentions_values, attention_score_function, attention_construct_function, name='attn_dec_train') decoder_output, _, _ = dynamic_rnn_decoder(decoder_cell, training_decoder_function, decoder_embedded_input, sequence_length, scope=decoding_scope) decoder_output_dropout = tf.nn.dropout(decoder_output, keep_prob) return output_function(decoder_output_dropout)
def decode_training_set(encoder_state, decoder_cell, decoder_embedded_input, sequence_length, decoding_scope, output_function, keep_prob, batch_size): attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size]) attention_keys, attention_values, attention_score_function, attention_construct_function \ = seq2seq.prepare_attention(attention_states, attention_option="bahdanau", num_units=decoder_cell.output_size) training_decoder_function = seq2seq.attention_decoder_fn_train( encoder_state[0], attention_keys, attention_values, attention_score_function, attention_construct_function, name="attn_dec_train") decoder_output, _, _ = seq2seq.dynamic_rnn_decoder( decoder_cell, training_decoder_function, decoder_embedded_input, sequence_length, scope=decoding_scope) decoder_output_dropout = tf.nn.dropout(decoder_output, keep_prob) return output_function(decoder_output_dropout)
def addDecoder(self): print('adding decoder...') cell = BasicRNNCell(2 * CONFIG.DIM_WordEmbedding) self.attention_states = self._encoder_outputs self.decoder_inputs_embedded = tf.nn.embedding_lookup( self.embedding, self.y_placeholder) # prepare attention: (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=self.attention_states, attention_option='bahdanau', num_units=2 * CONFIG.DIM_WordEmbedding) if (self.is_training): # new Seq2seq train version self.check_op = tf.add_check_numerics_ops() decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self._decoder_in_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = seq2seq.dynamic_rnn_decoder( cell=cell, decoder_fn=decoder_fn_train, inputs=self.decoder_inputs_embedded, sequence_length=self.y_lens, time_major=False) self.decoder_outputs = self.decoder_outputs_train else: # new Seq2seq version start_id = CONFIG.WORDS[CONFIG.STARTWORD] stop_id = CONFIG.WORDS[CONFIG.STOPWORD] decoder_fn_inference = seq2seq.attention_decoder_fn_inference( encoder_state=self._decoder_in_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding, start_of_sequence_id=start_id, end_of_sequence_id=stop_id, maximum_length=CONFIG.DIM_DECODER, num_decoder_symbols=CONFIG.DIM_VOCAB, output_fn=self.output_fn) (self.decoder_outputs_inference, self.decoder_state_inference, self.decoder_context_state_inference ) = seq2seq.dynamic_rnn_decoder(cell=cell, decoder_fn=decoder_fn_inference, time_major=False) self.decoder_outputs = self.decoder_outputs_inference
def decoder(self,encoder_state,attention_states,inputs=None,is_train=True): ''' 基于attention的解码器 1.调用seq2seq.prepare_attention 生成attention的keys/values/functions 2.训练时,定义dynamic_rnn_decoder用到的attention_decoder_fn_train 3.预测时,定义dynamic_rnn_decoder用到的attention_decoder_fn_inference 4.使用以上步骤得到的参数,调用seq2seq.dynamic_rnn_decoder函数 ''' with tf.variable_scope("decoder") as scope: #1. prepare attention keys,values,score_fn,construct_fn=seq2seq.prepare_attention(attention_states=attention_states,attention_option="luong",num_units=self.emb_dim) if is_train is True: decoder_fn=seq2seq.attention_decoder_fn_train(encoder_state,attention_keys=keys,attention_values=values,attention_score_fn=score_fn,attention_construct_fn=construct_fn,) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=inputs,sequence_length=self.seq_len,time_major=False,scope=scope) else: tf.get_variable_scope().reuse_variables() #解码时,通过decoder embedding和decoder bias计算每个词的概率 output_fn=lambda x:tf.nn.softmax(tf.matmul(x,self.dec_embedding,transpose_b=True)+self.dec_bias) decoder_fn=seq2seq.attention_decoder_fn_inference(output_fn=output_fn,encoder_state=encoder_state,attention_keys=keys,attention_values=values, attention_score_fn=score_fn,attention_construct_fn=construct_fn,embeddings=self.dec_embedding, start_of_sequence_id=0,end_of_sequence_id=1,maximum_length=5,num_decoder_symbols=self.vocab_size) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=None,sequence_length=self.seq_len,time_major=False,scope=scope) return outputs,final_state,final_context_state
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): self.test_outputs = outputs return tf.contrib.layers.linear(outputs, self.decoder_vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train(encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder' ) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=self.time_major, scope=scope, ) ) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax(self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=self.time_major, scope=scope, ) ) self.decoder_prediction_inference = tf.argmax(self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) #attention_states = tf.zeros([batch_size, 1, self.decoder_hidden_units]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = (seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope, )) self.decoder_outputs_train = tf.nn.dropout( self.decoder_outputs_train, _keep_prob) self.decoder_logits_train = output_fn(self.decoder_outputs_train) # reusing the scope of training to use the same variables for inference scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def decoder_adv(self, max_twee_len): with self.graph.as_default(): with tf.variable_scope("Decoder") as scope: self.decoder_length = max_twee_len + 3 def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) # self.decoder_cell = LSTMCell(self.decoder_hidden_nodes) self.decoder_cell = GRUCell(self.decoder_hidden_nodes) if not self.attention: decoder_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_final_state) decoder_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=self.decoder_length, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] self.attention_states = tf.transpose( self.encoder_output, [1, 0, 2]) (self.attention_keys, self.attention_values, self.attention_score_fn, self.attention_construct_fn) = \ seq2seq.prepare_attention(attention_states = self.attention_states, attention_option = "bahdanau", num_units = self.decoder_hidden_nodes) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, name="attention_decoder") decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length= 23, #max_twee_len + 3, #tf.reduce_max(self.de_out_len) + 3, num_decoder_symbols=self.vocab_size) self.decoder_train_inputs_embedded = tf.nn.embedding_lookup( self.embed, self.decoder_train_input) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope)) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope)) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference') return self.de_out, self.de_out_len, self.title_out, self.first_out, self.decoder_logits_train, \ self.decoder_prediction_train, self.loss_weights, self.decoder_train_targets, \ self.decoder_train_title, self.decoder_train_first, self.decoder_prediction_inference
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear( outputs, self.vocab_size, scope=scope ) #this is for calculatng outputs. In a greedy way if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state ) #This is the training function that we used in training dynamic_rnn_decoder #refer to https://github.com/tensorflow/tensorflow/blob/r1.0/tensorflow/contrib/seq2seq/python/ops/decoder_fn.py#L182 decoder_fn_inference = seq2seq.simple_decoder_fn_inference( #nference function for a sequence-to-sequence model. It should be used when dynamic_rnn_decoder is in the inference mode.final mode output_fn= output_fn, #this returns a decoder function . This function in used inside the dynamicRNN function encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [ 1, 0, 2 ]) #take the attention status as the encorder hidden states ( attention_keys, #Each Encoder hidden status multiplied in fully conected way and list of size [num units*Max_time] attention_values, #this is attention encoder states attention_score_fn, #score function of the attention Different ways to compute attention scores If we input the decoder state , encoder hidden states this will out put the context vector attention_construct_fn ) = seq2seq.prepare_attention( #this contruct will Function to compute attention vectors. This will output the concatanaded context vector and the attention wuary then make it as a inpit attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) print("Prininting the number of units .......................") print(self.decoder_hidden_units) print( "Printing the shape of the attetniton values ......................**********************************************" ) print(attention_keys) print( "Printing the attention score function++++++++++++++++++++++++++++++++++++++++++++++++++++" ) print(attention_score_fn) #this function can basically initialize input state of the decoder the nthe attention and other stuff then this will be passed to dy_decorder #decorder_function train will take time, cell_state, cell_input, cell_output, context_state decoder_fn_train = seq2seq.attention_decoder_fn_train( #this is for training the dynamic decorder. This will take care of encoder_state=self. encoder_state, # final state. We take the biderection and concatanate it (c or h) attention_keys= attention_keys, # The transformation of each encoder outputs attention_values= attention_values, #attention encododr status attention_score_fn= attention_score_fn, #this will give a context vector attention_construct_fn= attention_construct_fn, #calculating above thinhs also output the hidden state name='attention_decoder') #What can we achieve by running decorder_fn_ ? done, next state, next input, emit output, next context state #here the emit_output or cell_output will give the output of cell after all atention - non lieanrity applied #this also give the hidden vector output which was concatanated with rnn output and attention vector . Actually concatanated goes throug a linear unit #next_input = array_ops.concat([cell_input, attention], 1) #next cell input #context_state - this will modify when using the beam search #what is the contect state in decorder_fn inside the return funfction of the decorder fn train #the following function is same as the above but the only difference is it's use this in the inference .This has a greedy output #in the inference model cell_output = output_fn(cell_output) . Which means we get logits #next_input = array_ops.concat([cell_input, attention], 1) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( #this is used in the inference model output_fn= output_fn, #this will predict the output and the narcmax after that attention will be concatenaded encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, #doing same attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) #following function is to do all the decodinf with the helop of above functions #this can use in traning or inferense . But we need two separate finctions for trainin and iference #What is this context_state_train : one way to diversify the inference output is to use a stochastic decoder_fn, in which case one would want to store the decoded outputs, not just the RNN outputs. This can be done by maintaining a TensorArray in context_state and storing the decoded output of each iteration therein ( self. decoder_outputs_train, #outputs from the eacah cell [batch_size, max_time, cell.output_size] self. decoder_state_train, #The final state and will be shaped [batch_size, cell.state_size] self.decoder_context_state_train ) = ( #described above seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_train, #decoder_fn allows modeling of early stopping, output, state, and next input and context. inputs=self. decoder_train_inputs_embedded, #inputs to the decoder in the training #in the raning time only sequence_length=self. decoder_train_length, #sequence_length is needed at training time, i.e., when inputs is not None, for dynamic unrolling. At test time, when inputs is None, sequence_length is not needed. time_major= True, #input and output shape should be in [max_time, batch_size, ...] scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train ) #take the final output hidden status and run them throgh linearl layer #get the argmax self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() ( self. decoder_logits_inference, #same as above but no input provided. This will take the predicted things as inputs self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_inference, #difference decorder fucntion time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference' ) #predicted output at the each time step
def _add_decoder(self): """ Adds decoder (i.e., "writer" or "generator") to the computation graph/model :return: None """ log.debug('Adding decoder component to computation graph...') hps = self._hps vsize = self._vocab_size with tf.variable_scope('decoder') as scope: attn_states = self._encoder_outputs # Attention states: size [batch_size, hps.max_decoder_timesteps, num_units] log.trace('Attention States: %s', attn_states) attn_keys, attn_values, attn_score_fn, attn_const_fn = tf.contrib.seq2seq.prepare_attention( attn_states, attention_option=hps.attn_option, num_units=hps.num_hidden) log.trace("Attention Keys: %s", attn_keys) log.trace('Attention Values: %s', attn_values) log.trace('Attention Score Function: %s', attn_score_fn) log.trace('Attention Construction Function: %s', attn_const_fn) # Define the type of RNN cells the RNLM will use if hps.dec_layers > 1: cell = contrib.rnn.MultiRNNCell([self._rnn_cell(hps.num_hidden) for _ in xrange(hps.dec_layers)]) else: cell = self._rnn_cell(hps.num_hidden) log.trace('Decoder RNN Cell: %s', cell) # Setup weights for computing the final output def create_output_fn(): def _output_fn(x): return tf.contrib.layers.linear(x, vsize, scope=scope) return _output_fn output_fn = create_output_fn() # We don't need to add the training decoder unless we're training model if self._is_training: # Setup decoder in (1) training mode (i.e., consider gold previous words) and (2) with attention decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self._encoder_state, attention_keys=attn_keys, attention_values=attn_values, attention_score_fn=attn_score_fn, attention_construct_fn=attn_const_fn) # Setup RNLM for training self._emb_decoder_inputs = tf.nn.embedding_lookup(self._embedding, self._sections) decoder_outputs_train, decoder_state_train, _ = \ contrib.seq2seq.dynamic_rnn_decoder(cell=cell, decoder_fn=decoder_fn_train, inputs=self._emb_decoder_inputs, sequence_length=hps.dec_timesteps - 1, swap_memory=True, scope=scope) self._raw_decoder_outputs = decoder_outputs_train # Project RNLM outputs into vocabulary space self._decoder_outputs_train = outputs = output_fn(decoder_outputs_train) log.trace('(Projected) RNLM Decoder Training Outputs: %s', outputs) # Compute sequence loss self._decoder_loss = seq2seq.sequence_loss(outputs, self._targets, self._loss_weights) log.trace('RNLM Decoder Loss: %s', self._decoder_loss) tf.summary.scalar('decoder loss', tf.minimum(12.0, self._decoder_loss)) # If we have created a training decoder, tell the inference decoder to re-use the same weights scope.reuse_variables() # Inference decoder: use previously generated output to predict next output (e.g., no gold outputs) decoder_fn_inference = tf.contrib.seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self._encoder_state, attention_keys=attn_keys, attention_values=attn_values, attention_score_fn=attn_score_fn, attention_construct_fn=attn_const_fn, embeddings=self._embedding, start_of_sequence_id=record_data.PARAGRAPH_START_ID, end_of_sequence_id=record_data.PARAGRAPH_END_ID, maximum_length=hps.dec_timesteps - 1, num_decoder_symbols=self._vocab_size, dtype=tf.int32) decoder_outputs_inference, decoder_state_inference, _ = \ tf.contrib.seq2seq.dynamic_rnn_decoder(cell=cell, decoder_fn=decoder_fn_inference, swap_memory=True, scope=scope) log.trace('RNLM Decoder Inference Outputs: %s', decoder_outputs_inference) self._decoder_outputs_inference = decoder_outputs_inference
def _build_graph(self): # required only for training self.targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name="decoder_inputs") self.targets_length = tf.placeholder(shape=(None, ), dtype=tf.int32, name="decoder_inputs_length") self.global_step = tf.Variable(0, name="global_step", trainable=False) with tf.name_scope("DecoderTrainFeed"): sequence_size, batch_size = tf.unstack(tf.shape(self.targets)) EOS_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.EOS PAD_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.PAD self.train_inputs = tf.concat([EOS_SLICE, self.targets], axis=0) self.train_length = self.targets_length + 1 train_targets = tf.concat([self.targets, PAD_SLICE], axis=0) train_targets_seq_len, _ = tf.unstack(tf.shape(train_targets)) train_targets_eos_mask = tf.one_hot(self.train_length - 1, train_targets_seq_len, on_value=self.EOS, off_value=self.PAD, dtype=tf.int32) train_targets_eos_mask = tf.transpose(train_targets_eos_mask, [1, 0]) # hacky way using one_hot to put EOS symbol at the end of target sequence train_targets = tf.add(train_targets, train_targets_eos_mask) self.train_targets = train_targets self.loss_weights = tf.ones( [batch_size, tf.reduce_max(self.train_length)], dtype=tf.float32, name="loss_weights") with tf.variable_scope("embedding") as scope: self.inputs_embedded = tf.nn.embedding_lookup( self.embedding_matrix, self.train_inputs) with tf.variable_scope("Decoder") as scope: def logits_fn(outputs): return layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: train_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) inference_fn = seq2seq.simple_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units) train_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name="decoder_attention") inference_fn = seq2seq.attention_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) (self.train_outputs, self.train_state, self.train_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=train_fn, inputs=self.inputs_embedded, sequence_length=self.train_length, time_major=True, scope=scope) self.train_logits = logits_fn(self.train_outputs) self.train_prediction = tf.argmax(self.train_logits, axis=-1, name="train_prediction") self.train_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="train_prediction_probabilities") scope.reuse_variables() (self.inference_logits, self.inference_state, self.inference_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=inference_fn, time_major=True, scope=scope) self.inference_prediction = tf.argmax(self.inference_logits, axis=-1, name="inference_prediction") self.inference_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="inference_prediction_probabilities")
def __init__(self, vocab_size, embed_size, num_layers, hidden_size, eos, max_len, initial_embed=None): super(Seq2SeqAttn, self).__init__() self.vocab_size = vocab_size self.embed_size = embed_size self.num_layers = num_layers self.hidden_size = hidden_size # TO DO self.EOS = eos self.PAD = 0 # placeholders self.encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs') self.encoder_inputs_length = tf.placeholder( shape=(None, ), dtype=tf.int32, name='encoder_inputs_length') self.decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets') self.rewards = tf.placeholder(shape=(None, ), dtype=tf.float32, name='rewards') # self.decoder_targets_length = tf.placeholder(shape=(None,), dtype=tf.int32, name='decoder_targets_length') # LSTM cell cells = [] for _ in range(num_layers): cells.append(tf.contrib.rnn.LSTMCell(hidden_size)) self.encoder_cell = MultiRNNCell(cells) self.decoder_cell = MultiRNNCell(cells) # decoder train feeds with tf.name_scope('decoder_feeds'): sequence_size, batch_size = tf.unstack( tf.shape(self.decoder_targets)) EOS_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.EOS PAD_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.PAD self.decoder_train_inputs = tf.concat( [EOS_SLICE, self.decoder_targets], axis=0) # self.decoder_train_length = self.decoder_targets_length + 1 # one for EOS self.decoder_train_length = tf.cast( tf.ones(shape=(batch_size, )) * (max_len + 1), tf.int32) decoder_train_targets = tf.concat( [self.decoder_targets, PAD_SLICE], axis=0) # (seq_len + 1) * batch_size # decoder_train_targets_seq_len, _ = tf.unstack(tf.shape(decoder_train_targets)) # decoder_train_targets_eos_mask = tf.one_hot( # self.decoder_targets_length, # decoder_train_targets_seq_len, # on_value=self.EOS, # off_value=self.PAD, # dtype=tf.int32 # ) # batch_size * (seq_len + 1) # decoder_train_targets_eos_mask = tf.transpose(decoder_train_targets_eos_mask, [1, 0]) # decoder_train_targets = tf.add(decoder_train_targets, decoder_train_targets_eos_mask) self.decoder_train_targets = decoder_train_targets # (seq_len + 1) * batch_size with EOS at the end of each sentence # loss_weights loss_weights = tf.cast( tf.cast(self.decoder_train_targets, tf.bool), tf.float32) self.loss_weights = tf.transpose(loss_weights, perm=[1, 0]) # self.loss_weights = tf.ones([batch_size, tf.reduce_max(self.decoder_train_length)], dtype=tf.float32, name="loss_weights") # embedding layer with tf.variable_scope('embedding'): # if initial_embed: self.embedding = tf.Variable(initial_embed, name='matrix', dtype=tf.float32) # else: # self.embedding = tf.Variable(tf.random_normal([vocab_size, embed_size], - 0.5 / embed_size, 0.5 / embed_size), name='matrix', dtype=tf.float32) self.encoder_inputs_embedded = tf.nn.embedding_lookup( self.embedding, self.encoder_inputs) self.decoder_train_inputs_embedded = tf.nn.embedding_lookup( self.embedding, self.decoder_train_inputs) # encoder with tf.variable_scope('encoder'): self.encoder_outputs, self.encoder_state = tf.nn.dynamic_rnn( cell=self.encoder_cell, inputs=self.encoder_inputs_embedded, sequence_length=self.encoder_inputs_length, time_major=True, dtype=tf.float32) # decoder with tf.variable_scope('decoder') as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) attention_states = tf.transpose( self.encoder_outputs, [1, 0, 2]) # batch_size * seq_len * hidden_size (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.hidden_size, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=35, num_decoder_symbols=self.vocab_size, ) self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train = seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope, ) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference') # optimizer with tf.name_scope('optimizer'): self.global_step = tf.Variable(0, name="global_step", trainable=False) # self.policy_step = tf.Variable(0, name='policy_step', trainable=False) logits = tf.transpose( self.decoder_logits_train, [1, 0, 2]) # batch_size * sequence_length * vocab_size targets = tf.transpose(self.decoder_train_targets, [1, 0]) logits_inference = tf.transpose(self.decoder_logits_inference, [1, 0, 2]) output_prob = tf.reduce_max(tf.nn.softmax(logits_inference), axis=2) # batch_size * seq_len seq_log_prob = tf.reduce_sum(tf.log(output_prob), axis=1) self.policy_loss = -tf.reduce_sum(self.rewards * seq_log_prob) self.policy_op = tf.train.AdamOptimizer().minimize( self.policy_loss) self.loss = seq2seq.sequence_loss(logits=logits, targets=targets, weights=self.loss_weights) self.train_op = tf.train.AdamOptimizer().minimize( self.loss, global_step=self.global_step)
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tc.layers.fully_connected(outputs, self.output_symbol_size, activation_fn=None, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size) else: (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=self.encoder_outputs, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size, ) if self.is_training: (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=False, scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=False, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self, output_projection): ''' Decoder phase ''' with tf.variable_scope("Decoder") as scope: self.decoder_inputs_embedded = tf.nn.embedding_lookup(self.lookup_matrix, self.decoder_inputs) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=self.attention_states, attention_option="bahdanau", num_units=self.args.h_units_decoder, ) # attention is added decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder' ) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_projection, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.lookup_matrix, start_of_sequence_id=self.textData.goToken, end_of_sequence_id=self.textData.eosToken, maximum_length=tf.reduce_max(self.decoder_targets_length), num_decoder_symbols=self.textData.getVocabularySize(), ) # Check back here later...the hidden size of decoder_cell has to be in the same size of embedding layer? # !!! # decoder_outputs_train.shape = (batch_size, n_words, hidden_size) ( self.decoder_outputs_train, decoder_state_train, decoder_context_state_train) = seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_targets_length, time_major=False, scope=scope ) # self.decoder_logits_train = output_projection(self.decoder_outputs_train) # self.decoder_logits_train_trans = tf.reshape(self.decoder_outputs_train, [1,0,2]) self.decoder_logits_train = tf.transpose(tf.map_fn(output_projection, tf.transpose(self.decoder_outputs_train, [1, 0, 2])), [1, 0, 2]) self.decoder_prediction_train = tf.argmax(self.decoder_logits_train, axis=-1, name='decoder_prediction_train') # for both training and inference scope.reuse_variables() (self.decoder_logits_inference, decoder_state_inference, decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=False, scope=scope ) )
def __init_decoder(self): '''Initializes the decoder part of the model.''' with tf.variable_scope('decoder') as scope: output_fn = lambda outs: layers.linear( outs, self.__get_vocab_size(), scope=scope) if self.cfg.get('use_attention'): attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option='bahdanau', num_units=self.decoder_cell.output_size) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) else: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_traion') scope.reuse_variables() (self.decoder_logits_inference, decoder_state_inference, self.decoder_context_state_inference ) = seq2seq.dynamic_rnn_decoder(cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self, forward_only): with tf.variable_scope("decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.target_vocab_size, scope=scope) self.attention = True if not self.attention: if forward_only: decoder_fn = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = (seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.dec_hidden_size)) if forward_only: decoder_fn = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) if not forward_only: self.decoder_logits = output_fn(self.decoder_outputs) else: self.decoder_logits = self.decoder_outputs self.decoder_prediction = tf.argmax(self.decoder_logits, axis=-1, name='decoder_prediction') logits = tf.transpose(self.decoder_logits, [1, 0, 2]) targets = tf.transpose(self.decoder_targets, [1, 0]) if not forward_only: self.loss = seq2seq.sequence_loss(logits=logits, targets=targets, weights=self.target_weights)