def decode_train(cell, embeddings, encoder_state, targets, targets_length, scope='decoder', reuse=None): """ Args: cell: An RNNCell object embeddings: An embedding matrix with shape (vocab_size, word_dim) encoder_state: A tensor that contains the encoder state; its shape should match that of cell.zero_state targets: A int32 tensor with shape (batch, max_len), which contains word indices; should start and end with the proper <BOS> and <EOS> symbol targets_length: A int32 tensor with shape (batch,), which contains the length of each sample in a batch scope: A VariableScope object of a string which indicates the scope reuse: A boolean value or None which specifies whether to reuse variables already defined in the scope Returns: decoder_outputs, which is a float32 (batch, max_len, cell.output_size) tensor that contains the cell's hidden state per time step """ with tf.variable_scope(scope, initializer=tf.orthogonal_initializer(), reuse=reuse): decoder_fn = seq2seq.simple_decoder_fn_train(encoder_state=encoder_state) targets_embed = tf.nn.embedding_lookup(params=embeddings, ids=targets) decoder_outputs, _, _ = seq2seq.dynamic_rnn_decoder(cell=cell, decoder_fn=decoder_fn, inputs=targets_embed, sequence_length=targets_length, time_major=False, scope='rnn') return decoder_outputs
def decoder_components(self, stage, component_name, enc_state): if stage is "training": if toy: components = { "inputs": self.output_placeholder, "sequence_length": self.dec_seq_len } else: answers_batch_embedding = tf.nn.embedding_lookup(self.embedding_matrix, self.answer_ids) components = { "inputs": answers_batch_embedding, "sequence_length": self.dec_seq_len } if self.use_attention: keys, values, score_fn, construct_fn = prepare_attention(None, attention_option = "luong", num_units=self.n_cells, reuse=False) components["function"] = attention_decoder_fn_train(enc_state, keys, values, score_fn, construct_fn) else: components["function"] = seq2seq.simple_decoder_fn_train(enc_state) elif stage is "inference": output_fn, SOS_id, EOS_id = None, self.SOS_id, self.EOS_id components = { "inputs": None, "sequence_length": None } if self.use_attention: keys, values, score_fn, construct_fn = prepare_attention(None, attention_option = "luong", num_units=self.n_cells, reuse=False) components["function"] = attention_decoder_fn_inference(output_fn, enc_state, keys, values, score_fn, construct_fn, self.embedding_matrix, SOS_id, EOS_id, self.max_dec_len, self.vocab_size) else: components["function"] = seq2seq.simple_decoder_fn_inference(output_fn, enc_state, self.embedding_matrix, SOS_id, EOS_id, maximum_length=self.max_dec_len, num_decoder_symbols=self.vocab_size) return components[component_name]
def decoder_train_layer(self, encoded_state, token_embedding): with tf.variable_scope('decoder'): dynamic_fn_train = seq2seq.simple_decoder_fn_train(encoded_state) decoder_outputs, state, context = seq2seq.dynamic_rnn_decoder( self.decoder_cell, dynamic_fn_train, token_embedding, self.sequence_lengths) return decoder_outputs
def _init_decoder(self): """ decoder cell. attention적용 시 결과가 좋지 않음. """ with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=self.len_max, num_decoder_symbols=self.vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = (seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=[ self.len_max for _ in range(self.batch_size) ], time_major=True, scope=scope, )) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def decoder(self): cell = self._rnn_cell() initial_state = cell.zero_state(self.model.batch_size, dtype=tf.float32) decoder_fn = seq2seq.simple_decoder_fn_train(initial_state) output = seq2seq.dynamic_rnn_decoder(cell, decoder_fn, self.convolution, self.sequence_lengths, time_major=True) return output[0]
def decoder(self,encoder_state,inputs=None,is_train=True): ''' 解码器 ''' with tf.variable_scope("decoder") as scope: if is_train is True: decoder_fn=seq2seq.simple_decoder_fn_train(encoder_state) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=inputs,sequence_length=self.seq_len,time_major=False,scope=scope) else: tf.get_variable_scope().reuse_variables() #解码时,通过decoder embedding和decoder bias计算每个词的概率 output_fn=lambda x:tf.nn.softmax(tf.matmul(x,self.dec_embedding,transpose_b=True)+self.dec_bias) decoder_fn=seq2seq.simple_decoder_fn_inference(output_fn=output_fn,encoder_state=encoder_state,embeddings=self.embedding, start_of_sequence_id=0,end_of_sequence_id=0,maximum_length=self.subject_len, num_decoder_symbols=self.vocab_size,dtype=tf.int32) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=None,sequence_length=self.seq_len,time_major=False,scope=scope) return outputs,final_state,final_context_state
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): self.test_outputs = outputs return tf.contrib.layers.linear(outputs, self.decoder_vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train(encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder' ) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=self.time_major, scope=scope, ) ) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax(self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=self.time_major, scope=scope, ) ) self.decoder_prediction_inference = tf.argmax(self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) #attention_states = tf.zeros([batch_size, 1, self.decoder_hidden_units]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = (seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope, )) self.decoder_outputs_train = tf.nn.dropout( self.decoder_outputs_train, _keep_prob) self.decoder_logits_train = output_fn(self.decoder_outputs_train) # reusing the scope of training to use the same variables for inference scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def decoder_adv(self, max_twee_len): with self.graph.as_default(): with tf.variable_scope("Decoder") as scope: self.decoder_length = max_twee_len + 3 def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) # self.decoder_cell = LSTMCell(self.decoder_hidden_nodes) self.decoder_cell = GRUCell(self.decoder_hidden_nodes) if not self.attention: decoder_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_final_state) decoder_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=self.decoder_length, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] self.attention_states = tf.transpose( self.encoder_output, [1, 0, 2]) (self.attention_keys, self.attention_values, self.attention_score_fn, self.attention_construct_fn) = \ seq2seq.prepare_attention(attention_states = self.attention_states, attention_option = "bahdanau", num_units = self.decoder_hidden_nodes) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, name="attention_decoder") decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length= 23, #max_twee_len + 3, #tf.reduce_max(self.de_out_len) + 3, num_decoder_symbols=self.vocab_size) self.decoder_train_inputs_embedded = tf.nn.embedding_lookup( self.embed, self.decoder_train_input) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope)) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope)) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference') return self.de_out, self.de_out_len, self.title_out, self.first_out, self.decoder_logits_train, \ self.decoder_prediction_train, self.loss_weights, self.decoder_train_targets, \ self.decoder_train_title, self.decoder_train_first, self.decoder_prediction_inference
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear( outputs, self.vocab_size, scope=scope ) #this is for calculatng outputs. In a greedy way if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state ) #This is the training function that we used in training dynamic_rnn_decoder #refer to https://github.com/tensorflow/tensorflow/blob/r1.0/tensorflow/contrib/seq2seq/python/ops/decoder_fn.py#L182 decoder_fn_inference = seq2seq.simple_decoder_fn_inference( #nference function for a sequence-to-sequence model. It should be used when dynamic_rnn_decoder is in the inference mode.final mode output_fn= output_fn, #this returns a decoder function . This function in used inside the dynamicRNN function encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [ 1, 0, 2 ]) #take the attention status as the encorder hidden states ( attention_keys, #Each Encoder hidden status multiplied in fully conected way and list of size [num units*Max_time] attention_values, #this is attention encoder states attention_score_fn, #score function of the attention Different ways to compute attention scores If we input the decoder state , encoder hidden states this will out put the context vector attention_construct_fn ) = seq2seq.prepare_attention( #this contruct will Function to compute attention vectors. This will output the concatanaded context vector and the attention wuary then make it as a inpit attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) print("Prininting the number of units .......................") print(self.decoder_hidden_units) print( "Printing the shape of the attetniton values ......................**********************************************" ) print(attention_keys) print( "Printing the attention score function++++++++++++++++++++++++++++++++++++++++++++++++++++" ) print(attention_score_fn) #this function can basically initialize input state of the decoder the nthe attention and other stuff then this will be passed to dy_decorder #decorder_function train will take time, cell_state, cell_input, cell_output, context_state decoder_fn_train = seq2seq.attention_decoder_fn_train( #this is for training the dynamic decorder. This will take care of encoder_state=self. encoder_state, # final state. We take the biderection and concatanate it (c or h) attention_keys= attention_keys, # The transformation of each encoder outputs attention_values= attention_values, #attention encododr status attention_score_fn= attention_score_fn, #this will give a context vector attention_construct_fn= attention_construct_fn, #calculating above thinhs also output the hidden state name='attention_decoder') #What can we achieve by running decorder_fn_ ? done, next state, next input, emit output, next context state #here the emit_output or cell_output will give the output of cell after all atention - non lieanrity applied #this also give the hidden vector output which was concatanated with rnn output and attention vector . Actually concatanated goes throug a linear unit #next_input = array_ops.concat([cell_input, attention], 1) #next cell input #context_state - this will modify when using the beam search #what is the contect state in decorder_fn inside the return funfction of the decorder fn train #the following function is same as the above but the only difference is it's use this in the inference .This has a greedy output #in the inference model cell_output = output_fn(cell_output) . Which means we get logits #next_input = array_ops.concat([cell_input, attention], 1) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( #this is used in the inference model output_fn= output_fn, #this will predict the output and the narcmax after that attention will be concatenaded encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, #doing same attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) #following function is to do all the decodinf with the helop of above functions #this can use in traning or inferense . But we need two separate finctions for trainin and iference #What is this context_state_train : one way to diversify the inference output is to use a stochastic decoder_fn, in which case one would want to store the decoded outputs, not just the RNN outputs. This can be done by maintaining a TensorArray in context_state and storing the decoded output of each iteration therein ( self. decoder_outputs_train, #outputs from the eacah cell [batch_size, max_time, cell.output_size] self. decoder_state_train, #The final state and will be shaped [batch_size, cell.state_size] self.decoder_context_state_train ) = ( #described above seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_train, #decoder_fn allows modeling of early stopping, output, state, and next input and context. inputs=self. decoder_train_inputs_embedded, #inputs to the decoder in the training #in the raning time only sequence_length=self. decoder_train_length, #sequence_length is needed at training time, i.e., when inputs is not None, for dynamic unrolling. At test time, when inputs is None, sequence_length is not needed. time_major= True, #input and output shape should be in [max_time, batch_size, ...] scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train ) #take the final output hidden status and run them throgh linearl layer #get the argmax self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() ( self. decoder_logits_inference, #same as above but no input provided. This will take the predicted things as inputs self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_inference, #difference decorder fucntion time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference' ) #predicted output at the each time step
def _build_graph(self): # required only for training self.targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name="decoder_inputs") self.targets_length = tf.placeholder(shape=(None, ), dtype=tf.int32, name="decoder_inputs_length") self.global_step = tf.Variable(0, name="global_step", trainable=False) with tf.name_scope("DecoderTrainFeed"): sequence_size, batch_size = tf.unstack(tf.shape(self.targets)) EOS_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.EOS PAD_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.PAD self.train_inputs = tf.concat([EOS_SLICE, self.targets], axis=0) self.train_length = self.targets_length + 1 train_targets = tf.concat([self.targets, PAD_SLICE], axis=0) train_targets_seq_len, _ = tf.unstack(tf.shape(train_targets)) train_targets_eos_mask = tf.one_hot(self.train_length - 1, train_targets_seq_len, on_value=self.EOS, off_value=self.PAD, dtype=tf.int32) train_targets_eos_mask = tf.transpose(train_targets_eos_mask, [1, 0]) # hacky way using one_hot to put EOS symbol at the end of target sequence train_targets = tf.add(train_targets, train_targets_eos_mask) self.train_targets = train_targets self.loss_weights = tf.ones( [batch_size, tf.reduce_max(self.train_length)], dtype=tf.float32, name="loss_weights") with tf.variable_scope("embedding") as scope: self.inputs_embedded = tf.nn.embedding_lookup( self.embedding_matrix, self.train_inputs) with tf.variable_scope("Decoder") as scope: def logits_fn(outputs): return layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: train_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) inference_fn = seq2seq.simple_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units) train_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name="decoder_attention") inference_fn = seq2seq.attention_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) (self.train_outputs, self.train_state, self.train_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=train_fn, inputs=self.inputs_embedded, sequence_length=self.train_length, time_major=True, scope=scope) self.train_logits = logits_fn(self.train_outputs) self.train_prediction = tf.argmax(self.train_logits, axis=-1, name="train_prediction") self.train_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="train_prediction_probabilities") scope.reuse_variables() (self.inference_logits, self.inference_state, self.inference_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=inference_fn, time_major=True, scope=scope) self.inference_prediction = tf.argmax(self.inference_logits, axis=-1, name="inference_prediction") self.inference_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="inference_prediction_probabilities")
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tc.layers.fully_connected(outputs, self.output_symbol_size, activation_fn=None, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size) else: (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=self.encoder_outputs, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size, ) if self.is_training: (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=False, scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=False, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def __init_decoder(self): '''Initializes the decoder part of the model.''' with tf.variable_scope('decoder') as scope: output_fn = lambda outs: layers.linear( outs, self.__get_vocab_size(), scope=scope) if self.cfg.get('use_attention'): attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option='bahdanau', num_units=self.decoder_cell.output_size) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) else: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_traion') scope.reuse_variables() (self.decoder_logits_inference, decoder_state_inference, self.decoder_context_state_inference ) = seq2seq.dynamic_rnn_decoder(cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self, forward_only): with tf.variable_scope("decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.target_vocab_size, scope=scope) self.attention = True if not self.attention: if forward_only: decoder_fn = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = (seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.dec_hidden_size)) if forward_only: decoder_fn = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) if not forward_only: self.decoder_logits = output_fn(self.decoder_outputs) else: self.decoder_logits = self.decoder_outputs self.decoder_prediction = tf.argmax(self.decoder_logits, axis=-1, name='decoder_prediction') logits = tf.transpose(self.decoder_logits, [1, 0, 2]) targets = tf.transpose(self.decoder_targets, [1, 0]) if not forward_only: self.loss = seq2seq.sequence_loss(logits=logits, targets=targets, weights=self.target_weights)