def decoder_components(self, stage, component_name, enc_state): if stage is "training": if toy: components = { "inputs": self.output_placeholder, "sequence_length": self.dec_seq_len } else: answers_batch_embedding = tf.nn.embedding_lookup(self.embedding_matrix, self.answer_ids) components = { "inputs": answers_batch_embedding, "sequence_length": self.dec_seq_len } if self.use_attention: keys, values, score_fn, construct_fn = prepare_attention(None, attention_option = "luong", num_units=self.n_cells, reuse=False) components["function"] = attention_decoder_fn_train(enc_state, keys, values, score_fn, construct_fn) else: components["function"] = seq2seq.simple_decoder_fn_train(enc_state) elif stage is "inference": output_fn, SOS_id, EOS_id = None, self.SOS_id, self.EOS_id components = { "inputs": None, "sequence_length": None } if self.use_attention: keys, values, score_fn, construct_fn = prepare_attention(None, attention_option = "luong", num_units=self.n_cells, reuse=False) components["function"] = attention_decoder_fn_inference(output_fn, enc_state, keys, values, score_fn, construct_fn, self.embedding_matrix, SOS_id, EOS_id, self.max_dec_len, self.vocab_size) else: components["function"] = seq2seq.simple_decoder_fn_inference(output_fn, enc_state, self.embedding_matrix, SOS_id, EOS_id, maximum_length=self.max_dec_len, num_decoder_symbols=self.vocab_size) return components[component_name]
def _init_decoder(self): """ decoder cell. attention적용 시 결과가 좋지 않음. """ with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=self.len_max, num_decoder_symbols=self.vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = (seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=[ self.len_max for _ in range(self.batch_size) ], time_major=True, scope=scope, )) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def decoder_inference_layer(self, encoded_state, token_embedding): num_tokens = self.data.token_vocab_size with tf.variable_scope('decoder') as scope: scope.reuse_variables() dynamic_fn_test = seq2seq.simple_decoder_fn_inference( None, encoded_state, self.token_embedding_matrix, self.data.GO_CODE, self.data.STOP_CODE, self.data.token_sequence_length - 2, num_tokens) decoder_outputs, state, context = seq2seq.dynamic_rnn_decoder( self.decoder_cell, dynamic_fn_test) return decoder_outputs
def decode_inference(cell, embeddings, encoder_state, output_fn, vocab_size, bos_id, eos_id, max_length, scope='decoder', reuse=None): """ Args: cell: An RNNCell object embeddings: An embedding matrix with shape (vocab_size, word_dim) encoder_state: A tensor that contains the encoder state; its shape should match that of cell.zero_state output_fn: A function that projects a vector with length cell.output_size into a vector with length vocab_size; please beware of the scope, since it will be called inside 'scope/rnn' scope vocab_size: The size of a vocabulary set bos_id: The ID of the beginning-of-sentence symbol eos_id: The ID of the end-of-sentence symbol max_length: The maximum length of a generated sentence; it stops generating words when this number of words are generated and <EOS> is not appeared till then scope: A VariableScope object of a string which indicates the scope reuse: A boolean value or None which specifies whether to reuse variables already defined in the scope Returns: generated, which is a float32 (batch, <=max_len) tensor that contains IDs of generated words """ with tf.variable_scope(scope, initializer=tf.orthogonal_initializer(), reuse=reuse): decoder_fn = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=encoder_state, embeddings=embeddings, start_of_sequence_id=bos_id, end_of_sequence_id=eos_id, maximum_length=max_length, num_decoder_symbols=vocab_size) generated_logits, _, _ = seq2seq.dynamic_rnn_decoder( cell=cell, decoder_fn=decoder_fn, time_major=False, scope='rnn') generated = tf.argmax(generated_logits, axis=2) return generated
def decoder(self,encoder_state,inputs=None,is_train=True): ''' 解码器 ''' with tf.variable_scope("decoder") as scope: if is_train is True: decoder_fn=seq2seq.simple_decoder_fn_train(encoder_state) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=inputs,sequence_length=self.seq_len,time_major=False,scope=scope) else: tf.get_variable_scope().reuse_variables() #解码时,通过decoder embedding和decoder bias计算每个词的概率 output_fn=lambda x:tf.nn.softmax(tf.matmul(x,self.dec_embedding,transpose_b=True)+self.dec_bias) decoder_fn=seq2seq.simple_decoder_fn_inference(output_fn=output_fn,encoder_state=encoder_state,embeddings=self.embedding, start_of_sequence_id=0,end_of_sequence_id=0,maximum_length=self.subject_len, num_decoder_symbols=self.vocab_size,dtype=tf.int32) outputs,final_state,final_context_state=seq2seq.dynamic_rnn_decoder(self.decoder_cell,decoder_fn=decoder_fn,inputs=None,sequence_length=self.seq_len,time_major=False,scope=scope) return outputs,final_state,final_context_state
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): self.test_outputs = outputs return tf.contrib.layers.linear(outputs, self.decoder_vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train(encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder' ) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.decoder_embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 100, num_decoder_symbols=self.decoder_vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=self.time_major, scope=scope, ) ) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax(self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=self.time_major, scope=scope, ) ) self.decoder_prediction_inference = tf.argmax(self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def __init__(self, para): para.fac = int(para.bidirectional) + 1 self._para = para if para.rnn_type == 0: #basic rnn def unit_cell(fac): return tf.contrib.rnn.BasicRNNCell(para.hidden_size * fac) elif para.rnn_type == 1: #basic LSTM def unit_cell(fac): return tf.contrib.rnn.BasicLSTMCell(para.hidden_size * fac) elif para.rnn_type == 2: #full LSTM def unit_cell(fac): return tf.contrib.rnn.LSTMCell(para.hidden_size * fac, use_peepholes=True) elif para.rnn_type == 3: #GRU def unit_cell(fac): return tf.contrib.rnn.GRUCell(para.hidden_size * fac) rnn_cell = unit_cell #dropout layer if not self.is_test() and para.keep_prob < 1: def rnn_cell(fac): return tf.contrib.rnn.DropoutWrapper( unit_cell(fac), output_keep_prob=para.keep_prob) #multi-layer rnn encoder_cell =\ tf.contrib.rnn.MultiRNNCell([rnn_cell(1) for _ in range(para.layer_num)]) if para.bidirectional: b_encoder_cell = tf.contrib.rnn.MultiRNNCell( [rnn_cell(1) for _ in range(para.layer_num)]) #feed in data in batches if not self.is_test(): video, caption, v_len, c_len = self.get_single_example(para) videos, captions, v_lens, c_lens =\ tf.train.batch([video, caption, v_len, c_len], batch_size=para.batch_size, dynamic_pad=True) #sparse tensor cannot be sliced targets = tf.sparse_tensor_to_dense(captions) decoder_in = targets[:, :-1] decoder_out = targets[:, 1:] c_lens = tf.to_int32(c_lens) else: video, v_len = self.get_single_example(para) videos, v_lens =\ tf.train.batch([video, v_len], batch_size=para.batch_size, dynamic_pad=True) v_lens = tf.to_int32(v_lens) with tf.variable_scope('embedding'): if para.use_pretrained: W_E =\ tf.Variable(tf.constant(0., shape= [para.vocab_size, para.w_emb_dim]), trainable=False, name='W_E') self._embedding = tf.placeholder( tf.float32, [para.vocab_size, para.w_emb_dim]) self._embed_init = W_E.assign(self._embedding) else: W_E = tf.get_variable('W_E', [para.vocab_size, para.w_emb_dim], dtype=tf.float32) if not self.is_test(): decoder_in_embed = tf.nn.embedding_lookup(W_E, decoder_in) if para.v_emb_dim < para.video_dim: inputs = fully_connected(videos, para.v_emb_dim) else: inputs = videos if not self.is_test() and para.keep_prob < 1: inputs = tf.nn.dropout(inputs, para.keep_prob) if not para.bidirectional: encoder_outputs, encoder_states =\ tf.nn.dynamic_rnn(encoder_cell, inputs, sequence_length=v_lens, dtype=tf.float32) else: encoder_outputs, encoder_states =\ tf.nn.bidirectional_dynamic_rnn(encoder_cell, b_encoder_cell, inputs, sequence_length=v_lens, dtype=tf.float32) encoder_states = tuple([ LSTMStateTuple(tf.concat([f_st.c, f_st.c], 1), tf.concat([b_st.h, b_st.h], 1)) for f_st, b_st in zip(encoder_states[0], encoder_states[1]) ]) encoder_outputs = tf.concat( [encoder_outputs[0], encoder_outputs[1]], 2) with tf.variable_scope('softmax'): softmax_w = tf.get_variable( 'w', [para.hidden_size * para.fac, para.vocab_size], dtype=tf.float32) softmax_b = tf.get_variable('b', [para.vocab_size], dtype=tf.float32) output_fn = lambda output: tf.nn.xw_plus_b(output, softmax_w, softmax_b) decoder_cell =\ tf.contrib.rnn.MultiRNNCell([rnn_cell(para.fac) for _ in range(para.layer_num)]) if para.attention > 0: at_option = ["bahdanau", "luong"][para.attention - 1] at_keys, at_vals, at_score, at_cons =\ seq2seq.prepare_attention(attention_states=encoder_outputs, attention_option=at_option, num_units=para.hidden_size*para.fac) if self.is_test(): if para.attention: decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=encoder_states, attention_keys=at_keys, attention_values=at_vals, attention_score_fn=at_score, attention_construct_fn=at_cons, embeddings=W_E, start_of_sequence_id=2, end_of_sequence_id=3, maximum_length=20, num_decoder_symbols=para.vocab_size) else: decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=encoder_states, embeddings=W_E, start_of_sequence_id=2, end_of_sequence_id=3, maximum_length=20, num_decoder_symbols=para.vocab_size) with tf.variable_scope('decode', reuse=True): decoder_logits, _, _ =\ seq2seq.dynamic_rnn_decoder(cell=decoder_cell, decoder_fn=decoder_fn_inference) self._prob = tf.nn.softmax(decoder_logits) else: global_step = tf.contrib.framework.get_or_create_global_step() def decoder_fn_train(time, cell_state, cell_input, cell_output, context): if para.scheduled_sampling and cell_output is not None: epsilon = tf.cast( 1 - (global_step // (para.tot_train_num // para.batch_size + 1) / para.max_epoch), tf.float32) cell_input = tf.cond( tf.less(tf.random_uniform([1]), epsilon)[0], lambda: cell_input, lambda: tf.gather( W_E, tf.argmax(output_fn(cell_output), 1))) if cell_state is None: cell_state = encoder_states if para.attention: attention = _init_attention(encoder_states) else: if para.attention: cell_output = attention = at_cons( cell_output, at_keys, at_vals) if para.attention: nxt_cell_input = tf.concat([cell_input, attention], 1) else: nxt_cell_input = cell_input return None, encoder_states, nxt_cell_input, cell_output, context with tf.variable_scope('decode', reuse=None): (decoder_outputs, _, _) =\ seq2seq.dynamic_rnn_decoder(cell=decoder_cell, decoder_fn=decoder_fn_train, inputs=decoder_in_embed, sequence_length=c_lens) decoder_outputs =\ tf.reshape(decoder_outputs, [-1, para.hidden_size*para.fac]) c_len_max = tf.reduce_max(c_lens) logits = output_fn(decoder_outputs) logits = tf.reshape(logits, [para.batch_size, c_len_max, para.vocab_size]) self._prob = tf.nn.softmax(logits) msk = tf.sequence_mask(c_lens, dtype=tf.float32) loss = sequence_loss(logits, decoder_out, msk) self._cost = cost = tf.reduce_mean(loss) #if validation or testing, exit here if self.is_valid(): return #clip global gradient norm tvars = tf.trainable_variables() grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), para.max_grad_norm) optimizer = optimizers[para.optimizer](para.learning_rate) self._eval = optimizer.apply_gradients(zip(grads, tvars), global_step=global_step)
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) #attention_states = tf.zeros([batch_size, 1, self.decoder_hidden_units]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=data_utils.GO_ID, end_of_sequence_id=data_utils.EOS_ID, maximum_length=FLAGS.max_inf_target_len, num_decoder_symbols=self.vocab_size, ) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = (seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope, )) self.decoder_outputs_train = tf.nn.dropout( self.decoder_outputs_train, _keep_prob) self.decoder_logits_train = output_fn(self.decoder_outputs_train) # reusing the scope of training to use the same variables for inference scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def decoder_adv(self, max_twee_len): with self.graph.as_default(): with tf.variable_scope("Decoder") as scope: self.decoder_length = max_twee_len + 3 def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.vocab_size, scope=scope) # self.decoder_cell = LSTMCell(self.decoder_hidden_nodes) self.decoder_cell = GRUCell(self.decoder_hidden_nodes) if not self.attention: decoder_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_final_state) decoder_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=self.decoder_length, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] self.attention_states = tf.transpose( self.encoder_output, [1, 0, 2]) (self.attention_keys, self.attention_values, self.attention_score_fn, self.attention_construct_fn) = \ seq2seq.prepare_attention(attention_states = self.attention_states, attention_option = "bahdanau", num_units = self.decoder_hidden_nodes) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, name="attention_decoder") decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_final_state, attention_keys=self.attention_keys, attention_values=self.attention_values, attention_score_fn=self.attention_score_fn, attention_construct_fn=self.attention_construct_fn, embeddings=self.embed, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length= 23, #max_twee_len + 3, #tf.reduce_max(self.de_out_len) + 3, num_decoder_symbols=self.vocab_size) self.decoder_train_inputs_embedded = tf.nn.embedding_lookup( self.embed, self.decoder_train_input) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope)) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope)) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference') return self.de_out, self.de_out_len, self.title_out, self.first_out, self.decoder_logits_train, \ self.decoder_prediction_train, self.loss_weights, self.decoder_train_targets, \ self.decoder_train_title, self.decoder_train_first, self.decoder_prediction_inference
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear( outputs, self.vocab_size, scope=scope ) #this is for calculatng outputs. In a greedy way if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state ) #This is the training function that we used in training dynamic_rnn_decoder #refer to https://github.com/tensorflow/tensorflow/blob/r1.0/tensorflow/contrib/seq2seq/python/ops/decoder_fn.py#L182 decoder_fn_inference = seq2seq.simple_decoder_fn_inference( #nference function for a sequence-to-sequence model. It should be used when dynamic_rnn_decoder is in the inference mode.final mode output_fn= output_fn, #this returns a decoder function . This function in used inside the dynamicRNN function encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [ 1, 0, 2 ]) #take the attention status as the encorder hidden states ( attention_keys, #Each Encoder hidden status multiplied in fully conected way and list of size [num units*Max_time] attention_values, #this is attention encoder states attention_score_fn, #score function of the attention Different ways to compute attention scores If we input the decoder state , encoder hidden states this will out put the context vector attention_construct_fn ) = seq2seq.prepare_attention( #this contruct will Function to compute attention vectors. This will output the concatanaded context vector and the attention wuary then make it as a inpit attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) print("Prininting the number of units .......................") print(self.decoder_hidden_units) print( "Printing the shape of the attetniton values ......................**********************************************" ) print(attention_keys) print( "Printing the attention score function++++++++++++++++++++++++++++++++++++++++++++++++++++" ) print(attention_score_fn) #this function can basically initialize input state of the decoder the nthe attention and other stuff then this will be passed to dy_decorder #decorder_function train will take time, cell_state, cell_input, cell_output, context_state decoder_fn_train = seq2seq.attention_decoder_fn_train( #this is for training the dynamic decorder. This will take care of encoder_state=self. encoder_state, # final state. We take the biderection and concatanate it (c or h) attention_keys= attention_keys, # The transformation of each encoder outputs attention_values= attention_values, #attention encododr status attention_score_fn= attention_score_fn, #this will give a context vector attention_construct_fn= attention_construct_fn, #calculating above thinhs also output the hidden state name='attention_decoder') #What can we achieve by running decorder_fn_ ? done, next state, next input, emit output, next context state #here the emit_output or cell_output will give the output of cell after all atention - non lieanrity applied #this also give the hidden vector output which was concatanated with rnn output and attention vector . Actually concatanated goes throug a linear unit #next_input = array_ops.concat([cell_input, attention], 1) #next cell input #context_state - this will modify when using the beam search #what is the contect state in decorder_fn inside the return funfction of the decorder fn train #the following function is same as the above but the only difference is it's use this in the inference .This has a greedy output #in the inference model cell_output = output_fn(cell_output) . Which means we get logits #next_input = array_ops.concat([cell_input, attention], 1) decoder_fn_inference = seq2seq.attention_decoder_fn_inference( #this is used in the inference model output_fn= output_fn, #this will predict the output and the narcmax after that attention will be concatenaded encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, #doing same attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size, ) #following function is to do all the decodinf with the helop of above functions #this can use in traning or inferense . But we need two separate finctions for trainin and iference #What is this context_state_train : one way to diversify the inference output is to use a stochastic decoder_fn, in which case one would want to store the decoded outputs, not just the RNN outputs. This can be done by maintaining a TensorArray in context_state and storing the decoded output of each iteration therein ( self. decoder_outputs_train, #outputs from the eacah cell [batch_size, max_time, cell.output_size] self. decoder_state_train, #The final state and will be shaped [batch_size, cell.state_size] self.decoder_context_state_train ) = ( #described above seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_train, #decoder_fn allows modeling of early stopping, output, state, and next input and context. inputs=self. decoder_train_inputs_embedded, #inputs to the decoder in the training #in the raning time only sequence_length=self. decoder_train_length, #sequence_length is needed at training time, i.e., when inputs is not None, for dynamic unrolling. At test time, when inputs is None, sequence_length is not needed. time_major= True, #input and output shape should be in [max_time, batch_size, ...] scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train ) #take the final output hidden status and run them throgh linearl layer #get the argmax self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() ( self. decoder_logits_inference, #same as above but no input provided. This will take the predicted things as inputs self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn= decoder_fn_inference, #difference decorder fucntion time_major=True, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference' ) #predicted output at the each time step
def _build_graph(self): # required only for training self.targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name="decoder_inputs") self.targets_length = tf.placeholder(shape=(None, ), dtype=tf.int32, name="decoder_inputs_length") self.global_step = tf.Variable(0, name="global_step", trainable=False) with tf.name_scope("DecoderTrainFeed"): sequence_size, batch_size = tf.unstack(tf.shape(self.targets)) EOS_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.EOS PAD_SLICE = tf.ones([1, batch_size], dtype=tf.int32) * self.PAD self.train_inputs = tf.concat([EOS_SLICE, self.targets], axis=0) self.train_length = self.targets_length + 1 train_targets = tf.concat([self.targets, PAD_SLICE], axis=0) train_targets_seq_len, _ = tf.unstack(tf.shape(train_targets)) train_targets_eos_mask = tf.one_hot(self.train_length - 1, train_targets_seq_len, on_value=self.EOS, off_value=self.PAD, dtype=tf.int32) train_targets_eos_mask = tf.transpose(train_targets_eos_mask, [1, 0]) # hacky way using one_hot to put EOS symbol at the end of target sequence train_targets = tf.add(train_targets, train_targets_eos_mask) self.train_targets = train_targets self.loss_weights = tf.ones( [batch_size, tf.reduce_max(self.train_length)], dtype=tf.float32, name="loss_weights") with tf.variable_scope("embedding") as scope: self.inputs_embedded = tf.nn.embedding_lookup( self.embedding_matrix, self.train_inputs) with tf.variable_scope("Decoder") as scope: def logits_fn(outputs): return layers.linear(outputs, self.vocab_size, scope=scope) if not self.attention: train_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) inference_fn = seq2seq.simple_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.decoder_hidden_units) train_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name="decoder_attention") inference_fn = seq2seq.attention_decoder_fn_inference( output_fn=logits_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.vocab_size) (self.train_outputs, self.train_state, self.train_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=train_fn, inputs=self.inputs_embedded, sequence_length=self.train_length, time_major=True, scope=scope) self.train_logits = logits_fn(self.train_outputs) self.train_prediction = tf.argmax(self.train_logits, axis=-1, name="train_prediction") self.train_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="train_prediction_probabilities") scope.reuse_variables() (self.inference_logits, self.inference_state, self.inference_context_state) = seq2seq.dynamic_rnn_decoder( cell=self.cell, decoder_fn=inference_fn, time_major=True, scope=scope) self.inference_prediction = tf.argmax(self.inference_logits, axis=-1, name="inference_prediction") self.inference_prediction_probabilities = tf.nn.softmax( self.train_logits, dim=-1, name="inference_prediction_probabilities")
def _init_decoder(self): with tf.variable_scope("Decoder") as scope: def output_fn(outputs): return tc.layers.fully_connected(outputs, self.output_symbol_size, activation_fn=None, scope=scope) if not self.attention: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size) else: (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=self.encoder_outputs, attention_option="bahdanau", num_units=self.decoder_hidden_units, ) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embedding_matrix, start_of_sequence_id=self.EOS, end_of_sequence_id=self.EOS, maximum_length=tf.reduce_max(self.encoder_inputs_length), num_decoder_symbols=self.output_symbol_size, ) if self.is_training: (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=False, scope=scope, )) self.decoder_logits_train = output_fn( self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_train') scope.reuse_variables() (self.decoder_logits_inference, self.decoder_state_inference, self.decoder_context_state_inference) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=False, scope=scope, )) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def __init_decoder(self): '''Initializes the decoder part of the model.''' with tf.variable_scope('decoder') as scope: output_fn = lambda outs: layers.linear( outs, self.__get_vocab_size(), scope=scope) if self.cfg.get('use_attention'): attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = seq2seq.prepare_attention( attention_states=attention_states, attention_option='bahdanau', num_units=self.decoder_cell.output_size) decoder_fn_train = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') decoder_fn_inference = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) else: decoder_fn_train = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) decoder_fn_inference = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.embeddings, start_of_sequence_id=Config.EOS_WORD_IDX, end_of_sequence_id=Config.EOS_WORD_IDX, maximum_length=tf.reduce_max(self.encoder_inputs_length) + 3, num_decoder_symbols=self.__get_vocab_size()) (self.decoder_outputs_train, self.decoder_state_train, self.decoder_context_state_train) = seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn_train, inputs=self.decoder_train_inputs_embedded, sequence_length=self.decoder_train_length, time_major=True, scope=scope) self.decoder_logits_train = output_fn(self.decoder_outputs_train) self.decoder_prediction_train = tf.argmax( self.decoder_logits_train, axis=-1, name='decoder_prediction_traion') scope.reuse_variables() (self.decoder_logits_inference, decoder_state_inference, self.decoder_context_state_inference ) = seq2seq.dynamic_rnn_decoder(cell=self.decoder_cell, decoder_fn=decoder_fn_inference, time_major=True, scope=scope) self.decoder_prediction_inference = tf.argmax( self.decoder_logits_inference, axis=-1, name='decoder_prediction_inference')
def _init_decoder(self, forward_only): with tf.variable_scope("decoder") as scope: def output_fn(outputs): return tf.contrib.layers.linear(outputs, self.target_vocab_size, scope=scope) self.attention = True if not self.attention: if forward_only: decoder_fn = seq2seq.simple_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.simple_decoder_fn_train( encoder_state=self.encoder_state) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) else: # attention_states: size [batch_size, max_time, num_units] attention_states = tf.transpose(self.encoder_outputs, [1, 0, 2]) (attention_keys, attention_values, attention_score_fn, attention_construct_fn) = (seq2seq.prepare_attention( attention_states=attention_states, attention_option="bahdanau", num_units=self.dec_hidden_size)) if forward_only: decoder_fn = seq2seq.attention_decoder_fn_inference( output_fn=output_fn, encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, embeddings=self.dec_embedding_matrix, start_of_sequence_id=model_config.GO_ID, end_of_sequence_id=model_config.EOS_ID, maximum_length=self.buckets[-1][1], num_decoder_symbols=self.target_vocab_size, ) (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, time_major=True, scope=scope, )) else: decoder_fn = seq2seq.attention_decoder_fn_train( encoder_state=self.encoder_state, attention_keys=attention_keys, attention_values=attention_values, attention_score_fn=attention_score_fn, attention_construct_fn=attention_construct_fn, name='attention_decoder') (self.decoder_outputs, self.decoder_state, self.decoder_context_state) = ( seq2seq.dynamic_rnn_decoder( cell=self.decoder_cell, decoder_fn=decoder_fn, inputs=self.decoder_inputs_embedded, sequence_length=self.decoder_inputs_length, time_major=True, scope=scope, )) if not forward_only: self.decoder_logits = output_fn(self.decoder_outputs) else: self.decoder_logits = self.decoder_outputs self.decoder_prediction = tf.argmax(self.decoder_logits, axis=-1, name='decoder_prediction') logits = tf.transpose(self.decoder_logits, [1, 0, 2]) targets = tf.transpose(self.decoder_targets, [1, 0]) if not forward_only: self.loss = seq2seq.sequence_loss(logits=logits, targets=targets, weights=self.target_weights)