def target_step(x_t, h_tm1, h2_tm1, c, c2):
			#print 'z_t and r_t are combined!'
			all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgx) + T.dot(h_tm1, self.Ugh) + T.dot(c, self.Wgc) + self.bg)
			z_t = myutil.slice(all_t, 0, nh)
			r_t = myutil.slice(all_t, 1, nh)
			# candidate h_t
			ch_t = myutil.activation(activation, T.dot(x_t, self.Whx) + T.dot(r_t * h_tm1, self.Uhh) + T.dot(c, self.Whc) + self.bh)
			h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
			# second layer
			all2_t = T.nnet.sigmoid(T.dot(h_t, self.Wg2h) + T.dot(h2_tm1, self.Ug2h2) + T.dot(c2, self.Wg2c2) + self.bg2)
			z2_t = myutil.slice(all2_t, 0, nh2)
			r2_t = myutil.slice(all2_t, 1, nh2)
			ch2_t = myutil.activation(activation, T.dot(h_t, self.Wh2h) + T.dot(r2_t * h2_tm1, self.Uh2h2) + T.dot(c2, self.Wh2c2) + self.bh2)
			h2_t = (1.0 - z2_t) * h2_tm1 + z2_t * ch2_t
			return [h_t, h2_t]
		def source_step(x_t, h_tm1, h2_tm1):
			#print 'z_t and r_t are combined!'
			all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgx_src) + T.dot(h_tm1, self.Ugh_src) + self.bg_src)
			z_t = myutil.slice(all_t, 0, nh)
			r_t = myutil.slice(all_t, 1, nh)
			# candidate h_t
			ch_t = myutil.activation(activation, T.dot(x_t, self.Whx_src) + T.dot(r_t * h_tm1, self.Uhh_src) + self.bh_src)
			h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
			# second layer
			all2_t = T.nnet.sigmoid(T.dot(h_t, self.Wg2h_src) + T.dot(h2_tm1, self.Ug2h2_src) + self.bg2_src)
			z2_t = myutil.slice(all2_t, 0, nh2)
			r2_t = myutil.slice(all2_t, 1, nh2)
			ch2_t = myutil.activation(activation, T.dot(h_t, self.Wh2h_src) + T.dot(r2_t * h2_tm1, self.Uh2h2_src) + self.bh2_src)
			h2_t = (1.0 - z2_t) * h2_tm1 + z2_t * ch2_t
			return [h_t, h2_t]
 def target_step(dot_x_t_Wgx, dot_x_t_Whx, dot_x_t_Wax, h_tm1, h_src,
                 dot_h_src_Ua):
     # search c_t
     #z = T.tanh(T.dot(h_tm1, self.Wa) + T.dot(h_src, self.Ua) + self.ba)
     #z = T.tanh(T.dot(h_tm1, self.Wa) + dot_h_src_Ua)
     z = T.tanh(T.dot(h_tm1, self.Wah) + dot_x_t_Wax + dot_h_src_Ua)
     #print 'z:', z.ndim
     e = T.dot(self.va, z.T)
     #print 'e:', e.ndim
     max_e = T.max(e)
     exp_e = T.exp(e - max_e)
     a = exp_e / exp_e.sum()
     #print 'a:', a.ndim
     c_t = T.dot(a, h_src)
     #print 'c_t:', c_t.ndim
     #print 'z_t and r_t are combined!'
     all_t = T.nnet.sigmoid(dot_x_t_Wgx + T.dot(h_tm1, self.Ugh) +
                            T.dot(c_t, self.Wgc))
     z_t = myutil.slice(all_t, 0, nh)
     r_t = myutil.slice(all_t, 1, nh)
     # candidate h_t
     ch_t = myutil.activation(
         activation, dot_x_t_Whx + T.dot(r_t * h_tm1, self.Uhh) +
         T.dot(c_t, self.Whc))
     h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
     return [h_t, c_t]
 def source_step(dot_x_t_Wgx_src, dot_x_t_Whx_src, h_tm1):
     #print 'z_t, r_t are combined!'
     #all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgx_src) + T.dot(h_tm1, self.Ugh_src) + self.bg_src)
     all_t = T.nnet.sigmoid(dot_x_t_Wgx_src +
                            T.dot(h_tm1, self.Ugh_src))
     z_t = myutil.slice(all_t, 0, nh)
     r_t = myutil.slice(all_t, 1, nh)
     # candidate h_t
     #ch_t = myutil.activation(activation, T.dot(x_t, self.Whx_src) + T.dot(r_t * h_tm1, self.Uhh_src) + self.bh_src)
     ch_t = myutil.activation(
         activation, dot_x_t_Whx_src + T.dot(r_t * h_tm1, self.Uhh_src))
     h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
     return h_t
		def make_score(x, y, use_noise):
			# input layer dropout: ex. [0.2, 0.2, 0.5]
			if use_noise:
				print "X's projection layer dropout:", dropout_rate[0]
				dropout_x = myutil.dropout_from_layer(x, dropout_rate[0])
			else:
				dropout_x = x * (1.0 - dropout_rate[0])
			# recurrent for source language
			[h_src, h2_src], _ = theano.scan(fn=source_step,
								sequences=dropout_x,
								outputs_info=[self.h0_src, self.h20_src],
								n_steps=dropout_x.shape[0])
			# context
			c = h_src[-1]
			c2 = h2_src[-1]
			h0 = myutil.activation(activation, T.dot(c, self.Wh0c) + self.bh0)
			h20 = myutil.activation(activation, T.dot(c2, self.Wh20c2) + self.bh20)
			# output layer dropout: ex. [0.2, 0.2, 0.5]
			if use_noise:
				print "Y's projection layer dropout:", dropout_rate[1]
				dropout_y = myutil.dropout_from_layer(y, dropout_rate[1])
			else:
				dropout_y = y * (1.0 - dropout_rate[1])
			# forward recurrent for target language
			[h, h2], _ = theano.scan(fn=target_step,
								sequences=dropout_y,
								outputs_info=[h0, h20],
								non_sequences=[c, c2],
								n_steps=dropout_y.shape[0])
			# hidden layer dropout
			if use_noise:
				print "Y's hidden layer dropout:", dropout_rate[2]
				dropout_h2 = myutil.dropout_from_layer(h2, dropout_rate[2])
			else:
				dropout_h2 = h2 * (1.0 - dropout_rate[2])
			# score
			score = T.dot(dropout_h2, self.Wyh2) + T.dot(dropout_y, self.Wyy) + T.dot(c2, self.Wyc2) + self.by
			return score, h_src, h2_src, h0, h20
 def target_step(x_t, h_tm1, c):
     #print 'z_t and r_t are combined!'
     all_t = T.nnet.sigmoid(
         T.dot(x_t, self.Wgx) + T.dot(h_tm1, self.Ugh) +
         T.dot(c, self.Wgc) + self.bg)
     z_t = myutil.slice(all_t, 0, nh)
     r_t = myutil.slice(all_t, 1, nh)
     # candidate h_t
     ch_t = myutil.activation(
         activation,
         T.dot(x_t, self.Whx) + T.dot(r_t * h_tm1, self.Uhh) +
         T.dot(c, self.Whc) + self.bh)
     h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
     return h_t
		def greedy_search_step(h_tm1, h2_tm1, y_tm1, h_src, h2_src):
			x_t = self.emb[y_tm1]
			c = h_src[-1]
			c2 = h2_src[-1]
			#print 'z_t and r_t are combined!'
			all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgx) + T.dot(h_tm1, self.Ugh) + T.dot(c, self.Wgc) + self.bg)
			z_t = myutil.slice(all_t, 0, nh)
			r_t = myutil.slice(all_t, 1, nh)
			# candidate h_t
			ch_t = myutil.activation(activation, T.dot(x_t, self.Whx) + T.dot(r_t * h_tm1, self.Uhh) + T.dot(c, self.Whc) + self.bh)
			h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
			# second layer
			all2_t = T.nnet.sigmoid(T.dot(h_t, self.Wg2h) + T.dot(h2_tm1, self.Ug2h2) + T.dot(c2, self.Wg2c2) + self.bg2)
			z2_t = myutil.slice(all2_t, 0, nh2)
			r2_t = myutil.slice(all2_t, 1, nh2)
			ch2_t = myutil.activation(activation, T.dot(h_t, self.Wh2h) + T.dot(r2_t * h2_tm1, self.Uh2h2) + T.dot(c2, self.Wh2c2) + self.bh2)
			h2_t = (1.0 - z2_t) * h2_tm1 + z2_t * ch2_t
			# score
			s = T.dot(h2_t, self.Wyh2) + T.dot(x_t, self.Wyy) + T.dot(c2, self.Wyc2) + self.by
	   		max_s, y_t = T.max_and_argmax(s)
			exp_s = T.exp(s - max_s)
			p_y = exp_s / exp_s.sum()
			log_p_y = T.log(exp_s / exp_s.sum())
			return [h_t, h2_t, y_t, log_p_y], theano.scan_module.until(T.eq(y_t,1)) # 1 --> '</s>'
    def __init__(self, hyper_param, word2idx_dic):
        '''
		nh :: dimension of the hidden layer
		ne :: number of word embeddings in the vocabulary
		de :: dimension of the word embeddings
		nf :: number of feature
		nfe:: number of feature embeddings in the vocabulary - by leeck
		dfe:: dimension of the feature embeddings - by leeck
		cs :: word window context size
		emb_file :: word embedding file
		weight_decay :: weight decay
		dropout_rate :: dropout rate
		activation :: activation function: simg, tanh, relu
		word2idx_dic :: word to index dictionary
		'''
        self.hyper_param = hyper_param
        nh = hyper_param['nhidden']
        ne = hyper_param['vocsize']
        de = hyper_param['emb_dimension']
        weight_decay = hyper_param['weight_decay']
        dropout_rate = hyper_param['dropout_rate']
        activation = hyper_param['activation']
        learning_method = hyper_param['learning_method']
        verbose = False
        # parameters of the model
        if hyper_param['load_model'] != '':
            self.load_param(hyper_param, hyper_param['load_model'])
        else:
            self.build_param(hyper_param, word2idx_dic)

        # parameters
        self.params = [self.emb, self.Wgx_src, self.Wgx, self.Wgc, \
          self.Whx_src, self.Whx, self.Whc, self.Wh0c, self.Ugh_src, self.Ugh, \
          self.Uhh_src, self.Uhh, self.Wyh, self.Wyc, self.Wyy, \
          self.bg_src, self.bg, self.bh_src, self.bh, self.bh0, self.by, \
          self.h0_src]

        if hyper_param['fixed_emb']:
            print 'fixed embeddig.'
            self.params.remove(self.emb)

        # as many columns as context window size
        # as many lines as words in the sentence
        x_sentence = T.ivector('x_sentence')  # x_sentence : n_steps
        x_org = self.emb[x_sentence].reshape((x_sentence.shape[0], de))
        x = x_org[:-1]  # remove '</s>'
        x_reverse = x[::-1]  # reverse for backward

        y_sentence = T.ivector('y_sentence')  # labels
        y_input_sentence = T.concatenate([y_sentence[-1:], y_sentence[:-1]],
                                         axis=0)
        y = self.emb[y_input_sentence].reshape((y_input_sentence.shape[0], de))

        # for scan
        def source_step(x_t, h_tm1):
            #print 'z_t and r_t are combined!'
            all_t = T.nnet.sigmoid(
                T.dot(x_t, self.Wgx_src) + T.dot(h_tm1, self.Ugh_src) +
                self.bg_src)
            z_t = myutil.slice(all_t, 0, nh)
            r_t = myutil.slice(all_t, 1, nh)
            # candidate h_t
            ch_t = myutil.activation(
                activation,
                T.dot(x_t, self.Whx_src) + T.dot(r_t * h_tm1, self.Uhh_src) +
                self.bh_src)
            h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
            return h_t

        def target_step(x_t, h_tm1, c):
            #print 'z_t and r_t are combined!'
            all_t = T.nnet.sigmoid(
                T.dot(x_t, self.Wgx) + T.dot(h_tm1, self.Ugh) +
                T.dot(c, self.Wgc) + self.bg)
            z_t = myutil.slice(all_t, 0, nh)
            r_t = myutil.slice(all_t, 1, nh)
            # candidate h_t
            ch_t = myutil.activation(
                activation,
                T.dot(x_t, self.Whx) + T.dot(r_t * h_tm1, self.Uhh) +
                T.dot(c, self.Whc) + self.bh)
            h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
            return h_t

        # make score, h_src, h0 (for beam search)
        def make_score(x, y, use_noise):
            # input layer dropout: ex. [0.2, 0.2, 0.5]
            if use_noise:
                print "X's projection layer dropout:", dropout_rate[0]
                dropout_x = myutil.dropout_from_layer(x, dropout_rate[0])
            else:
                dropout_x = x * (1.0 - dropout_rate[0])
            # recurrent for source language
            h_src, _ = theano.scan(fn=source_step,
                                   sequences=dropout_x,
                                   outputs_info=self.h0_src,
                                   n_steps=dropout_x.shape[0])
            # context
            c = h_src[-1]
            h0 = myutil.activation(activation, T.dot(c, self.Wh0c) + self.bh0)
            # output layer dropout: ex. [0.2, 0.2, 0.5]
            if use_noise:
                print "Y's projection layer dropout:", dropout_rate[1]
                dropout_y = myutil.dropout_from_layer(y, dropout_rate[1])
            else:
                dropout_y = y * (1.0 - dropout_rate[1])
            # forward recurrent for target language
            h, _ = theano.scan(fn=target_step,
                               sequences=dropout_y,
                               outputs_info=h0,
                               non_sequences=[c],
                               n_steps=dropout_y.shape[0])
            # hidden layer dropout
            if use_noise:
                print "Y's hidden layer dropout:", dropout_rate[2]
                dropout_h = myutil.dropout_from_layer(h, dropout_rate[2])
            else:
                dropout_h = h * (1.0 - dropout_rate[2])
            # score
            score = T.dot(dropout_h, self.Wyh) + T.dot(
                dropout_y, self.Wyy) + T.dot(c, self.Wyc) + self.by
            return score, h_src, h0

        # dropout version (for training)
        if 'reverse_input' in hyper_param and hyper_param['reverse_input']:
            print 'reverse input.'
            dropout_score, _, _ = make_score(x_reverse, y, True)
        else:
            dropout_score, _, _ = make_score(x, y, True)
        dropout_p_y_given_x = myutil.mysoftmax(dropout_score)
        # scaled version (for prediction)
        if 'reverse_input' in hyper_param and hyper_param['reverse_input']:
            print 'reverse input.'
            score, h_src, h0 = make_score(x_reverse, y, False)
        else:
            score, h_src, h0 = make_score(x, y, False)
        p_y_given_x = myutil.mysoftmax(score)

        # prediction
        y_pred = T.argmax(p_y_given_x, axis=1)
        test_nll = -T.mean(
            T.log(p_y_given_x)[T.arange(y.shape[0]), y_sentence])

        # beam search decoding: input=[c, h_tm1, y_tm1], output=[h_t, log_p_y_t]
        input_h_src = T.fmatrix('input_h_src')
        input_h_tm1 = T.fvector('input_h_tm1')
        input_y_tm1 = T.iscalar('input_y_tm1')  # input_y_tm1 == x_t
        x_t = self.emb[input_y_tm1]
        c = input_h_src[-1]
        all_t = T.nnet.sigmoid(
            T.dot(x_t, self.Wgx) + T.dot(input_h_tm1, self.Ugh) +
            T.dot(c, self.Wgc) + self.bg)
        z_t = myutil.slice(all_t, 0, nh)
        r_t = myutil.slice(all_t, 1, nh)
        # candidate h_t
        ch_t = myutil.activation(
            activation,
            T.dot(x_t, self.Whx) + T.dot(r_t * input_h_tm1, self.Uhh) +
            T.dot(c, self.Whc) + self.bh)
        h_t = (1.0 - z_t) * input_h_tm1 + z_t * ch_t
        # prediction
        score_y_t = T.dot(h_t, self.Wyh) + T.dot(x_t, self.Wyy) + T.dot(
            c, self.Wyc) + self.by
        max_s = T.max(score_y_t)
        exp_s = T.exp(score_y_t - max_s)
        log_p_y_t = T.log(exp_s / exp_s.sum())

        # cost and gradients and learning rate
        lr = T.scalar('lr')  # for SGD

        # NLL + L2-norm
        nll = -T.mean(
            T.log(dropout_p_y_given_x)[T.arange(y.shape[0]), y_sentence])
        cost = nll
        for param in self.params:
            if param.name == 'emb':
                continue
            cost += weight_decay * T.sum(param**2)

        # SGD
        #gradients = T.grad(cost, self.params)
        #sgd_updates = OrderedDict((p, p - lr*g) for p, g in zip(self.params, gradients))
        sgd_updates = myutil.sgd_updates(self.params, cost, lr)
        # SGD + momentum (0.9)
        momentum_updates = myutil.sgd_updates_momentum(self.params, cost, lr,
                                                       0.9)
        # RMSProp (rho = 0.9)
        rmsprop_updates = myutil.sgd_updates_rmsprop(self.params, cost, lr,
                                                     0.9, 1)
        # AdaDelta (lr --> rho = 0.95)
        adadelta_updates = myutil.sgd_updates_adadelta(self.params, cost, lr,
                                                       1e-6, 9)

        # theano functions to compile
        self.classify = theano.function(inputs=[x_sentence, y_sentence],
                                        outputs=[y_pred, test_nll])
        # for beam search
        self.encoding_src_lang = theano.function(inputs=[x_sentence],
                                                 outputs=[h_src, h0])
        self.search_next_word = theano.function(
            inputs=[input_h_src, input_h_tm1, input_y_tm1],
            outputs=[log_p_y_t, h_t])
        # for reranking
        self.get_nll = theano.function(
            inputs=[x_sentence, input_h_src, input_h_tm1, y_sentence],
            outputs=test_nll,
            on_unused_input='ignore')
        # SGD
        self.train_sgd = theano.function(inputs=[x_sentence, y_sentence, lr],
                                         outputs=[cost, nll],
                                         updates=sgd_updates)
        # SGD with momentum
        self.train_momentum = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=momentum_updates)
        # RMSProp
        self.train_rmsprop = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=rmsprop_updates)
        # AdaDelta
        self.train_adadelta = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=adadelta_updates)
 def make_score(x, y, use_noise):
     # input layer dropout: ex. [0.2, 0.2, 0.5]
     if use_noise:
         print "X's projection layer dropout:", dropout_rate[0]
         dropout_x = myutil.dropout_from_layer(x, dropout_rate[0])
     else:
         dropout_x = x * (1.0 - dropout_rate[0])
     dropout_x_reverse = dropout_x[::-1]  # reverse for backward
     # RNN encoder
     dot_x_Wgx_src = T.dot(dropout_x, self.Wgx_src) + self.bg_src
     dot_x_Whx_src = T.dot(dropout_x, self.Whx_src) + self.bh_src
     dot_x_rev_Wgx_src = T.dot(dropout_x_reverse,
                               self.Wgxb_src) + self.bgb_src
     dot_x_rev_Whx_src = T.dot(dropout_x_reverse,
                               self.Whxb_src) + self.bhb_src
     # forward recurrent for source language
     hf_src, _ = theano.scan(fn=source_step,
                             sequences=[dot_x_Wgx_src, dot_x_Whx_src],
                             outputs_info=self.h0_src,
                             n_steps=dropout_x.shape[0])
     # backward recurrent for source language
     hb_src_reverse, _ = theano.scan(
         fn=source_backward_step,
         sequences=[dot_x_rev_Wgx_src, dot_x_rev_Whx_src],
         outputs_info=self.h0b_src,
         n_steps=dropout_x_reverse.shape[0])
     hb_src = hb_src_reverse[::-1]
     h_src = T.concatenate([hf_src, hb_src], axis=1)
     # global context
     #c_global = h_src[0]
     c_global = T.concatenate([hf_src[-1], hb_src[0]], axis=0)
     # output layer dropout: ex. [0.2, 0.2, 0.5]
     # output layer (target language input layer) dropout: ex. [0.2, 0.2, 0.5]
     if use_noise:
         print "Y's projection layer dropout:", dropout_rate[1]
         dropout_y = myutil.dropout_from_layer(y, dropout_rate[1])
     else:
         dropout_y = y * (1.0 - dropout_rate[1])
     # RNN decoder
     dot_y_Wgx = T.dot(dropout_y, self.Wgx) + self.bg
     dot_y_Whx = T.dot(dropout_y, self.Whx) + self.bh
     dot_y_Wax = T.dot(dropout_y, self.Wax)
     dot_h_src_Ua = T.dot(h_src, self.Ua) + self.ba
     h0 = myutil.activation(activation,
                            T.dot(c_global, self.Wh0c) + self.bh0)
     # forward recurrent for target language
     [h,
      c], _ = theano.scan(fn=target_step,
                          sequences=[dot_y_Wgx, dot_y_Whx, dot_y_Wax],
                          outputs_info=[h0, None],
                          non_sequences=[h_src, dot_h_src_Ua],
                          n_steps=dropout_y.shape[0])
     # h2 - Deep Output RNN
     print 'Deep Output RNN: ReLU'
     # hidden layer dropout
     if use_noise:
         print "Y's hidden layer dropout:", dropout_rate[2]
         dropout_h = myutil.dropout_from_layer(h, dropout_rate[2])
     else:
         dropout_h = h * (1.0 - dropout_rate[2])
     # h2 - Deep Output RNN
     print 'Deep Output RNN: ReLU'
     h2 = myutil.activation('relu',
                            T.dot(dropout_h, self.Wh2h) + self.bh2)
     # score
     score = T.dot(h2, self.Wyh2) + T.dot(dropout_h, self.Wyh) + T.dot(dropout_y, self.Wyy) + \
       T.dot(c, self.Wyc) + self.by
     return score, h_src, h0
    def __init__(self, hyper_param, word2idx_dic):
        '''
		nh :: dimension of the hidden layer
		ne :: number of word embeddings in the vocabulary
		de :: dimension of the word embeddings
		nf :: number of feature
		nfe:: number of feature embeddings in the vocabulary - by leeck
		dfe:: dimension of the feature embeddings - by leeck
		cs :: word window context size
		emb_file :: word embedding file
		weight_decay :: weight decay
		dropout_rate :: dropout rate
		activation :: activation function: simg, tanh, relu
		word2idx_dic :: word to index dictionary
		'''
        self.hyper_param = hyper_param
        nh = hyper_param['nhidden']
        ne = hyper_param['vocsize']
        de = hyper_param['emb_dimension']
        weight_decay = hyper_param['weight_decay']
        dropout_rate = hyper_param['dropout_rate']
        activation = hyper_param['activation']
        learning_method = hyper_param['learning_method']
        # parameters of the model
        if hyper_param['load_model'] != '':
            self.load_param(hyper_param, hyper_param['load_model'])
        else:
            self.build_param(hyper_param, word2idx_dic)

        # parameters
        self.params = [self.emb, self.Wgx_src, self.Wgxb_src, self.Wgx, self.Wgc, \
          self.Whx_src, self.Whxb_src, self.Whx, self.Whc, self.Wh0c, \
          self.Ugh_src, self.Ughb_src, self.Ugh, self.Uhh_src, self.Uhhb_src, self.Uhh, \
          self.bg_src, self.bgb_src, self.bg, self.bh_src, self.bhb_src, self.bh, self.bh2, self.bh0, \
          self.Wah, self.Wax, self.Ua, self.ba, self.va, \
          self.Wh2h, self.Wyh2, self.Wyh, self.Wyc, self.Wyy, self.by, \
          self.h0_src, self.h0b_src]

        if hyper_param['fixed_emb']:
            print 'fixed embeddig.'
            self.params.remove(self.emb)

        # as many lines as words in the sentence
        x_sentence = T.ivector('x_sentence')  # x_sentence : n_steps
        x = self.emb[x_sentence].reshape(
            (x_sentence.shape[0], de))  # don't remove '</s>'
        x_reverse = x[::-1]  # reverse for backward

        y_sentence = T.ivector('y_sentence')  # labels
        y_input_sentence = T.concatenate(
            [y_sentence[-1:], y_sentence[:-1]],
            axis=0)  # move '</s>' to first position
        y = self.emb[y_input_sentence].reshape((y_input_sentence.shape[0], de))

        # for scan
        #def source_step(x_t, h_tm1):
        def source_step(dot_x_t_Wgx_src, dot_x_t_Whx_src, h_tm1):
            #print 'z_t, r_t are combined!'
            #all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgx_src) + T.dot(h_tm1, self.Ugh_src) + self.bg_src)
            all_t = T.nnet.sigmoid(dot_x_t_Wgx_src +
                                   T.dot(h_tm1, self.Ugh_src))
            z_t = myutil.slice(all_t, 0, nh)
            r_t = myutil.slice(all_t, 1, nh)
            # candidate h_t
            #ch_t = myutil.activation(activation, T.dot(x_t, self.Whx_src) + T.dot(r_t * h_tm1, self.Uhh_src) + self.bh_src)
            ch_t = myutil.activation(
                activation, dot_x_t_Whx_src + T.dot(r_t * h_tm1, self.Uhh_src))
            h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
            return h_t

        #def source_backward_step(x_t, h_tm1):
        def source_backward_step(dot_x_t_Wgxb_src, dot_x_t_Whxb_src, h_tm1):
            #print 'z_t and r_t are combined!'
            #all_t = T.nnet.sigmoid(T.dot(x_t, self.Wgxb_src) + T.dot(h_tm1, self.Ughb_src) + self.bgb_src)
            all_t = T.nnet.sigmoid(dot_x_t_Wgxb_src +
                                   T.dot(h_tm1, self.Ughb_src))
            z_t = myutil.slice(all_t, 0, nh)
            r_t = myutil.slice(all_t, 1, nh)
            # candidate h_t
            #ch_t = myutil.activation(activation, T.dot(x_t, self.Whxb_src) + T.dot(r_t * h_tm1, self.Uhhb_src) + self.bhb_src)
            ch_t = myutil.activation(
                activation,
                dot_x_t_Whxb_src + T.dot(r_t * h_tm1, self.Uhhb_src))
            h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
            return h_t

        #def target_step(x_t, h_tm1, h_src, dot_h_src_Ua):
        def target_step(dot_x_t_Wgx, dot_x_t_Whx, dot_x_t_Wax, h_tm1, h_src,
                        dot_h_src_Ua):
            # search c_t
            #z = T.tanh(T.dot(h_tm1, self.Wa) + T.dot(h_src, self.Ua) + self.ba)
            #z = T.tanh(T.dot(h_tm1, self.Wa) + dot_h_src_Ua)
            z = T.tanh(T.dot(h_tm1, self.Wah) + dot_x_t_Wax + dot_h_src_Ua)
            #print 'z:', z.ndim
            e = T.dot(self.va, z.T)
            #print 'e:', e.ndim
            max_e = T.max(e)
            exp_e = T.exp(e - max_e)
            a = exp_e / exp_e.sum()
            #print 'a:', a.ndim
            c_t = T.dot(a, h_src)
            #print 'c_t:', c_t.ndim
            #print 'z_t and r_t are combined!'
            all_t = T.nnet.sigmoid(dot_x_t_Wgx + T.dot(h_tm1, self.Ugh) +
                                   T.dot(c_t, self.Wgc))
            z_t = myutil.slice(all_t, 0, nh)
            r_t = myutil.slice(all_t, 1, nh)
            # candidate h_t
            ch_t = myutil.activation(
                activation, dot_x_t_Whx + T.dot(r_t * h_tm1, self.Uhh) +
                T.dot(c_t, self.Whc))
            h_t = (1.0 - z_t) * h_tm1 + z_t * ch_t
            return [h_t, c_t]

        # make score, h_src, h0 (for beam search)
        def make_score(x, y, use_noise):
            # input layer dropout: ex. [0.2, 0.2, 0.5]
            if use_noise:
                print "X's projection layer dropout:", dropout_rate[0]
                dropout_x = myutil.dropout_from_layer(x, dropout_rate[0])
            else:
                dropout_x = x * (1.0 - dropout_rate[0])
            dropout_x_reverse = dropout_x[::-1]  # reverse for backward
            # RNN encoder
            dot_x_Wgx_src = T.dot(dropout_x, self.Wgx_src) + self.bg_src
            dot_x_Whx_src = T.dot(dropout_x, self.Whx_src) + self.bh_src
            dot_x_rev_Wgx_src = T.dot(dropout_x_reverse,
                                      self.Wgxb_src) + self.bgb_src
            dot_x_rev_Whx_src = T.dot(dropout_x_reverse,
                                      self.Whxb_src) + self.bhb_src
            # forward recurrent for source language
            hf_src, _ = theano.scan(fn=source_step,
                                    sequences=[dot_x_Wgx_src, dot_x_Whx_src],
                                    outputs_info=self.h0_src,
                                    n_steps=dropout_x.shape[0])
            # backward recurrent for source language
            hb_src_reverse, _ = theano.scan(
                fn=source_backward_step,
                sequences=[dot_x_rev_Wgx_src, dot_x_rev_Whx_src],
                outputs_info=self.h0b_src,
                n_steps=dropout_x_reverse.shape[0])
            hb_src = hb_src_reverse[::-1]
            h_src = T.concatenate([hf_src, hb_src], axis=1)
            # global context
            #c_global = h_src[0]
            c_global = T.concatenate([hf_src[-1], hb_src[0]], axis=0)
            # output layer dropout: ex. [0.2, 0.2, 0.5]
            # output layer (target language input layer) dropout: ex. [0.2, 0.2, 0.5]
            if use_noise:
                print "Y's projection layer dropout:", dropout_rate[1]
                dropout_y = myutil.dropout_from_layer(y, dropout_rate[1])
            else:
                dropout_y = y * (1.0 - dropout_rate[1])
            # RNN decoder
            dot_y_Wgx = T.dot(dropout_y, self.Wgx) + self.bg
            dot_y_Whx = T.dot(dropout_y, self.Whx) + self.bh
            dot_y_Wax = T.dot(dropout_y, self.Wax)
            dot_h_src_Ua = T.dot(h_src, self.Ua) + self.ba
            h0 = myutil.activation(activation,
                                   T.dot(c_global, self.Wh0c) + self.bh0)
            # forward recurrent for target language
            [h,
             c], _ = theano.scan(fn=target_step,
                                 sequences=[dot_y_Wgx, dot_y_Whx, dot_y_Wax],
                                 outputs_info=[h0, None],
                                 non_sequences=[h_src, dot_h_src_Ua],
                                 n_steps=dropout_y.shape[0])
            # h2 - Deep Output RNN
            print 'Deep Output RNN: ReLU'
            # hidden layer dropout
            if use_noise:
                print "Y's hidden layer dropout:", dropout_rate[2]
                dropout_h = myutil.dropout_from_layer(h, dropout_rate[2])
            else:
                dropout_h = h * (1.0 - dropout_rate[2])
            # h2 - Deep Output RNN
            print 'Deep Output RNN: ReLU'
            h2 = myutil.activation('relu',
                                   T.dot(dropout_h, self.Wh2h) + self.bh2)
            # score
            score = T.dot(h2, self.Wyh2) + T.dot(dropout_h, self.Wyh) + T.dot(dropout_y, self.Wyy) + \
              T.dot(c, self.Wyc) + self.by
            return score, h_src, h0

        # dropout version (for training)
        dropout_score, _, _ = make_score(x, y, True)
        dropout_p_y_given_x = myutil.mysoftmax(dropout_score)

        # scaled version (for prediction)
        score, h_src, h0 = make_score(x, y, False)
        p_y_given_x = myutil.mysoftmax(score)

        # prediction
        y_pred = T.argmax(p_y_given_x, axis=1)
        test_nll = -T.mean(
            T.log(p_y_given_x)[T.arange(y.shape[0]), y_sentence])

        # beam search decoding: input=[h_src, h_tm1, y_tm1], output=[h_t, log_p_y_t, alignment]
        input_h_src = T.fmatrix('input_h_src')
        input_h_tm1 = T.fvector('input_h_tm1')
        input_y_tm1 = T.iscalar('input_y_tm1')  # input_y_tm1 == x_t
        x_t = self.emb[input_y_tm1]
        # search c_t
        #z = T.tanh(T.dot(input_h_tm1, self.Wa) + T.dot(input_h_src, self.Ua) + self.ba)
        z = T.tanh(
            T.dot(input_h_tm1, self.Wah) + T.dot(x_t, self.Wax) +
            T.dot(input_h_src, self.Ua) + self.ba)
        e = T.dot(self.va, z.T)
        max_e = T.max(e)
        exp_e = T.exp(e - max_e)
        alignment = exp_e / exp_e.sum()
        c_t = T.dot(alignment, input_h_src)
        all_t = T.nnet.sigmoid(
            T.dot(x_t, self.Wgx) + T.dot(input_h_tm1, self.Ugh) +
            T.dot(c_t, self.Wgc) + self.bg)
        z_t = myutil.slice(all_t, 0, nh)
        r_t = myutil.slice(all_t, 1, nh)
        # candidate h_t
        ch_t = myutil.activation(
            activation,
            T.dot(x_t, self.Whx) + T.dot(r_t * input_h_tm1, self.Uhh) +
            T.dot(c_t, self.Whc) + self.bh)
        h_t = (1.0 - z_t) * input_h_tm1 + z_t * ch_t
        # h2 - Deep Output RNN
        h2_t = myutil.activation('relu', T.dot(h_t, self.Wh2h) + self.bh2)
        # prediction
        score_y_t = T.dot(h2_t, self.Wyh2) + T.dot(h_t, self.Wyh) + T.dot(x_t, self.Wyy) + \
          T.dot(c_t, self.Wyc) + self.by
        max_s = T.max(score_y_t)
        exp_s = T.exp(score_y_t - max_s)
        log_p_y_t = T.log(exp_s / exp_s.sum())

        # cost and gradients and learning rate
        lr = T.scalar('lr')  # for SGD

        # NLL + L2-norm
        nll = -T.mean(
            T.log(dropout_p_y_given_x)[T.arange(y.shape[0]), y_sentence])
        cost = nll
        for param in self.params:
            if param.name == 'emb':
                continue
            cost += weight_decay * T.sum(param**2)

        # SGD
        sgd_updates = myutil.sgd_updates(self.params, cost, lr)
        # SGD + momentum
        momentum_updates = myutil.sgd_updates_momentum(self.params, cost, lr,
                                                       0.9)
        # RMSProp (rho = 0.9)
        rmsprop_updates = myutil.sgd_updates_rmsprop(self.params, cost, lr,
                                                     0.9, 1)
        # AdaDelta (lr --> rho = 0.95)
        adadelta_updates = myutil.sgd_updates_adadelta(self.params, cost, lr,
                                                       1e-6, 9)

        # theano functions to compile
        self.classify = theano.function(inputs=[x_sentence, y_sentence],
                                        outputs=[y_pred, test_nll])
        # for beam search
        self.encoding_src_lang = theano.function(inputs=[x_sentence],
                                                 outputs=[h_src, h0])
        self.search_next_word = theano.function(
            inputs=[input_h_src, input_h_tm1, input_y_tm1],
            outputs=[log_p_y_t, h_t, alignment])
        # for reranking
        self.get_nll = theano.function(
            inputs=[x_sentence, input_h_src, input_h_tm1, y_sentence],
            outputs=test_nll,
            on_unused_input='ignore')
        # SGD
        self.train_sgd = theano.function(inputs=[x_sentence, y_sentence, lr],
                                         outputs=[cost, nll],
                                         updates=sgd_updates)
        # SGD with momentum
        self.train_momentum = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=momentum_updates)
        # RMSProp
        self.train_rmsprop = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=rmsprop_updates)
        # AdaDelta
        self.train_adadelta = theano.function(
            inputs=[x_sentence, y_sentence, lr],
            outputs=[cost, nll],
            updates=adadelta_updates)