def bi_lstm(self, name, x, x_mask, input_dim, hidden_dim, drop_x, drop_h, **kwargs): n = namer(name) fwd_h = self.lstm(n('fwd'), x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=False, **kwargs) bck_h = self.lstm(n('bck'), x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=True, **kwargs) bi_h = tt.concatenate([fwd_h, bck_h], axis=2) # (timesteps, batch_size, 2*hidden_dim) return bi_h
def bi_lstm(self, name, x, x_mask, input_dim, hidden_dim, drop_x, drop_h, **kwargs): n = namer(name) fwd_h = self.lstm(n('fwd'), x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=False, **kwargs) bck_h = self.lstm(n('bck'), x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=True, **kwargs) bi_h = tt.concatenate([fwd_h, bck_h], axis=2) # (timesteps, batch_size, 2*hidden_dim) return bi_h
def stacked_bi_lstm(self, name, x, x_mask, num_layers, input_dim, hidden_dim, drop_x, drop_h, **kwargs): n = namer(name) h = x for l in range(1, num_layers+1): h = self.bi_lstm(n('l%d' % l), h, x_mask, input_dim if l == 1 else 2*hidden_dim, hidden_dim, drop_x, drop_h, **kwargs) return h # (timesteps, batch_size, 2*hidden_dim)
def stacked_bi_lstm(self, name, x, x_mask, num_layers, input_dim, hidden_dim, drop_x, drop_h, **kwargs): n = namer(name) h = x for l in range(1, num_layers+1): h = self.bi_lstm(n('l%d' % l), h, x_mask, input_dim if l == 1 else 2*hidden_dim, hidden_dim, drop_x, drop_h, **kwargs) return h # (timesteps, batch_size, 2*hidden_dim)
def linear(self, name, x, input_dim, output_dim, with_bias=True, w_init='uniform', bias_init=0): # x (..., input_dim) n = namer(name) W = self.make_param(n('W'), (input_dim, output_dim), w_init) y = tt.dot(x, W) # (..., output_dim) if with_bias: b = self.make_param(n('b'), (output_dim,), bias_init) y += b return y
def linear(self, name, x, input_dim, output_dim, with_bias=True, w_init='uniform', bias_init=0): # x (..., input_dim) n = namer(name) W = self.make_param(n('W'), (input_dim, output_dim), w_init) y = tt.dot(x, W) # (..., output_dim) if with_bias: b = self.make_param(n('b'), (output_dim,), bias_init) y += b return y
def stacked_bi_lstm(self, name, x, x_mask, num_layers, input_dim, hidden_dim, drop_x, drop_h, **kwargs): # can be changed: returning last hidden states for a stacked bi-lstm is unsupported if 'return_last' in kwargs: assert not kwargs['return_last'] n = namer(name) h = x for l in range(1, num_layers + 1): h = self.bi_lstm(n('l%d' % l), h, x_mask, input_dim if l == 1 else 2 * hidden_dim, hidden_dim, drop_x, drop_h, **kwargs) return h # (timesteps, batch_size, 2*hidden_dim)
def ff(self, name, x, dims, activation, dropout_ps, **kwargs): assert len(dims) >= 2 if dropout_ps: if isinstance(dropout_ps, numbers.Number): dropout_ps = [dropout_ps] * (len(dims) - 1) else: assert len(dropout_ps) == len(dims) - 1 n = namer(name) h = x if activation == 'relu': f = tt.nnet.relu elif activation == 'sigmoid': f = tt.nnet.sigmoid elif activation == 'tanh': f = tt.tanh else: raise AssertionError('unrecognized activation function') for i, (input_dim, output_dim) in enumerate(zip(dims[:-1], dims[1:])): if dropout_ps: h = self.dropout(h, dropout_ps[i]) h = f(self.linear(n('l%d' % (i+1)), h, input_dim, output_dim, **kwargs)) return h
def ff(self, name, x, dims, activation, dropout_ps, **kwargs): assert len(dims) >= 2 if dropout_ps: if isinstance(dropout_ps, numbers.Number): dropout_ps = [dropout_ps] * (len(dims) - 1) else: assert len(dropout_ps) == len(dims) - 1 n = namer(name) h = x if activation == 'relu': f = tt.nnet.relu elif activation == 'sigmoid': f = tt.nnet.sigmoid elif activation == 'tanh': f = tt.tanh else: raise AssertionError('unrecognized activation function') for i, (input_dim, output_dim) in enumerate(zip(dims[:-1], dims[1:])): if dropout_ps: h = self.dropout(h, dropout_ps[i]) h = f(self.linear(n('l%d' % (i+1)), h, input_dim, output_dim, **kwargs)) return h
def lstm(self, name, x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=False, couple_i_and_f=False, learn_initial_state=False, tie_x_dropout=True, sep_x_dropout=False, sep_h_dropout=False, w_init='uniform', u_init='orthogonal', forget_bias_init=1, other_bias_init=0): """Customizable uni-directional LSTM layer. Handles masks, can learn initial state, input and forget gate can be coupled, with recurrent dropout, no peephole connections. Args: x: Theano tensor, shape (timesteps, batch_size, input_dim) x_mask: Theano tensor, shape (timesteps, batch_size) input_dim: int, dimension of input vectors hidden_dim: int, dimension of hidden state drop_x: float, dropout rate to apply to inputs drop_h: float, dropout rate to apply to hidden state backward: boolean, whether to recur over timesteps in reveresed order couple_i_and_f: boolean, whether to have input gate = 1 - forget gate learn_initial_state: boolean, whether to have initial cell state and initial hidden state as learnt parameters tie_x_dropout: boolean, whether to have the same dropout masks across timesteps for input sep_x_dropout: boolean, if True dropout is applied over weights of lin. trans. of input; otherwise it is applied over input activations sep_h_dropout: boolean, if True dropout is applied over weights of lin. trans. of hidden state; otherwise it is applied over hidden state activations w_init: string, initialization scheme for weights of lin. trans. of input u_init: string, initialization scheme for weights of lin. trans. of hidden state forget_bias_init: string, initialization scheme for forget gate's bias other_bias_init: string, initialization scheme for other biases Note: Proper variational dropout (Gal 2015) is: tie_x_dropout=True, sep_x_dropout=True, sep_h_dropout=True A faster alternative is: tie_x_dropout=True, sep_x_dropout=False, sep_h_dropout=False Returns: h: Theano variable, recurrent hidden states at each timestep, shape (timesteps, batch_size, hidden_dim) """ n = namer(name) timesteps, batch_size = x.shape[0], x.shape[1] num_non_lin = 3 if couple_i_and_f else 4 num_gates = num_non_lin - 1 W = self.make_concat_param(n('W'), # (input_dim, [3|4]*hidden_dim) num_non_lin*[(input_dim, hidden_dim)], num_non_lin*[w_init], axis=1) b = self.make_concat_param(n('b'), # ([3|4]*hidden_dim,) num_non_lin*[(hidden_dim,)], [forget_bias_init] + num_gates*[other_bias_init], axis=0) U = self.make_concat_param(n('U'), # (hidden_dim, [3|4]*hidden_dim) num_non_lin*[(hidden_dim, hidden_dim)], num_non_lin*[u_init], axis=1) if not sep_x_dropout: if tie_x_dropout: x = self.apply_dropout_noise(x, self.get_dropout_noise((batch_size, input_dim), drop_x)) else: x = self.dropout(x, drop_x) lin_x = tt.dot(x, W) + b # (timesteps, batch_size, [3|4]*hidden_dim) else: if tie_x_dropout: x_for_f = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) x_for_o = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) if num_gates == 3: x_for_i = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) x_for_g = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) else: x_for_f = self.dropout(x, drop_x) x_for_o = self.dropout(x, drop_x) if num_gates == 3: x_for_i = self.dropout(x, drop_x) x_for_g = self.dropout(x, drop_x) lin_x_tensors = [tt.dot(x_for_f, W[:,:hidden_dim]), tt.dot(x_for_o, W[:,hidden_dim:2*hidden_dim])] if num_gates == 3: lin_x_tensors.append(tt.dot(x_for_i, W[:,2*hidden_dim:3*hidden_dim])) lin_x_tensors.append(tt.dot(x_for_g, W[:,num_gates*hidden_dim:])) lin_x = tt.concatenate(lin_x_tensors, axis=2) + b # (timesteps, batch_size, [3|4]*hidden_dim) def step_fn(lin_x_t, x_mask_t, h_tm1, c_tm1, h_noise, U): # lin_x_t (batch_size, [3|4]*hidden_dim) # x_mask_t (batch_size, 1) # h_tm1 (batch_size, hidden_dim) # c_tm1 (batch_size, hidden_dim) # h_noise (batch_size, [1|3|4]*hidden_dim) # 1 if not sep_h_dropout, otherwise: 3 or 4 depending on num_non_lin # U (hidden_dim, [3|4]*hidden_dim) if not sep_h_dropout: h_tm1 = self.apply_dropout_noise(h_tm1, h_noise) lin_h_tm1 = tt.dot(h_tm1, U) # (batch_size, [3|4]*hidden_dim) else: h_tm1_for_f = self.apply_dropout_noise(h_tm1, h_noise[:,:hidden_dim]) h_tm1_for_o = self.apply_dropout_noise(h_tm1, h_noise[:,hidden_dim:2*hidden_dim]) if num_gates == 3: h_tm1_for_i = self.apply_dropout_noise(h_tm1, h_noise[:,2*hidden_dim:3*hidden_dim]) h_tm1_for_g = self.apply_dropout_noise(h_tm1, h_noise[:,num_gates*hidden_dim:]) lin_h_tm1_tensors = [tt.dot(h_tm1_for_f, U[:,:hidden_dim]), tt.dot(h_tm1_for_o, U[:,hidden_dim:2*hidden_dim])] if num_gates == 3: lin_h_tm1_tensors.append(tt.dot(h_tm1_for_i, U[:,2*hidden_dim:3*hidden_dim])) lin_h_tm1_tensors.append(tt.dot(h_tm1_for_g, U[:,num_gates*hidden_dim:])) lin_h_tm1 = tt.concatenate(lin_h_tm1_tensors, axis=1) # (batch_size, [3|4]*hidden_dim) lin = lin_x_t + lin_h_tm1 # (batch_size, [3|4]*hidden_dim) gates = tt.nnet.sigmoid(lin[:, :num_gates*hidden_dim]) # (batch_size, [3|4]*hidden_dim) f_gate = gates[:, :hidden_dim] # (batch_size, hidden_dim) o_gate = gates[:, hidden_dim:2*hidden_dim] # (batch_size, hidden_dim) i_gate = gates[:, 2*hidden_dim:] if num_gates == 3 else 1 - f_gate # (batch_size, hidden_dim) g = tt.tanh(lin[:, num_gates*hidden_dim:]) # (batch_size, hidden_dim) c_t = f_gate * c_tm1 + i_gate * g h_t = o_gate * tt.tanh(c_t) h_t = tt.switch(x_mask_t, h_t, h_tm1) c_t = tt.switch(x_mask_t, c_t, c_tm1) return h_t, c_t # end of step_fn if learn_initial_state: h0 = self.make_param(n('h0'), (hidden_dim,), 0) c0 = self.make_param(n('c0'), (hidden_dim,), 0) batch_h0 = tt.extra_ops.repeat(h0[None,:], batch_size, axis=0) batch_c0 = tt.extra_ops.repeat(c0[None,:], batch_size, axis=0) else: batch_h0 = batch_c0 = tt.zeros((batch_size, hidden_dim)) x_mask = tt.shape_padright(x_mask) # (timesteps, batch_size, 1) original_x_mask = x_mask if backward: lin_x = lin_x[::-1] x_mask = x_mask[::-1] h_noise = self.get_dropout_noise( (batch_size, hidden_dim if not sep_h_dropout else num_non_lin*hidden_dim), drop_h) results, _ = theano.scan(step_fn, sequences = [lin_x, x_mask], outputs_info = [batch_h0, batch_c0], non_sequences = [h_noise, U], name = n('scan')) h = results[0] # (timesteps, batch_size, hidden_dim) if backward: h = h[::-1] h *= original_x_mask return h
def lstm(self, name, x, x_mask, input_dim, hidden_dim, drop_x, drop_h, backward=False, return_last=False, couple_i_and_f=False, learn_initial_state=False, tie_x_dropout=True, sep_x_dropout=False, sep_h_dropout=False, w_init='uniform', u_init='orthogonal', forget_bias_init=1, other_bias_init=0): """Customizable uni-directional LSTM layer. Handles masks, can learn initial state, input and forget gate can be coupled, with recurrent dropout, no peephole connections. Args: x: Theano tensor, shape (timesteps, batch_size, input_dim) x_mask: Theano tensor, shape (timesteps, batch_size) input_dim: int, dimension of input vectors hidden_dim: int, dimension of hidden state drop_x: float, dropout rate to apply to inputs drop_h: float, dropout rate to apply to hidden state backward: boolean, whether to recur over timesteps in reveresed order return_last: boolean, whether to in last hidden state in return value couple_i_and_f: boolean, whether to have input gate = 1 - forget gate learn_initial_state: boolean, whether to have initial cell state and initial hidden state as learnt parameters tie_x_dropout: boolean, whether to have the same dropout masks across timesteps for input sep_x_dropout: boolean, if True dropout is applied over weights of lin. trans. of input; otherwise it is applied over input activations sep_h_dropout: boolean, if True dropout is applied over weights of lin. trans. of hidden state; otherwise it is applied over hidden state activations w_init: string, initialization scheme for weights of lin. trans. of input u_init: string, initialization scheme for weights of lin. trans. of hidden state forget_bias_init: string, initialization scheme for forget gate's bias other_bias_init: string, initialization scheme for other biases Note: Proper variational dropout (Gal 2015) is: tie_x_dropout=True, sep_x_dropout=True, sep_h_dropout=True A faster alternative is: tie_x_dropout=True, sep_x_dropout=False, sep_h_dropout=False Returns: h: Theano variable, recurrent hidden states at each timestep, shape (timesteps, batch_size, hidden_dim) """ n = namer(name) timesteps, batch_size = x.shape[0], x.shape[1] num_non_lin = 3 if couple_i_and_f else 4 num_gates = num_non_lin - 1 W = self.make_concat_param( n('W'), # (input_dim, [3|4]*hidden_dim) num_non_lin * [(input_dim, hidden_dim)], num_non_lin * [w_init], axis=1) b = self.make_concat_param( n('b'), # ([3|4]*hidden_dim,) num_non_lin * [(hidden_dim, )], [forget_bias_init] + num_gates * [other_bias_init], axis=0) U = self.make_concat_param( n('U'), # (hidden_dim, [3|4]*hidden_dim) num_non_lin * [(hidden_dim, hidden_dim)], num_non_lin * [u_init], axis=1) if not sep_x_dropout: if tie_x_dropout: x = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) else: x = self.dropout(x, drop_x) lin_x = tt.dot(x, W) + b # (timesteps, batch_size, [3|4]*hidden_dim) else: if tie_x_dropout: x_for_f = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) x_for_o = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) if num_gates == 3: x_for_i = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) x_for_g = self.apply_dropout_noise( x, self.get_dropout_noise((batch_size, input_dim), drop_x)) else: x_for_f = self.dropout(x, drop_x) x_for_o = self.dropout(x, drop_x) if num_gates == 3: x_for_i = self.dropout(x, drop_x) x_for_g = self.dropout(x, drop_x) lin_x_tensors = [ tt.dot(x_for_f, W[:, :hidden_dim]), tt.dot(x_for_o, W[:, hidden_dim:2 * hidden_dim]) ] if num_gates == 3: lin_x_tensors.append( tt.dot(x_for_i, W[:, 2 * hidden_dim:3 * hidden_dim])) lin_x_tensors.append(tt.dot(x_for_g, W[:, num_gates * hidden_dim:])) lin_x = tt.concatenate( lin_x_tensors, axis=2) + b # (timesteps, batch_size, [3|4]*hidden_dim) def step_fn(lin_x_t, x_mask_t, h_tm1, c_tm1, h_noise, U): # lin_x_t (batch_size, [3|4]*hidden_dim) # x_mask_t (batch_size, 1) # h_tm1 (batch_size, hidden_dim) # c_tm1 (batch_size, hidden_dim) # h_noise (batch_size, [1|3|4]*hidden_dim) # 1 if not sep_h_dropout, otherwise: 3 or 4 depending on num_non_lin # U (hidden_dim, [3|4]*hidden_dim) if not sep_h_dropout: h_tm1 = self.apply_dropout_noise(h_tm1, h_noise) lin_h_tm1 = tt.dot(h_tm1, U) # (batch_size, [3|4]*hidden_dim) else: h_tm1_for_f = self.apply_dropout_noise(h_tm1, h_noise[:, :hidden_dim]) h_tm1_for_o = self.apply_dropout_noise( h_tm1, h_noise[:, hidden_dim:2 * hidden_dim]) if num_gates == 3: h_tm1_for_i = self.apply_dropout_noise( h_tm1, h_noise[:, 2 * hidden_dim:3 * hidden_dim]) h_tm1_for_g = self.apply_dropout_noise( h_tm1, h_noise[:, num_gates * hidden_dim:]) lin_h_tm1_tensors = [ tt.dot(h_tm1_for_f, U[:, :hidden_dim]), tt.dot(h_tm1_for_o, U[:, hidden_dim:2 * hidden_dim]) ] if num_gates == 3: lin_h_tm1_tensors.append( tt.dot(h_tm1_for_i, U[:, 2 * hidden_dim:3 * hidden_dim])) lin_h_tm1_tensors.append( tt.dot(h_tm1_for_g, U[:, num_gates * hidden_dim:])) lin_h_tm1 = tt.concatenate( lin_h_tm1_tensors, axis=1) # (batch_size, [3|4]*hidden_dim) lin = lin_x_t + lin_h_tm1 # (batch_size, [3|4]*hidden_dim) gates = tt.nnet.sigmoid( lin[:, :num_gates * hidden_dim]) # (batch_size, [3|4]*hidden_dim) f_gate = gates[:, :hidden_dim] # (batch_size, hidden_dim) o_gate = gates[:, hidden_dim:2 * hidden_dim] # (batch_size, hidden_dim) i_gate = gates[:, 2 * hidden_dim:] if num_gates == 3 else 1 - f_gate # (batch_size, hidden_dim) g = tt.tanh(lin[:, num_gates * hidden_dim:]) # (batch_size, hidden_dim) c_t = f_gate * c_tm1 + i_gate * g h_t = o_gate * tt.tanh(c_t) h_t = tt.switch(x_mask_t, h_t, h_tm1) c_t = tt.switch(x_mask_t, c_t, c_tm1) return h_t, c_t # end of step_fn if learn_initial_state: h0 = self.make_param(n('h0'), (hidden_dim, ), 0) c0 = self.make_param(n('c0'), (hidden_dim, ), 0) batch_h0 = tt.extra_ops.repeat(h0[None, :], batch_size, axis=0) batch_c0 = tt.extra_ops.repeat(c0[None, :], batch_size, axis=0) else: batch_h0 = batch_c0 = tt.zeros((batch_size, hidden_dim)) x_mask = tt.shape_padright(x_mask) # (timesteps, batch_size, 1) original_x_mask = x_mask if backward: lin_x = lin_x[::-1] x_mask = x_mask[::-1] h_noise = self.get_dropout_noise( (batch_size, hidden_dim if not sep_h_dropout else num_non_lin * hidden_dim), drop_h) results, _ = theano.scan(step_fn, sequences=[lin_x, x_mask], outputs_info=[batch_h0, batch_c0], non_sequences=[h_noise, U], name=n('scan')) h = results[0] # (timesteps, batch_size, hidden_dim) last_h = h[-1] # (batch_size, hidden_dim) if backward: h = h[::-1] h *= original_x_mask if return_last: return h, last_h return h