Example #1
0
    def __init__(self, inpt, nin, nunits, conv_sz=1, learn_init_state=True):
        # inpt is transposed a priori
        tablet_wd, _ = inpt.shape
        if conv_sz > 1:
            inpt_clipped = inpt[:conv_sz * (tablet_wd // conv_sz), :]
            inpt_conv = inpt_clipped.reshape(
                (tablet_wd // conv_sz, nin * conv_sz))
        else:
            inpt_conv = inpt

        wio = share(init_wts(nin * conv_sz, nunits))  # input to output
        woo = share(init_wts(nunits, nunits))  # output to output
        bo = share(init_wts(nunits))
        h0 = share(init_wts(nunits))

        def step(in_t, out_tm1):
            return tt.tanh(tt.dot(out_tm1, woo) + tt.dot(in_t, wio) + bo)

        self.output, _ = theano.scan(step,
                                     sequences=[inpt_conv],
                                     outputs_info=[h0])

        self.params = [wio, woo, bo]
        if learn_init_state:
            self.params += [h0]
        self.nout = nunits
Example #2
0
    def __init__(self, inpt, nin, nunits, conv_sz=1,
                 learn_init_state=True):
        # inpt is transposed a priori
        tablet_wd, _ = inpt.shape
        if conv_sz > 1:
            inpt_clipped = inpt[:conv_sz * (tablet_wd // conv_sz), :]
            inpt_conv = inpt_clipped.reshape(
                (tablet_wd // conv_sz, nin * conv_sz))
        else:
            inpt_conv = inpt

        wio = share(init_wts(nin * conv_sz, nunits))  # input to output
        woo = share(init_wts(nunits, nunits))  # output to output
        bo = share(init_wts(nunits))
        h0 = share(init_wts(nunits))

        def step(in_t, out_tm1):
            return TT.tanh(TT.dot(out_tm1, woo) + TT.dot(in_t, wio) + bo)

        self.output, _ = theano.scan(
            step,
            sequences=[inpt_conv],
            outputs_info=[h0]
        )

        self.params = [wio, woo, bo]
        if learn_init_state:
            self.params += [h0]
        self.nout = nunits
Example #3
0
 def __init__(self, inpt, in_sz, n_classes, tied=False):
     if tied:
         b = share(init_wts(n_classes-1))
         w = share(init_wts(in_sz, n_classes-1))
         w1 = TT.horizontal_stack(w, TT.zeros((in_sz, 1)))
         b1 = TT.concatenate((b, TT.zeros(1)))
         self.output = TT.dot(inpt, w1) + b1
     else:
         b = share(init_wts(n_classes))
         w = share(init_wts(in_sz, n_classes))
         self.output = TT.dot(inpt, w) + b
     self.params = [w, b]
Example #4
0
 def __init__(self, inpt, in_sz, n_classes, tied=False):
     if tied:
         b = share(init_wts(n_classes-1))
         w = share(init_wts(in_sz, n_classes-1))
         w1 = tt.horizontal_stack(w, tt.zeros((in_sz, 1)))
         b1 = tt.concatenate((b, tt.zeros(1)))
         self.output = tt.dot(inpt, w1) + b1
     else:
         b = share(init_wts(n_classes))
         w = share(init_wts(in_sz, n_classes))
         self.output = tt.dot(inpt, w) + b
     self.params = [w, b]
Example #5
0
    def __init__(self,
                 inpt,
                 nin,
                 nunits,
                 forget=False,
                 actvn_pre='tanh',
                 actvn_post='linear',
                 learn_init_states=True):
        """
        Init
        :param inpt: Lower layer's excitation.
        :param nin: Dimension of lower layer.
        :param nunits: Number of units.
        :param forget: Want a seperate forget gate (or use 1-input)?
        :param actvn_pre: Activation applied to new candidate for cell value.
        :param actvn_post: Activation applied to cell value before output.
        :param learn_init_states: Should the intial states be learnt?
        :return: Output
        """
        # TODO: Incell connections

        num_activations = 3 + forget
        w = stacked_ortho_wts(nin, nunits, num_activations)
        u = stacked_ortho_wts(nunits, nunits, num_activations)
        b = share(np.zeros(num_activations * nunits))
        out0 = share(np.zeros(nunits))
        cell0 = share(np.zeros(nunits))

        actvn_pre = activation_by_name(actvn_pre)
        actvn_post = activation_by_name(actvn_post)

        def step(in_t, out_tm1, cell_tm1):
            """
            Scan function.
            :param in_t: Current input from bottom layer
            :param out_tm1: Prev output of LSTM layer
            :param cell_tm1: Prev cell value
            :return: Current output and cell value
            """
            tmp = tt.dot(out_tm1, u) + in_t

            inn_gate = sigmoid(tmp[:nunits])
            out_gate = sigmoid(tmp[nunits:2 * nunits])
            fgt_gate = sigmoid(tmp[2 * nunits:3 *
                                   nunits]) if forget else 1 - inn_gate

            cell_val = actvn_pre(tmp[-nunits:])
            cell_val = fgt_gate * cell_tm1 + inn_gate * cell_val
            out = out_gate * actvn_post(cell_val)

            return out, cell_val

        inpt = tt.dot(inpt, w) + b
        # seqlen x nin * nin x 3*nout + 3 * nout  = seqlen x 3*nout

        rval, updates = th.scan(
            step,
            sequences=[inpt],
            outputs_info=[out0, cell0],
        )

        self.output = rval[0]
        self.params = [w, u, b]
        if learn_init_states:
            self.params += [out0, cell0]
        self.nout = nunits
Example #6
0
    def __init__(self, inpt,
                 nin, nunits,
                 forget=False,
                 pre_activation='tanh',
                 post_activation='linear',
                 learn_init_states=True):
        """
        Init
        :@param inpt: activations from incoming layer.
        :@param nin: dimensions of incoming layer.
        :@param nunits: number of units.
        :@param forget: use forget gate
        :@param pre_activation: activation pre-synaptic to central cell.
        :@param post_activation: activation applied to central cell b4 output.
        :@param learn_init_states: learn the initial states
        :@return: Output
        """

        num_activations = 3 + forget
        w = stacked_ortho_wts(nin, nunits, num_activations)
        u = stacked_ortho_wts(nunits, nunits, num_activations)
        b = share(np.zeros(num_activations * nunits))
        out0 = share(np.zeros(nunits))
        cell0 = share(np.zeros(nunits))

        pre_activation = activation_by_name(pre_activation)
        post_activation = activation_by_name(post_activation)

        def step(in_t, out_tm1, cell_tm1):
            """
            Scan function.
            :@param in_t: current input from incoming layer
            :@param out_tm1: prev output of LSTM layer
            :@param cell_tm1: prev central cell value
            :@return: current output and central cell value
            """
            tmp = TT.dot(out_tm1, u) + in_t

            inn_gate = sigmoid(tmp[:nunits])
            out_gate = sigmoid(tmp[nunits:2 * nunits])
            fgt_gate = sigmoid(
                tmp[2 * nunits:3 * nunits]) if forget else 1 - inn_gate

            cell_val = pre_activation(tmp[-nunits:])
            cell_val = fgt_gate * cell_tm1 + inn_gate * cell_val
            out = out_gate * post_activation(cell_val)

            return out, cell_val

        inpt = TT.dot(inpt, w) + b
        # seqlen x nin * nin x 3*nout + 3 * nout  = seqlen x 3*nout

        rval, updates = theano.scan(step,
                                sequences=[inpt],
                                outputs_info=[out0, cell0], )

        self.output = rval[0]
        self.params = [w, u, b]
        if learn_init_states:
            self.params += [out0, cell0]
        self.nout = nunits
Example #7
0
File: lstm.py Project: Duum/rnn_ctc
    def __init__(self, inpt,
                 nin, nunits,
                 forget=False,
                 actvn_pre='tanh',
                 actvn_post='linear',
                 learn_init_states=True):
        """
        Init
        :param inpt: Lower layer's excitation.
        :param nin: Dimension of lower layer.
        :param nunits: Number of units.
        :param forget: Want a seperate forget gate (or use 1-input)?
        :param actvn_pre: Activation applied to new candidate for cell value.
        :param actvn_post: Activation applied to cell value before output.
        :param learn_init_states: Should the intial states be learnt?
        :return: Output
        """
        # TODO: Incell connections

        num_activations = 3 + forget
        w = stacked_ortho_wts(nin, nunits, num_activations)
        u = stacked_ortho_wts(nunits, nunits, num_activations)
        b = share(np.zeros(num_activations * nunits))
        out0 = share(np.zeros(nunits))
        cell0 = share(np.zeros(nunits))

        actvn_pre = activation_by_name(actvn_pre)
        actvn_post = activation_by_name(actvn_post)

        def step(in_t, out_tm1, cell_tm1):
            """
            Scan function.
            :param in_t: Current input from bottom layer
            :param out_tm1: Prev output of LSTM layer
            :param cell_tm1: Prev cell value
            :return: Current output and cell value
            """
            tmp = tt.dot(out_tm1, u) + in_t

            inn_gate = sigmoid(tmp[:nunits])
            out_gate = sigmoid(tmp[nunits:2 * nunits])
            fgt_gate = sigmoid(
                tmp[2 * nunits:3 * nunits]) if forget else 1 - inn_gate

            cell_val = actvn_pre(tmp[-nunits:])
            cell_val = fgt_gate * cell_tm1 + inn_gate * cell_val
            out = out_gate * actvn_post(cell_val)

            return out, cell_val

        inpt = tt.dot(inpt, w) + b
        # seqlen x nin * nin x 3*nout + 3 * nout  = seqlen x 3*nout

        rval, updates = th.scan(step,
                                sequences=[inpt],
                                outputs_info=[out0, cell0], )

        self.output = rval[0]
        self.params = [w, u, b]
        if learn_init_states:
            self.params += [out0, cell0]
        self.nout = nunits