Ejemplo n.º 1
0
def get_activation_func(activation_func, x):
    """Returns a Theano activation function based on the given parameter.

    Parameters
    ----------
    activation_func : string
        activation function of neurons, choices: sigmoid, tanh, rect, softsign
    x : Theano variable
        data

    Returns
    -------
    Theano function
        activation function
    """

    if activation_func == 'tanh':
        return T.tanh(x)
    if activation_func == 'sigmoid':
        return T.nnet.sigmoid(x)
    if activation_func == 'rect':
        return T.maximum(0, x)
    if activation_func == 'softsign':
        from theano.sandbox.softsign import softsign
        return softsign(x)

    raise ValueError('unknown activation function: %s' % activation_func)
Ejemplo n.º 2
0
def get_activation_func(activation_func, x):
    """Returns a Theano activation function based on the given parameter.

    Parameters
    ----------
    activation_func : string
        activation function of neurons, choices: sigmoid, tanh, rect, softsign
    x : Theano variable
        data

    Returns
    -------
    Theano function
        activation function
    """

    if activation_func == 'tanh':
        return T.tanh(x)
    if activation_func == 'sigmoid':
        return T.nnet.sigmoid(x)
    if activation_func == 'rect':
        return T.maximum(0, x)
    if activation_func == 'softsign':
        from theano.sandbox.softsign import softsign
        return softsign(x)

    raise ValueError('unknown activation function: %s' % activation_func)
Ejemplo n.º 3
0
    def new_filters_expbounds(
        cls, rng, input, n_in, n_out, n_terms, dtype=None, eps=1e-1, exponent_range=(1.0, 3.0), filter_range=1.0
    ):
        """Return a KouhLayer instance with random parameters

        The parameters are drawn on a range [typically] suitable for fine-tuning by gradient
        descent.


        :param input: a tensor of shape (n_examples, n_in)

        :type n_in: positive int
        :param n_in: number of input dimensions

        :type n_out: positive int
        :param n_out: number of dimensions in rval.output

        :param nterms: each (of n_out) complex-cell firing rate will be determined from this
        many 'simple cell' responses.

        :param eps: this amount is added to the softplus of filter responses as a baseline
        firing rate (that prevents a subsequent error from ``pow(0, p)``)

        :returns: KouhLayer instance with freshly-allocated random weights.

        """
        if input.type.ndim != 2:
            raise TypeError("matrix expected for input")

        if dtype is None:
            dtype = input.dtype
        _logger.debug("dtype %s" % dtype)

        def shared_uniform(low, high, size, name):
            return _shared_uniform(rng, low, high, size, dtype, name)

        f_list = [
            shared_uniform(
                low=-2.0 / numpy.sqrt(n_in), high=2.0 / numpy.sqrt(n_in), size=(n_in, n_out), name="f_%i" % i
            )
            for i in xrange(n_terms)
        ]

        b_list = [shared_uniform(low=0, high=0.01, size=(n_out,), name="b_%i" % i) for i in xrange(n_terms)]
        # x_list = [theano._asarray(eps, dtype=dtype)+softplus(tensor.dot(input, f_list[i])) for i in xrange(n_terms)]
        filter_range = theano._asarray(filter_range, dtype=dtype)
        half_filter_range = theano._asarray(filter_range / 2, dtype=dtype)
        x_list = [
            theano._asarray(filter_range + eps, dtype=dtype)
            + half_filter_range * softsign(tensor.dot(input, f_list[i]) + b_list[i])
            for i in xrange(n_terms)
        ]

        rval = cls.new_expbounds(rng, x_list, n_out, dtype=dtype, params=f_list + b_list, exponent_range=exponent_range)
        rval.f_list = f_list
        rval.input = input  # add the input to the returned object
        rval.filter_l1 = sum(abs(fi).sum() for fi in f_list)
        rval.filter_l2_sqr = sum((fi ** 2).sum() for fi in f_list)
        return rval
Ejemplo n.º 4
0
def activation_function(r):
    from hyperparameters import HYPERPARAMETERS
    if HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "sigmoid":
        return sigmoid(r)
    elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "tanh":
        return t.tanh(r)
    elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "softsign":
        from theano.sandbox.softsign import softsign
        return softsign(r)
    else:
        assert 0
Ejemplo n.º 5
0
def activation_function(r):
    from hyperparameters import HYPERPARAMETERS
    if HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "sigmoid":
        return sigmoid(r)
    elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "tanh":
        return t.tanh(r)
    elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "softsign":
        from theano.sandbox.softsign import softsign
        return softsign(r)
    else:
        assert 0
Ejemplo n.º 6
0
    def new_expbounds(cls, rng, x_list, n_out, dtype=None, params=None, updates=None, exponent_range=(1.0, 3.0)):
        """
        """
        if params is None:
            params = []
        if updates is None:
            updates = []
        if dtype is None:
            dtype = x_list[0].dtype
        n_terms = len(x_list)

        def shared_uniform(low, high, size, name):
            return _shared_uniform(rng, low, high, size, dtype, name)

        use_softmax_w = True

        if use_softmax_w:
            w = shared_uniform(low=-0.1, high=0.1, size=(n_out, n_terms), name="Kouh2008::w")
            w_sm = theano.tensor.nnet.softmax(w)
            w_list = [w_sm[:, i] for i in xrange(n_terms)]
            w_l1 = abs(w).sum()
            w_l2_sqr = (w ** 2).sum()
        else:
            w_list = [
                shared_uniform(low=-2.0 / n_terms, high=2.0 / n_terms, size=(n_out,), name="w_%i" % i)
                for i in xrange(n_terms)
            ]
            w_l1 = sum(abs(wi).sum() for wi in w_list)
            w_l2_sqr = sum((wi ** 2).sum() for wi in w_list)

        e_range_low, e_range_high = exponent_range
        e_range_low = theano._asarray(e_range_low, dtype=dtype)
        e_range_high = theano._asarray(e_range_high, dtype=dtype)
        e_range_mag = e_range_high - e_range_low
        if e_range_mag < 0:
            raise ValueError("exponent range must have low <= high")

        p_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="p")
        q_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="q")
        r_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="r")
        k_unbounded = shared_uniform(low=-0.2, high=0.2, size=(n_out,), name="k")  # biases

        p = tensor.nnet.sigmoid(p_unbounded) * e_range_mag + e_range_low
        q = tensor.nnet.sigmoid(q_unbounded) * e_range_mag + e_range_low
        r = tensor.nnet.sigmoid(r_unbounded) * theano._asarray(
            1.0 / e_range_low - 1.0 / e_range_high, dtype=dtype
        ) + theano._asarray(1.0 / e_range_high, dtype=dtype)

        k = softsign(k_unbounded)

        if use_softmax_w:
            rval = cls(
                w_list,
                x_list,
                p,
                q,
                r,
                k,
                params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded, w] + params,
                updates=updates,
            )
        else:
            rval = cls(
                w_list,
                x_list,
                p,
                q,
                r,
                k,
                params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded] + w_list + params,
                updates=updates,
            )
        rval.p_unbounded = p_unbounded
        rval.q_unbounded = q_unbounded
        rval.r_unbounded = r_unbounded
        rval.k_unbounded = k_unbounded
        rval.exp_l1 = abs(p_unbounded).sum() + abs(q_unbounded).sum() + abs(r_unbounded).sum()
        rval.exp_l2_sqr = (p_unbounded ** 2).sum() + (q_unbounded ** 2).sum() + (r_unbounded ** 2).sum()
        rval.w_l1 = w_l1
        rval.w_l2_sqr = w_l2_sqr
        return rval
Ejemplo n.º 7
0
def softsign(x):
    return T_softsign.softsign(x)
    def new_expbounds(cls,
                      rng,
                      x_list,
                      n_out,
                      dtype=None,
                      params=None,
                      updates=None,
                      exponent_range=(1.0, 3.0)):
        """
        """
        if params is None:
            params = []
        if updates is None:
            updates = []
        if dtype is None:
            dtype = x_list[0].dtype
        n_terms = len(x_list)

        def shared_uniform(low, high, size, name):
            return _shared_uniform(rng, low, high, size, dtype, name)

        use_softmax_w = True

        if use_softmax_w:
            w = shared_uniform(low=-.1,
                               high=.1,
                               size=(n_out, n_terms),
                               name='Kouh2008::w')
            w_sm = theano.tensor.nnet.softmax(w)
            w_list = [w_sm[:, i] for i in xrange(n_terms)]
            w_l1 = abs(w).sum()
            w_l2_sqr = (w**2).sum()
        else:
            w_list = [
                shared_uniform(low=-2.0 / n_terms,
                               high=2.0 / n_terms,
                               size=(n_out, ),
                               name='w_%i' % i) for i in xrange(n_terms)
            ]
            w_l1 = sum(abs(wi).sum() for wi in w_list)
            w_l2_sqr = sum((wi**2).sum() for wi in w_list)

        e_range_low, e_range_high = exponent_range
        e_range_low = theano._asarray(e_range_low, dtype=dtype)
        e_range_high = theano._asarray(e_range_high, dtype=dtype)
        e_range_mag = e_range_high - e_range_low
        if e_range_mag < 0:
            raise ValueError('exponent range must have low <= high')

        p_unbounded = shared_uniform(low=-0.1,
                                     high=0.1,
                                     size=(n_out, ),
                                     name='p')
        q_unbounded = shared_uniform(low=-0.1,
                                     high=0.1,
                                     size=(n_out, ),
                                     name='q')
        r_unbounded = shared_uniform(low=-0.1,
                                     high=0.1,
                                     size=(n_out, ),
                                     name='r')
        k_unbounded = shared_uniform(low=-0.2,
                                     high=0.2,
                                     size=(n_out, ),
                                     name='k')  # biases

        p = tensor.nnet.sigmoid(p_unbounded) * e_range_mag + e_range_low
        q = tensor.nnet.sigmoid(q_unbounded) * e_range_mag + e_range_low
        r = tensor.nnet.sigmoid(r_unbounded) * \
                theano._asarray(1.0/e_range_low - 1.0/e_range_high, dtype=dtype) \
                + theano._asarray(1.0/e_range_high, dtype=dtype)

        k = softsign(k_unbounded)

        if use_softmax_w:
            rval = cls(
                w_list,
                x_list,
                p,
                q,
                r,
                k,
                params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded, w
                        ] + params,
                updates=updates)
        else:
            rval = cls(
                w_list,
                x_list,
                p,
                q,
                r,
                k,
                params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded] +
                w_list + params,
                updates=updates)
        rval.p_unbounded = p_unbounded
        rval.q_unbounded = q_unbounded
        rval.r_unbounded = r_unbounded
        rval.k_unbounded = k_unbounded
        rval.exp_l1 = abs(p_unbounded).sum() + abs(q_unbounded).sum() + abs(
            r_unbounded).sum()
        rval.exp_l2_sqr = (p_unbounded**2).sum() + (q_unbounded**2).sum() + (
            r_unbounded**2).sum()
        rval.w_l1 = w_l1
        rval.w_l2_sqr = w_l2_sqr
        return rval
    def new_filters_expbounds(cls,
                              rng,
                              input,
                              n_in,
                              n_out,
                              n_terms,
                              dtype=None,
                              eps=1e-1,
                              exponent_range=(1.0, 3.0),
                              filter_range=1.0):
        """Return a KouhLayer instance with random parameters

        The parameters are drawn on a range [typically] suitable for fine-tuning by gradient
        descent.


        :param input: a tensor of shape (n_examples, n_in)

        :type n_in: positive int
        :param n_in: number of input dimensions

        :type n_out: positive int
        :param n_out: number of dimensions in rval.output

        :param nterms: each (of n_out) complex-cell firing rate will be determined from this
        many 'simple cell' responses.

        :param eps: this amount is added to the softplus of filter responses as a baseline
        firing rate (that prevents a subsequent error from ``pow(0, p)``)

        :returns: KouhLayer instance with freshly-allocated random weights.

        """
        if input.type.ndim != 2:
            raise TypeError('matrix expected for input')

        if dtype is None:
            dtype = input.dtype
        _logger.debug('dtype %s' % dtype)

        def shared_uniform(low, high, size, name):
            return _shared_uniform(rng, low, high, size, dtype, name)

        f_list = [
            shared_uniform(low=-2.0 / numpy.sqrt(n_in),
                           high=2.0 / numpy.sqrt(n_in),
                           size=(n_in, n_out),
                           name='f_%i' % i) for i in xrange(n_terms)
        ]

        b_list = [
            shared_uniform(low=0, high=.01, size=(n_out, ), name='b_%i' % i)
            for i in xrange(n_terms)
        ]
        #x_list = [theano._asarray(eps, dtype=dtype)+softplus(tensor.dot(input, f_list[i])) for i in xrange(n_terms)]
        filter_range = theano._asarray(filter_range, dtype=dtype)
        half_filter_range = theano._asarray(filter_range / 2, dtype=dtype)
        x_list = [
            theano._asarray(filter_range + eps, dtype=dtype) +
            half_filter_range *
            softsign(tensor.dot(input, f_list[i]) + b_list[i])
            for i in xrange(n_terms)
        ]

        rval = cls.new_expbounds(rng,
                                 x_list,
                                 n_out,
                                 dtype=dtype,
                                 params=f_list + b_list,
                                 exponent_range=exponent_range)
        rval.f_list = f_list
        rval.input = input  # add the input to the returned object
        rval.filter_l1 = sum(abs(fi).sum() for fi in f_list)
        rval.filter_l2_sqr = sum((fi**2).sum() for fi in f_list)
        return rval