def get_activation_func(activation_func, x):
"""Returns a Theano activation function based on the given parameter.
Parameters
----------
activation_func : string
activation function of neurons, choices: sigmoid, tanh, rect, softsign
x : Theano variable
data
Returns
-------
Theano function
activation function
"""
if activation_func == 'tanh':
return T.tanh(x)
if activation_func == 'sigmoid':
return T.nnet.sigmoid(x)
if activation_func == 'rect':
return T.maximum(0, x)
if activation_func == 'softsign':
from theano.sandbox.softsign import softsign
return softsign(x)
raise ValueError('unknown activation function: %s' % activation_func)
def get_activation_func(activation_func, x):
"""Returns a Theano activation function based on the given parameter.
Parameters
----------
activation_func : string
activation function of neurons, choices: sigmoid, tanh, rect, softsign
x : Theano variable
data
Returns
-------
Theano function
activation function
"""
if activation_func == 'tanh':
return T.tanh(x)
if activation_func == 'sigmoid':
return T.nnet.sigmoid(x)
if activation_func == 'rect':
return T.maximum(0, x)
if activation_func == 'softsign':
from theano.sandbox.softsign import softsign
return softsign(x)
raise ValueError('unknown activation function: %s' % activation_func)
def new_filters_expbounds(
cls, rng, input, n_in, n_out, n_terms, dtype=None, eps=1e-1, exponent_range=(1.0, 3.0), filter_range=1.0
):
"""Return a KouhLayer instance with random parameters
The parameters are drawn on a range [typically] suitable for fine-tuning by gradient
descent.
:param input: a tensor of shape (n_examples, n_in)
:type n_in: positive int
:param n_in: number of input dimensions
:type n_out: positive int
:param n_out: number of dimensions in rval.output
:param nterms: each (of n_out) complex-cell firing rate will be determined from this
many 'simple cell' responses.
:param eps: this amount is added to the softplus of filter responses as a baseline
firing rate (that prevents a subsequent error from ``pow(0, p)``)
:returns: KouhLayer instance with freshly-allocated random weights.
"""
if input.type.ndim != 2:
raise TypeError("matrix expected for input")
if dtype is None:
dtype = input.dtype
_logger.debug("dtype %s" % dtype)
def shared_uniform(low, high, size, name):
return _shared_uniform(rng, low, high, size, dtype, name)
f_list = [
shared_uniform(
low=-2.0 / numpy.sqrt(n_in), high=2.0 / numpy.sqrt(n_in), size=(n_in, n_out), name="f_%i" % i
)
for i in xrange(n_terms)
]
b_list = [shared_uniform(low=0, high=0.01, size=(n_out,), name="b_%i" % i) for i in xrange(n_terms)]
# x_list = [theano._asarray(eps, dtype=dtype)+softplus(tensor.dot(input, f_list[i])) for i in xrange(n_terms)]
filter_range = theano._asarray(filter_range, dtype=dtype)
half_filter_range = theano._asarray(filter_range / 2, dtype=dtype)
x_list = [
theano._asarray(filter_range + eps, dtype=dtype)
+ half_filter_range * softsign(tensor.dot(input, f_list[i]) + b_list[i])
for i in xrange(n_terms)
]
rval = cls.new_expbounds(rng, x_list, n_out, dtype=dtype, params=f_list + b_list, exponent_range=exponent_range)
rval.f_list = f_list
rval.input = input # add the input to the returned object
rval.filter_l1 = sum(abs(fi).sum() for fi in f_list)
rval.filter_l2_sqr = sum((fi ** 2).sum() for fi in f_list)
return rval
def activation_function(r):
from hyperparameters import HYPERPARAMETERS
if HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "sigmoid":
return sigmoid(r)
elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "tanh":
return t.tanh(r)
elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "softsign":
from theano.sandbox.softsign import softsign
return softsign(r)
else:
assert 0
def activation_function(r):
from hyperparameters import HYPERPARAMETERS
if HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "sigmoid":
return sigmoid(r)
elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "tanh":
return t.tanh(r)
elif HYPERPARAMETERS["ACTIVATION_FUNCTION"] == "softsign":
from theano.sandbox.softsign import softsign
return softsign(r)
else:
assert 0
def new_expbounds(cls, rng, x_list, n_out, dtype=None, params=None, updates=None, exponent_range=(1.0, 3.0)):
"""
"""
if params is None:
params = []
if updates is None:
updates = []
if dtype is None:
dtype = x_list[0].dtype
n_terms = len(x_list)
def shared_uniform(low, high, size, name):
return _shared_uniform(rng, low, high, size, dtype, name)
use_softmax_w = True
if use_softmax_w:
w = shared_uniform(low=-0.1, high=0.1, size=(n_out, n_terms), name="Kouh2008::w")
w_sm = theano.tensor.nnet.softmax(w)
w_list = [w_sm[:, i] for i in xrange(n_terms)]
w_l1 = abs(w).sum()
w_l2_sqr = (w ** 2).sum()
else:
w_list = [
shared_uniform(low=-2.0 / n_terms, high=2.0 / n_terms, size=(n_out,), name="w_%i" % i)
for i in xrange(n_terms)
]
w_l1 = sum(abs(wi).sum() for wi in w_list)
w_l2_sqr = sum((wi ** 2).sum() for wi in w_list)
e_range_low, e_range_high = exponent_range
e_range_low = theano._asarray(e_range_low, dtype=dtype)
e_range_high = theano._asarray(e_range_high, dtype=dtype)
e_range_mag = e_range_high - e_range_low
if e_range_mag < 0:
raise ValueError("exponent range must have low <= high")
p_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="p")
q_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="q")
r_unbounded = shared_uniform(low=-0.1, high=0.1, size=(n_out,), name="r")
k_unbounded = shared_uniform(low=-0.2, high=0.2, size=(n_out,), name="k") # biases
p = tensor.nnet.sigmoid(p_unbounded) * e_range_mag + e_range_low
q = tensor.nnet.sigmoid(q_unbounded) * e_range_mag + e_range_low
r = tensor.nnet.sigmoid(r_unbounded) * theano._asarray(
1.0 / e_range_low - 1.0 / e_range_high, dtype=dtype
) + theano._asarray(1.0 / e_range_high, dtype=dtype)
k = softsign(k_unbounded)
if use_softmax_w:
rval = cls(
w_list,
x_list,
p,
q,
r,
k,
params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded, w] + params,
updates=updates,
)
else:
rval = cls(
w_list,
x_list,
p,
q,
r,
k,
params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded] + w_list + params,
updates=updates,
)
rval.p_unbounded = p_unbounded
rval.q_unbounded = q_unbounded
rval.r_unbounded = r_unbounded
rval.k_unbounded = k_unbounded
rval.exp_l1 = abs(p_unbounded).sum() + abs(q_unbounded).sum() + abs(r_unbounded).sum()
rval.exp_l2_sqr = (p_unbounded ** 2).sum() + (q_unbounded ** 2).sum() + (r_unbounded ** 2).sum()
rval.w_l1 = w_l1
rval.w_l2_sqr = w_l2_sqr
return rval
def softsign(x):
return T_softsign.softsign(x)
def new_expbounds(cls,
rng,
x_list,
n_out,
dtype=None,
params=None,
updates=None,
exponent_range=(1.0, 3.0)):
"""
"""
if params is None:
params = []
if updates is None:
updates = []
if dtype is None:
dtype = x_list[0].dtype
n_terms = len(x_list)
def shared_uniform(low, high, size, name):
return _shared_uniform(rng, low, high, size, dtype, name)
use_softmax_w = True
if use_softmax_w:
w = shared_uniform(low=-.1,
high=.1,
size=(n_out, n_terms),
name='Kouh2008::w')
w_sm = theano.tensor.nnet.softmax(w)
w_list = [w_sm[:, i] for i in xrange(n_terms)]
w_l1 = abs(w).sum()
w_l2_sqr = (w**2).sum()
else:
w_list = [
shared_uniform(low=-2.0 / n_terms,
high=2.0 / n_terms,
size=(n_out, ),
name='w_%i' % i) for i in xrange(n_terms)
]
w_l1 = sum(abs(wi).sum() for wi in w_list)
w_l2_sqr = sum((wi**2).sum() for wi in w_list)
e_range_low, e_range_high = exponent_range
e_range_low = theano._asarray(e_range_low, dtype=dtype)
e_range_high = theano._asarray(e_range_high, dtype=dtype)
e_range_mag = e_range_high - e_range_low
if e_range_mag < 0:
raise ValueError('exponent range must have low <= high')
p_unbounded = shared_uniform(low=-0.1,
high=0.1,
size=(n_out, ),
name='p')
q_unbounded = shared_uniform(low=-0.1,
high=0.1,
size=(n_out, ),
name='q')
r_unbounded = shared_uniform(low=-0.1,
high=0.1,
size=(n_out, ),
name='r')
k_unbounded = shared_uniform(low=-0.2,
high=0.2,
size=(n_out, ),
name='k') # biases
p = tensor.nnet.sigmoid(p_unbounded) * e_range_mag + e_range_low
q = tensor.nnet.sigmoid(q_unbounded) * e_range_mag + e_range_low
r = tensor.nnet.sigmoid(r_unbounded) * \
theano._asarray(1.0/e_range_low - 1.0/e_range_high, dtype=dtype) \
+ theano._asarray(1.0/e_range_high, dtype=dtype)
k = softsign(k_unbounded)
if use_softmax_w:
rval = cls(
w_list,
x_list,
p,
q,
r,
k,
params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded, w
] + params,
updates=updates)
else:
rval = cls(
w_list,
x_list,
p,
q,
r,
k,
params=[p_unbounded, q_unbounded, r_unbounded, k_unbounded] +
w_list + params,
updates=updates)
rval.p_unbounded = p_unbounded
rval.q_unbounded = q_unbounded
rval.r_unbounded = r_unbounded
rval.k_unbounded = k_unbounded
rval.exp_l1 = abs(p_unbounded).sum() + abs(q_unbounded).sum() + abs(
r_unbounded).sum()
rval.exp_l2_sqr = (p_unbounded**2).sum() + (q_unbounded**2).sum() + (
r_unbounded**2).sum()
rval.w_l1 = w_l1
rval.w_l2_sqr = w_l2_sqr
return rval
def new_filters_expbounds(cls,
rng,
input,
n_in,
n_out,
n_terms,
dtype=None,
eps=1e-1,
exponent_range=(1.0, 3.0),
filter_range=1.0):
"""Return a KouhLayer instance with random parameters
The parameters are drawn on a range [typically] suitable for fine-tuning by gradient
descent.
:param input: a tensor of shape (n_examples, n_in)
:type n_in: positive int
:param n_in: number of input dimensions
:type n_out: positive int
:param n_out: number of dimensions in rval.output
:param nterms: each (of n_out) complex-cell firing rate will be determined from this
many 'simple cell' responses.
:param eps: this amount is added to the softplus of filter responses as a baseline
firing rate (that prevents a subsequent error from ``pow(0, p)``)
:returns: KouhLayer instance with freshly-allocated random weights.
"""
if input.type.ndim != 2:
raise TypeError('matrix expected for input')
if dtype is None:
dtype = input.dtype
_logger.debug('dtype %s' % dtype)
def shared_uniform(low, high, size, name):
return _shared_uniform(rng, low, high, size, dtype, name)
f_list = [
shared_uniform(low=-2.0 / numpy.sqrt(n_in),
high=2.0 / numpy.sqrt(n_in),
size=(n_in, n_out),
name='f_%i' % i) for i in xrange(n_terms)
]
b_list = [
shared_uniform(low=0, high=.01, size=(n_out, ), name='b_%i' % i)
for i in xrange(n_terms)
]
#x_list = [theano._asarray(eps, dtype=dtype)+softplus(tensor.dot(input, f_list[i])) for i in xrange(n_terms)]
filter_range = theano._asarray(filter_range, dtype=dtype)
half_filter_range = theano._asarray(filter_range / 2, dtype=dtype)
x_list = [
theano._asarray(filter_range + eps, dtype=dtype) +
half_filter_range *
softsign(tensor.dot(input, f_list[i]) + b_list[i])
for i in xrange(n_terms)
]
rval = cls.new_expbounds(rng,
x_list,
n_out,
dtype=dtype,
params=f_list + b_list,
exponent_range=exponent_range)
rval.f_list = f_list
rval.input = input # add the input to the returned object
rval.filter_l1 = sum(abs(fi).sum() for fi in f_list)
rval.filter_l2_sqr = sum((fi**2).sum() for fi in f_list)
return rval