def ber(self, y):
tp = T.and_(T.eq(y, 1), T.eq(self.y_pred, 1)).sum()
tn = T.and_(T.eq(y, 0), T.eq(self.y_pred, 0)).sum()
fp = T.and_(T.eq(y, 0), T.eq(self.y_pred, 1)).sum()
fn = T.and_(T.eq(y, 1), T.eq(self.y_pred, 0)).sum()
ber = 0.5 * (T.true_div(fp, tp + fp) + T.true_div(fn, tn + fn))
return ber
def call(self, inputs, mask=None):
if not isinstance(inputs, list) or len(inputs) <= 1:
raise TypeError('SpkLifeLongMemory must be called on a list of tensors '
'(at least 2). Got: ' + str(inputs))
# (None(batch), 1), index of speaker
target_spk_l = inputs[0]
target_spk_l = K.reshape(target_spk_l, (target_spk_l.shape[0], ))
if K.dtype(target_spk_l) != 'int32':
target_spk_l = K.cast(target_spk_l, 'int32')
# (None(batch), embed_dim)
spk_vector_l = inputs[1]
# Start to update life-long memory based on the learned speech vector
# First do normalization
spk_vector_eps = K.switch(K.equal(spk_vector_l, 0.), np.spacing(1), spk_vector_l) # avoid zero
spk_vector_eps = K.sqrt(K.sum(spk_vector_eps**2, axis=1))
spk_vector_eps = spk_vector_eps.dimshuffle((0, 'x'))
spk_vector = T.true_div(spk_vector_l, K.repeat_elements(spk_vector_eps, self.vec_dim, axis=1))
# Store speech vector into life-long memory according to the speaker identity.
life_long_mem = T.inc_subtensor(self.life_long_mem[target_spk_l, :], spk_vector)
# Normalization for memory
life_long_mem_eps = K.switch(K.equal(life_long_mem, 0.), np.spacing(1), life_long_mem) # avoid 0
life_long_mem_eps = K.sqrt(K.sum(life_long_mem_eps**2, axis=1))
life_long_mem_eps = life_long_mem_eps.dimshuffle((0, 'x'))
life_long_mem = T.true_div(life_long_mem, K.repeat_elements(life_long_mem_eps, self.vec_dim, axis=1))
# (None(batch), spk_size, embed_dim)
return life_long_mem
def theano_periodic_sinc(in_sig, bandwidth):
eps = TT.constant(1e-10)
denominator = TT.mul(TT.sin(TT.true_div(in_sig, bandwidth)), bandwidth)
idx_modi = TT.lt(TT.abs_(denominator), eps)
numerator = TT.switch(idx_modi, TT.cos(in_sig), TT.sin(in_sig))
denominator = TT.switch(idx_modi,
TT.cos(TT.true_div(in_sig,
bandwidth)), denominator)
return TT.true_div(numerator, denominator)
def f1_score(self, y):
n_total = y.shape[0]
n_relevant_documents_predicted = T.sum(T.eq(T.ones(self.y_pred.shape), self.y_pred))
two_vector = T.add(T.ones(self.y_pred.shape), T.ones(self.y_pred.shape))
n_relevant_predicted_correctly = T.sum(T.eq(T.add(self.y_pred, y), two_vector))
precision = T.true_div(n_relevant_predicted_correctly, n_relevant_documents_predicted)
recall = T.true_div(n_relevant_predicted_correctly, n_total)
f1_score = T.mul(2.0, T.true_div(T.mul(precision, recall), T.add(precision, recall)))
return [f1_score, precision, recall]
def BGMM1_lpdf(node, sample, kw):
r, weights, mus, sigmas, low, high, draw_shape = node.inputs
assert weights.ndim == 1
assert mus.ndim == 1
assert sigmas.ndim == 1
_sample = sample
if sample.ndim != 1:
sample = sample.flatten()
erf = theano.tensor.erf
effective_weights = 0.5 * weights * (erf((high - mus) / sigmas) - erf(
(low - mus) / sigmas))
dist = (sample.dimshuffle(0, 'x') - mus)
mahal = ((dist**2) / (sigmas**2))
# POSTCONDITION: mahal.shape == (n_samples, n_components)
Z = tensor.sqrt(2 * numpy.pi * sigmas**2)
rval = tensor.log(
tensor.sum(tensor.true_div(
tensor.exp(-.5 * mahal) * weights, Z * effective_weights.sum()),
axis=1))
if not sample is _sample:
rval = rval.reshape(_sample.shape)
assert rval.ndim != 1
return rval
def BGMM1_lpdf(node, sample, kw):
r, weights, mus, sigmas, low, high, draw_shape = node.inputs
assert weights.ndim == 1
assert mus.ndim == 1
assert sigmas.ndim == 1
_sample = sample
if sample.ndim != 1:
sample = sample.flatten()
erf = theano.tensor.erf
effective_weights = 0.5 * weights * (erf((high - mus) / sigmas) - erf((low - mus) / sigmas))
dist = sample.dimshuffle(0, "x") - mus
mahal = (dist ** 2) / (sigmas ** 2)
# POSTCONDITION: mahal.shape == (n_samples, n_components)
Z = tensor.sqrt(2 * numpy.pi * sigmas ** 2)
rval = tensor.log(
tensor.sum(tensor.true_div(tensor.exp(-0.5 * mahal) * weights, Z * effective_weights.sum()), axis=1)
)
if not sample is _sample:
rval = rval.reshape(_sample.shape)
assert rval.ndim != 1
return rval
def minus_corr(u, v):
um = T.sub(u, T.mean(u))
vm = T.sub(v, T.mean(v))
r_num = T.sum(T.mul(um, vm))
r_den = T.sqrt(T.mul(T.sum(T.sqr(um)), T.sum(T.sqr(vm))))
r = T.true_div(r_num, r_den)
r = T.neg(r)
return r
def division(self):
"""
Note: This maps to the `true_div` operation.
"""
assert len(self.children) == 2
n = self.get_input(0)
d = self.get_input(1)
return T.true_div(n, d)
def SlopeLin2(x, slope):
"""
Linear unit with different slopes
:param slope: slope of negative quadrant
:return: x if x > 0 else x/slope
"""
import theano.tensor as T
return T.switch(T.gt(x, 0), x, T.true_div(x, slope))
def add_termination(self,
name,
pstc,
decoded_input=None,
encoded_input=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population multiplied by this termination's
transform matrix
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
"""
# make sure one and only one of
# (decoded_input, encoded_input) is specified
if decoded_input is not None: assert (encoded_input is None)
elif encoded_input is not None: assert (decoded_input is None)
else: assert False
if decoded_input:
if self.mode is not 'direct':
# rescale decoded_input by this neuron's radius
source = TT.true_div(decoded_input, self.radius)
# ignore radius in direct mode
else:
source = decoded_input
name = self.get_unique_name(name, self.decoded_input)
self.decoded_input[name] = filter.Filter(name=name,
pstc=pstc,
source=source,
shape=(self.array_size,
self.dimensions))
elif encoded_input:
name = self.get_unique_name(name, self.encoded_input)
self.encoded_input[name] = filter.Filter(name=name,
pstc=pstc,
source=encoded_input,
shape=(self.array_size,
self.neurons_num))
def quadratic_weighted_kappa_loss(y_true, y_pred):
min_rating = T.minimum(T.min(y_true), T.min(y_pred))
max_rating = T.maximum(T.max(y_true), T.max(y_pred))
hist_true = T.bincount(y_true, minlength=max_rating)
hist_pred = T.bincount(y_pred, minlength=max_rating)
num_ratings = (max_rating - min_rating) + 1
num_scored = float(len(y_true))
numerator = T.zeros(1)
denominator = T.zeros(1)
z = T.zeros(len(y_true))
for i_true in range(min_rating, max_rating + 1):
for j_pred in range(min_rating, max_rating + 1):
expected = T.true_div(T.mul(hist_true[i_true], hist_pred[j_pred]), num_scored)
d = T.true_div(T.sqr(i_true - j_pred), T.sqr(num_ratings - 1.))
conf_mat_cell = T.sum(T.and_(T.eq(T.sub(y_true, i_true), z), T.eq(T.sub(y_pred, j_pred), z)))
numerator = T.add(numerator, T.true_div(T.mul(d, conf_mat_cell), num_scored))
denominator = T.add(denominator, T.true_div(T.mul(d, expected), num_scored))
return T.true_div(numerator, denominator)
def add_filtered_input(self, pstc, decoded_input=None,
encoded_input=None, learn_input=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population multiplied by this termination's
transform matrix
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
"""
# make sure one and only one of
# (decoded_input, encoded_input, learn_input) is specified
if decoded_input is not None:
assert (encoded_input is None) and (learn_input is None)
elif encoded_input is not None:
assert (decoded_input is None) and (learn_input is None)
elif learn_input is not None:
assert (decoded_input is None) and (encoded_input is None)
else:
assert False
# make sure there's an accumulator for given pstc
if pstc not in self.accumulators:
self.accumulators[pstc] = Accumulator(self, pstc)
# add this termination's contribution to
# the set of terminations with the same pstc
if decoded_input:
# rescale decoded_input by this neuron's radius
# to put us in the right range
self.accumulators[pstc].add_decoded_input(
TT.true_div(decoded_input, self.radius))
elif encoded_input:
self.accumulators[pstc].add_encoded_input(encoded_input)
elif learn_input:
self.accumulators[pstc].add_learn_input(learn_input)
def get_recall(self, y_true):
""" ~~~~~~~~~~ Untested ~~~~~~~~~~
Returns recall/sensitivity of binary predictions.
:type y_true: Theano shared variable
:param y_true: Target
:rtype Theano shared variable
:return recall/sensitivity
"""
true_positives = T.sum(T.mul(T.eq(self.y_pred, 1.), T.eq(y_true, 1.)))
condition_positives = T.sum(T.eq(y_true, 1.))
return T.true_div(true_positives, condition_positives)
def get_precision(self, y_true):
""" ~~~~~~~~~~ Untested ~~~~~~~~~~
Returns precision of binary predictions.
:type y_true: Theano shared variable
:param y_true: Target
:rtype Theano shared variable
:return precision
"""
true_positives = T.sum(T.mul(T.eq(self.y_pred, 1.), T.eq(y_true, 1.)))
test_outcome_positives = T.sum(T.eq(self.y_pred, 1.))
return T.true_div(true_positives, test_outcome_positives)
def get_specificity(self, y_true):
""" ~~~~~~~~~~ Untested ~~~~~~~~~~
Returns specificity of binary predictions.
:type y_true: Theano shared variable
:param y_true: Target
:rtype Theano shared variable
:return specificity
"""
true_negatives = T.sum(T.mul(T.eq(self.y_pred, 0.), T.eq(y_true, 0.)))
condition_negatives = T.sum(T.eq(y_true, 0.))
return T.true_div(true_negatives, condition_negatives)
def call(self, inputs, mask=None):
if not isinstance(inputs, list) or len(inputs) <= 1:
raise TypeError(
'SpkLifeLongMemory must be called on a list of tensors '
'(at least 2). Got: ' + str(inputs))
# (None(batch), 1)，说话人对应的idx
target_spk_l = inputs[0]
target_spk_l = K.reshape(target_spk_l, (target_spk_l.shape[0], ))
if K.dtype(target_spk_l) != 'int32':
target_spk_l = K.cast(target_spk_l, 'int32')
# (None(batch), embed_dim)
spk_vector_l = inputs[1]
# 开始更新长时记忆单元，训练好的语音向量
# 先对语音向量进行 normalization
spk_vector_eps = K.switch(K.equal(spk_vector_l, 0.), np.spacing(1),
spk_vector_l) # 防止分母为0
spk_vector_eps = K.sqrt(K.sum(spk_vector_eps**2, axis=1))
spk_vector_eps = spk_vector_eps.dimshuffle((0, 'x'))
spk_vector = T.true_div(
spk_vector_l,
K.repeat_elements(spk_vector_eps, self.vec_dim, axis=1))
# 根据说话人idx，将语音向量增量式写入长时记忆单元中
life_long_mem = T.inc_subtensor(self.life_long_mem[target_spk_l, :],
spk_vector)
# 对长时Memory进行整体 normalization
life_long_mem_eps = K.switch(K.equal(life_long_mem, 0.), np.spacing(1),
life_long_mem) # 防止分母为0
life_long_mem_eps = K.sqrt(K.sum(life_long_mem_eps**2, axis=1))
life_long_mem_eps = life_long_mem_eps.dimshuffle((0, 'x'))
life_long_mem = T.true_div(
life_long_mem,
K.repeat_elements(life_long_mem_eps, self.vec_dim, axis=1))
# # 更新记忆参数，由于life_long_mem必须是numpy才行，因而不能在这里做，放到外面做。
# K.set_value(self.life_long_mem, life_long_mem)
# # 从长时记忆单元中提取说话人的声纹记忆特征
# spk_life_long_memory = K.gather(self.life_long_mem, target_spk_l)
# (None(batch), spk_size, embed_dim)
return life_long_mem
def add_termination(self, name, pstc, decoded_input=None, encoded_input=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population multiplied by this termination's
transform matrix
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
"""
# make sure one and only one of
# (decoded_input, encoded_input) is specified
if decoded_input is not None:
assert encoded_input is None
elif encoded_input is not None:
assert decoded_input is None
else:
assert False
if decoded_input:
if self.mode is not "direct":
# rescale decoded_input by this neuron's radius
source = TT.true_div(decoded_input, self.radius)
# ignore radius in direct mode
else:
source = decoded_input
name = helpers.get_unique_name(name, self.decoded_input)
self.decoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions)
)
elif encoded_input:
name = helpers.get_unique_name(name, self.encoded_input)
self.encoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=encoded_input, shape=(self.array_size, self.neurons_num)
)
def fit(self, weights, o_error, tpo):
gradients = T.grad(o_error, weights)
updates = []
for c, v, w, g in zip(self.t_cache, self.t_velocity, weights,
gradients):
new_velocity = T.sub(T.mul(tpo["momentum_rate"], v),
T.mul(tpo["learn_rate"], g))
new_cache = T.add(T.mul(tpo["decay_rate"], c),
T.mul(T.sub(1, tpo["decay_rate"]), T.sqr(g)))
new_weights = T.sub(
T.add(w, new_velocity),
T.true_div(T.mul(g, tpo["learn_rate"]),
T.sqrt(T.add(new_cache, 0.1**8))))
updates.append((w, new_weights))
updates.append((v, new_velocity))
updates.append((c, new_cache))
return updates
def add_filtered_input(self, pstc, decoded_input=None, encoded_input=None):
"""Create a new termination that takes the given input (a theano object) and filters it with the given pstc
Adds its contributions to the set of decoded or encoded input with the same pstc
Decoded inputs are represented signals, encoded inputs are neuron activities * weight matrix
Can only have decoded OR encoded input != None
:param float pstc: post-synaptic time constant
:param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix
:param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix
"""
# make sure one and only one of (decoded_input, encoded_input) is specified
assert (decoded_input is None or encoded_input is None)
assert (decoded_input is not None or encoded_input is not None)
if pstc not in self.accumulators: # make sure there's an accumulator for given pstc
self.accumulators[pstc] = Accumulator(self, pstc)
# add this termination's contribution to the set of terminations with the same pstc
if decoded_input is not None:
# rescale decoded_input by this neurons radius to put us in the right range
self.accumulators[pstc].add_decoded_input(TT.true_div(decoded_input, self.radius))
else:
self.accumulators[pstc].add_encoded_input(encoded_input)
def Tnanmean(x,axis=None,keepdims=False):
"""
Theano version of numpy nanmean.
Parameters
----------
x : Theano.tensor or array_like
The array to be nanmeaned
axis : int or None, optional
Axis to do nanmean along.
If None or default, nanmean is done over entire array
keepdims : Boolean, optional
If this is set to True, the axes which are reduced are left in the result as dimensions with size one.
Returns
-------
An array-like
Nan is returned for slices that contain only NaNs.
"""
mask1 = T.switch(T.isnan(x),0,x)
mask2 = T.switch(T.isnan(x),0,1)
return T.true_div(mask1.sum(axis=axis,keepdims=keepdims),mask2.sum(axis=axis,keepdims=keepdims))
def add_filtered_input(self, pstc, decoded_input=None, encoded_input=None):
"""Create a new termination that takes the given input (a theano object) and filters it with the given pstc
Adds its contributions to the set of decoded or encoded input with the same pstc
Decoded inputs are represented signals, encoded inputs are neuron activities * weight matrix
Can only have decoded OR encoded input != None
:param float pstc: post-synaptic time constant
:param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix
:param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix
"""
# make sure one and only one of (decoded_input, encoded_input) is specified
assert (decoded_input is None or encoded_input is None)
assert (decoded_input is not None or encoded_input is not None)
if pstc not in self.accumulators: # make sure there's an accumulator for given pstc
self.accumulators[pstc] = Accumulator(self, pstc)
# add this termination's contribution to the set of terminations with the same pstc
if decoded_input is not None:
# rescale decoded_input by this neurons radius to put us in the right range
self.accumulators[pstc].add_decoded_input(
TT.true_div(decoded_input, self.radius))
else:
self.accumulators[pstc].add_encoded_input(encoded_input)
def Tnanmean(x, axis=None, keepdims=False):
"""
Theano version of numpy nanmean.
Parameters
----------
x : Theano.tensor or array_like
The array to be nanmeaned
axis : int or None, optional
Axis to do nanmean along.
If None or default, nanmean is done over entire array
keepdims : Boolean, optional
If this is set to True, the axes which are reduced are left in the result as dimensions with size one.
Returns
-------
An array-like
Nan is returned for slices that contain only NaNs.
"""
mask1 = T.switch(T.isnan(x), 0, x)
mask2 = T.switch(T.isnan(x), 0, 1)
return T.true_div(mask1.sum(axis=axis, keepdims=keepdims),
mask2.sum(axis=axis, keepdims=keepdims))
def div(x, y):
z = T.true_div(x, y)
if isinstance(get_shape(x), (tuple, list)):
output_shape = auto_infer_shape(T.true_div, x, y)
add_shape(z, output_shape)
return z
def max_f1_score_loss(coding_dist, true_dist):
"""
f1 = (2 * N_WWW) / (N_D + N_W)
"""
def set_2dim(j, matrix):
return T.switch(T.eq(matrix[j], 0), -100, T.log(matrix[j]))
def set_1dim(i, matrix):
matrix_2dim, updates=theano.scan(set_2dim,\
outputs_info=None,\
sequences=T.arange(matrix.shape[1]),\
non_sequences=[matrix[i]])
return matrix_2dim
if true_dist.ndim == coding_dist.ndim:
'''
#parameter
tau=-2
#ce_loss: [batch_size,]
ce_loss = T.log(T.sum(true_dist * (coding_dist + 1e-14), axis=1))
#N_D : >tau=1; <tau=0
N_d = ce_loss
N_D, updates = theano.scan(lambda i, x, y: T.switch(T.lt(x[i], y), 0, 1),\
outputs_info=None,\
sequences=T.arange(N_d.shape[0]),\
non_sequences=[N_d,tau])
N_D = T.sum(N_D, axis=0)
# N_D = T.nnet.sigmoid(N_D)
#N_W: [1, class_numbers]
N_W = T.sum(true_dist ,axis=0)
#N_WWW: [1, class_numbers]
matrix = true_dist * (coding_dist + 1e-14)
matrix_log, updates = theano.scan(set_1dim,\
outputs_info=None,\
sequences=T.arange(matrix.shape[0]),\
non_sequences=[matrix])
#indictor function
I = T.switch(T.lt(matrix_log, tau), 0, 1)
N_WWW = T.sum(I, axis=0)
# N_WWW = T.nnet.sigmoid(N_WWW)
'''
classid = T.argmax(coding_dist, axis=1)
#N_D
def N_D_count_2dim(j, x, y):
return T.switch(T.eq(j, y), 1, 0)
def N_D_count_1dim(i, x, y):
N_d, updates = theano.scan(N_D_count_2dim,\
outputs_info=None,\
sequences=T.arange(x.shape[1]),\
non_sequences=[x[i], y[i]])
return N_d
predict, updates = theano.scan(N_D_count_1dim,\
outputs_info=None,\
sequences=T.arange(coding_dist.shape[0]),\
non_sequences=[coding_dist, classid])
N_D = T.reshape(T.sum(predict, axis=0), (1, -1))
N_D = T.nnet.sigmoid(N_D)
#N_W: [1, class_numbers]
N_W = T.reshape(T.sum(true_dist, axis=0), (1, -1))
#N_WWW: [1, class_numbers]
matrix = true_dist * predict
N_WWW = T.reshape(T.sum(matrix, axis=0), (1, -1))
N_WWW = T.nnet.sigmoid(N_WWW)
#F1-SCORE
F1 = T.true_div(2 * N_WWW, (N_D + N_W + 1e-14))
return F1, F1
else:
print "true_dist.ndim != coding_dist.ndim"
def beam_search_backward(self, top, idx):
max_idx = top[idx]
idx = T.true_div(max_idx, self.target_vocab_size)
max_idx = T.divmod(max_idx, self.target_vocab_size)
return idx, max_idx
theta = np.array([2, 2.5])
tau = np.array([.01, .01])
activation = T.vector('activation')
t = T.scalar('t')
input_duration = T.scalar('input_duration')
input_intensity = T.scalar('input_intensity')
P = input_intensity * ((T.sgn(input_duration - t) + 1) / 2)
corrected_sigmoid = \
1 / (1 + T.exp(-T.mul(a,(T.dot(c,activation) + P) - theta))) \
- 1 / (1 + T.exp(np.multiply(a, theta)))
#corrected_sigmoid = theano.function([t, activation, input_duration, input_intensity], corrected_s)
#d_a = T.true_div(-activation + T.mul( 1 - T.mul(r, activation), corrected_s), tau)
d_a = T.true_div(-activation + T.mul( k - T.mul(r, activation), corrected_sigmoid), tau)
d_activation = theano.function(inputs=[activation, t, input_duration, input_intensity],
outputs=d_a, on_unused_input='warn')
J = theano.function(inputs=[activation, t, input_duration, input_intensity],
outputs=T.jacobian(d_a, activation), on_unused_input='warn')
activation_0 = np.array([0, 0])
t_0 = 0
t_1 = .125
dt = .0001
times = np.arange(t_0, t_1, dt)
intens = 1
duration = .125
params = (duration, intens)
def add_termination(self, name, pstc, decoded_input=None,
encoded_input=None, input_socket=None, transform=None, case=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population (just value, not a shared variable)
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
:param transform:
the transform that needs to be applied (dot product) to the
decoded output of the pre population
:param case:
used to generate an encoded input by applying the transform matrix
onto the decoded pre output is a special way
"""
# make sure one and only one of
# (decoded_input, encoded_input) is specified
if decoded_input is not None: assert (encoded_input is None)
elif encoded_input is not None: assert (decoded_input is None)
else: assert False
if decoded_input is not None and self.mode == 'direct':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
self.decoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.dimensions),
pre_output=pre_output)
# decoded_input in this case will be the output of pre node
elif decoded_input is not None and self.mode == 'spiking':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
source = TT.true_div(source, self.radius)
self.decoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.dimensions),
pre_output=pre_output)
elif encoded_input is not None:
# encoded_input is NOT the shared variable at the origin
pre_output = theano.shared(encoded_input)
source = case(transform, pre_output)
self.encoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.neurons_num),
pre_output=pre_output)
class Ensemble:
"""An ensemble is a collection of neurons representing a vector space.
"""
def __init__(self, neurons, dimensions, dt, tau_ref=0.002, tau_rc=0.02,
max_rate=(200, 300), intercept=(-1.0, 1.0), radius=1.0,
encoders=None, seed=None, neuron_type='lif',
array_size=1, eval_points=None, decoder_noise=0.1,
noise_type='uniform', noise=None, mode='spiking',
is_subensemble=False, decoders=None, bias=None, alpha=None):
self.dt = dt
np.random.seed(97)
if seed is None:
seed = np.random.randint(1000)
self.seed = seed
self.neurons_num = neurons
self.dimensions = dimensions
self.array_size = array_size
self.radius = radius
self.noise = noise
self.noise_type = noise_type
self.decoder_noise = decoder_noise
self.is_subensemble = is_subensemble
self.mode = mode
# make sure that eval_points is the right shape
if eval_points is not None:
eval_points = np.array(eval_points)
if len(eval_points.shape) == 1:
eval_points.shape = [1, eval_points.shape[0]]
self.eval_points = eval_points
# make sure intercept is the right shape
if isinstance(intercept, (int,float)): intercept = [intercept, 1]
elif len(intercept) == 1: intercept.append(1)
self.cache_key = cache.generate_ensemble_key(neurons=neurons,
dimensions=dimensions, tau_rc=tau_rc, tau_ref=tau_ref,
max_rate=max_rate, intercept=intercept, radius=radius,
encoders=(encoders if not is_subensemble else None),
decoder_noise=decoder_noise,
eval_points=eval_points, noise=noise, seed=seed, dt=dt)
# make dictionary for origins
self.origin = {}
# set up a dictionary for decoded_input
self.decoded_input = {}
# if we're creating a spiking ensemble
if self.mode == 'spiking':
# TODO: handle different neuron types,
self.neurons = neuron.types[neuron_type](
size=(array_size, self.neurons_num),
tau_rc=tau_rc, tau_ref=tau_ref)
# compute alpha and bias
self.srng = RandomStreams(seed=seed)
self.max_rate = max_rate
if is_subensemble:
self.bias = bias
self.alpha = alpha
self.encoders = encoders
else:
max_rates = self.srng.uniform(
size=(self.array_size, self.neurons_num),
low=max_rate[0], high=max_rate[1])
threshold = self.srng.uniform(
size=(self.array_size, self.neurons_num),
low=intercept[0], high=intercept[1])
self.alpha, self.bias = theano.function(
[], self.neurons.make_alpha_bias(max_rates, threshold))()
# force to 32 bit for consistency / speed
self.bias = self.bias.astype(FLOAT_TYPE)
# compute encoders
self.encoders = self.make_encoders(encoders=encoders)
# combine encoders and gain for simplification
self.encoders = (self.encoders.T * self.alpha.T).T
self.shared_encoders = theano.shared(self.encoders,
name='ensemble.shared_encoders')
# set up a dictionary for encoded_input connections
self.encoded_input = {}
# list of learned terminations on ensemble
self.learned_terminations = []
# make default origin
self.add_origin('X', func=None, dt=dt,
eval_points=self.eval_points, decoders=decoders)
elif self.mode == 'direct':
# make default origin
self.add_origin('X', func=None, dimensions=self.dimensions*self.array_size)
# reset neurons_num to 0
self.neurons_num = 0
def add_termination(self, name, pstc, decoded_input=None,
encoded_input=None, input_socket=None, transform=None, case=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population (just value, not a shared variable)
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
:param transform:
the transform that needs to be applied (dot product) to the
decoded output of the pre population
:param case:
used to generate an encoded input by applying the transform matrix
onto the decoded pre output is a special way
"""
# make sure one and only one of
# (decoded_input, encoded_input) is specified
if decoded_input is not None: assert (encoded_input is None)
elif encoded_input is not None: assert (decoded_input is None)
else: assert False
if decoded_input is not None and self.mode == 'direct':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
self.decoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.dimensions),
pre_output=pre_output)
# decoded_input in this case will be the output of pre node
elif decoded_input is not None and self.mode == 'spiking':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
source = TT.true_div(source, self.radius)
self.decoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.dimensions),
pre_output=pre_output)
elif encoded_input is not None:
# encoded_input is NOT the shared variable at the origin
pre_output = theano.shared(encoded_input)
source = case(transform, pre_output)
self.encoded_input[name] = filter.Filter(
name=name, pstc=pstc, source=source,
shape=(self.array_size, self.neurons_num),
pre_output=pre_output)
def R2loss(y_true, y_pred):
y_pred = y_pred.flatten()
y_true = y_true.flatten()
tot = T.sum(T.sqr(T.sub(y_true, T.mean(y_true))))
res = T.sum(T.sqr(T.sub(y_true, y_pred)))
return T.true_div(res, tot)
def inner(x):
return T.switch(T.gt(x, 0), x, T.true_div(x, slope))
def l2_normalize(x, dim, epsilon=1e-12):
square_sum = T.sum(T.sqr(x), axis=dim)
x_inv_norm = T.true_div(1, T.sqrt(T.maximum(square_sum, epsilon)))
x_inv_norm = x_inv_norm.dimshuffle(0, 'x')
return T.mul(x, x_inv_norm)
def fit(self,
X_train,
y_train,
error_function,
optimizer,
regularizer=None,
sample_weight=None,
batch_size=128,
nb_epoch=100,
shuffle=True,
verbosity=0,
memory=True):
"""Learns the weights of a factorization machine with mini-batch gradient
descent. The weights that minimize the loss function (across epochs) are
retained."""
# ************************************************************
# * Learning (Symbolic)
# ************************************************************
# *** Loss Function ***
error = error_function.apply(self.y, self.y_hat)
mean_error = T.true_div(T.sum(T.mul(error, self.s)), T.sum(self.s))
loss = mean_error
# regularization
if regularizer is not None:
loss = regularizer.regularize(loss, self.w0[0], self.w1, self.v)
params = [self.w0, self.w1, self.v]
updates = optimizer.update(loss, params, self.e)
theano_train = theano.function(inputs=[self.X, self.y, self.s, self.e],
on_unused_input='ignore',
outputs=loss,
updates=updates,
allow_input_downcast=True)
theano_cost = theano.function(inputs=[self.X, self.y, self.s],
on_unused_input='ignore',
outputs=loss,
allow_input_downcast=True)
# ************************************************************
# * Learning (Numeric)
# ************************************************************
n = X_train.shape[0]
if batch_size > n:
batch_size = n
if sample_weight is None:
sample_weight = np.ones(n)
else:
sample_weight = (float(n) / np.sum(sample_weight)) * sample_weight
min_loss = float('inf')
min_loss_weights = self.get_weights()
for epoch in utils.xrange(1, nb_epoch + 1):
if shuffle:
indices = np.arange(n)
np.random.shuffle(indices)
X_train, y_train = X_train[indices], y_train[indices]
for start in itertools.count(0, batch_size):
if start >= n:
break
stop = min(start + batch_size, n)
theano_train(X_train[start:stop], y_train[start:stop],
sample_weight[start:stop], epoch)
current_loss = theano_cost(X_train, y_train, sample_weight)
if not np.isfinite(current_loss):
raise ArithmeticError("Non-finite loss function.")
if current_loss < min_loss:
min_loss = current_loss
min_loss_weights = self.get_weights()
if verbosity > 0 and (epoch % verbosity == 0):
print('Epoch {}/{}'.format(epoch, nb_epoch))
print(' loss: {}, min_loss: {}'.format(current_loss, min_loss))
weights = min_loss_weights if memory else self.get_weights()
self.set_weights(weights)
class Ensemble:
"""An ensemble is a collection of neurons representing a vector space.
"""
def __init__(self,
neurons,
dimensions,
dt,
tau_ref=0.002,
tau_rc=0.02,
max_rate=(200, 300),
intercept=(-1.0, 1.0),
radius=1.0,
encoders=None,
seed=None,
neuron_type='lif',
array_size=1,
eval_points=None,
decoder_noise=0.1,
noise_type='uniform',
noise=None,
mode='spiking'):
"""Construct an ensemble composed of the specific neuron model,
with the specified neural parameters.
:param int neurons: number of neurons in this population
:param int dimensions:
number of dimensions in the vector space
that these neurons represent
:param float tau_ref: length of refractory period
:param float tau_rc:
RC constant; approximately how long until 2/3
of the threshold voltage is accumulated
:param tuple max_rate:
lower and upper bounds on randomly generated
firing rates for each neuron
:param tuple intercept:
lower and upper bounds on randomly generated
x offsets for each neuron
:param float radius:
the range of input values (-radius:radius)
per dimension this population is sensitive to
:param list encoders: set of possible preferred directions
:param int seed: seed value for random number generator
:param string neuron_type:
type of neuron model to use, options = {'lif'}
:param int array_size: number of sub-populations for network arrays
:param list eval_points:
specific set of points to optimize decoders over by default
:param float decoder_noise: amount of noise to assume when computing
decoder
:param string noise_type:
the type of noise added to the input current.
Possible options = {'uniform', 'gaussian'}.
Default is 'uniform' to match the Nengo implementation.
:param float noise:
noise parameter for noise added to input current,
sampled at every timestep.
If noise_type = uniform, this is the lower and upper
bound on the distribution.
If noise_type = gaussian, this is the variance.
"""
self.dt = dt
if seed is None:
seed = np.random.randint(1000)
self.seed = seed
self.neurons_num = neurons
self.dimensions = dimensions
self.array_size = array_size
self.radius = radius
self.noise = noise
self.noise_type = noise_type
self.decoder_noise = decoder_noise
self.mode = mode
# make sure that eval_points is the right shape
if eval_points is not None:
eval_points = np.array(eval_points)
if len(eval_points.shape) == 1:
eval_points.shape = [1, eval_points.shape[0]]
self.eval_points = eval_points
# make sure intercept is the right shape
if isinstance(intercept, (int, float)): intercept = [intercept, 1]
elif len(intercept) == 1: intercept.append(1)
self.cache_key = cache.generate_ensemble_key(
neurons=neurons,
dimensions=dimensions,
tau_rc=tau_rc,
tau_ref=tau_ref,
max_rate=max_rate,
intercept=intercept,
radius=radius,
encoders=encoders,
decoder_noise=decoder_noise,
eval_points=eval_points,
noise=noise,
seed=seed,
dt=dt)
# make dictionary for origins
self.origin = {}
# set up a dictionary for decoded_input
self.decoded_input = {}
# if we're creating a spiking ensemble
if self.mode == 'spiking':
# TODO: handle different neuron types,
self.neurons = neuron.types[neuron_type](size=(array_size,
self.neurons_num),
tau_rc=tau_rc,
tau_ref=tau_ref)
# compute alpha and bias
self.srng = RandomStreams(seed=seed)
self.max_rate = max_rate
max_rates = self.srng.uniform(size=(self.array_size,
self.neurons_num),
low=max_rate[0],
high=max_rate[1])
threshold = self.srng.uniform(size=(self.array_size,
self.neurons_num),
low=intercept[0],
high=intercept[1])
self.alpha, self.bias = theano.function(
[], self.neurons.make_alpha_bias(max_rates, threshold))()
# force to 32 bit for consistency / speed
self.bias = self.bias.astype('float32')
# compute encoders
self.encoders = self.make_encoders(encoders=encoders)
# combine encoders and gain for simplification
self.encoders = (self.encoders.T * self.alpha.T).T
self.shared_encoders = theano.shared(
self.encoders, name='ensemble.shared_encoders')
# set up a dictionary for encoded_input connections
self.encoded_input = {}
# list of learned terminations on ensemble
self.learned_terminations = []
# make default origin
self.add_origin('X',
func=None,
dt=dt,
eval_points=self.eval_points)
elif self.mode == 'direct':
# make default origin
self.add_origin('X',
func=None,
dimensions=self.dimensions * self.array_size)
# reset neurons_num to 0
self.neurons_num = 0
def add_termination(self,
name,
pstc,
decoded_input=None,
encoded_input=None,
input_socket=None,
transform=None,
case=None):
"""Accounts for a new termination that takes the given input
(a theano object) and filters it with the given pstc.
Adds its contributions to the set of decoded, encoded,
or learn input with the same pstc. Decoded inputs
are represented signals, encoded inputs are
decoded_output * weight matrix, learn input is
activities * weight_matrix.
Can only have one of decoded OR encoded OR learn input != None.
:param float pstc: post-synaptic time constant
:param decoded_input:
theano object representing the decoded output of
the pre population (just value, not a shared variable)
:param encoded_input:
theano object representing the encoded output of
the pre population multiplied by a connection weight matrix
:param learn_input:
theano object representing the learned output of
the pre population multiplied by a connection weight matrix
:param transform:
the transform that needs to be applied (dot product) to the
decoded output of the pre population
:param case:
used to generate an encoded input by applying the transform matrix
onto the decoded pre output is a special way
"""
# make sure one and only one of
# (decoded_input, encoded_input) is specified
if decoded_input is not None: assert (encoded_input is None)
elif encoded_input is not None: assert (decoded_input is None)
else: assert False
if decoded_input is not None and self.mode is 'direct':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
self.decoded_input[name] = filter.Filter(name=name,
pstc=pstc,
source=source,
shape=(self.array_size,
self.dimensions),
pre_output=pre_output)
# decoded_input in this case will be the output of pre node
elif decoded_input is not None and self.mode is 'spiking':
# decoded_input is NOT the shared variable at the origin
pre_output = theano.shared(decoded_input)
source = TT.dot(transform, pre_output)
source = TT.true_div(source, self.radius)
self.decoded_input[name] = filter.Filter(name=name,
pstc=pstc,
source=source,
shape=(self.array_size,
self.dimensions),
pre_output=pre_output)
elif encoded_input is not None:
# encoded_input is NOT the shared variable at the origin
pre_output = theano.shared(encoded_input)
source = case(transform, pre_output)
self.encoded_input[name] = filter.Filter(name=name,
pstc=pstc,
source=source,
shape=(self.array_size,
self.neurons_num),
pre_output=pre_output)
def train(self,
x_ip,
y_ip,
lr,
optimizer,
num_epochs,
batch_size,
momentum=0.9,
rmsprop_decay_rate=0.9,
adam_beta1=0.9,
adam_beta2=0.999):
self.output = self.layers[0].forward(self.X, True)
for p in self.layers[0].norm_params:
self.norm_params.append(p)
for p in self.layers[0].n_norm_params:
self.n_norm_params.append(p)
self.intermediate_vals = [self.output]
for l in self.layers[1:]:
self.intermediate_vals.append(
l.forward(self.intermediate_vals[-1], True))
for p in l.norm_params:
self.norm_params.append(p)
for p in l.n_norm_params:
self.n_norm_params.append(p)
self.cache = [
theano.shared(np.zeros_like(p.get_value()))
for p in self.parameters
]
self.momentum_params = [
theano.shared(np.zeros_like(p.get_value()))
for p in self.parameters
]
self.adamfm = [
theano.shared(np.zeros_like(p.get_value()))
for p in self.parameters
]
self.adamsm = [
theano.shared(np.zeros_like(p.get_value()))
for p in self.parameters
]
self.loss = T.mean(T.sqr(self.Y - self.layers[-1].A))
self.gradients = [T.grad(self.loss, p) for p in self.parameters]
self.updates = [(p, np)
for p, np in zip(self.norm_params, self.n_norm_params)]
self.adamc = theano.shared(1)
if optimizer == 'SGD':
self.updates += [(p, p - lr * g)
for p, g in zip(self.parameters, self.gradients)]
elif optimizer == 'SGD with Momentum':
self.updates += [
(mp, -momentum * mp + lr * g)
for mp, g in zip(self.momentum_params, self.gradients)
]
self.updates += [
(p, p - mp)
for p, mp in zip(self.parameters, self.momentum_params)
]
elif optimizer == 'AdaGrad':
self.updates += [(c, c + T.sqr(g))
for c, g in zip(self.cache, self.gradients)]
#print(self.updates)
self.updates += [
(p, p - (lr * g / T.sqrt(c + 1e-8)))
for p, g, c in zip(self.parameters, self.gradients, self.cache)
]
#print(self.updates)
elif optimizer == 'RMSProp':
self.updates += [
(c,
rmsprop_decay_rate * c + (1 - rmsprop_decay_rate) * T.sqr(g))
for c, g in zip(self.cache, self.gradients)
]
self.updates += [
(p, p - lr * T.true_div(g, T.sqrt(c + 1e-8)))
for p, g, c in zip(self.parameters, self.gradients, self.cache)
]
#self.updates += [(c, rmsprop_decay_rate*c + (1-rmsprop_decay_rate)*T.sqr(g)) for c,g in zip(self.cache, self.gradients)]
elif optimizer == 'Adam':
self.updates += [
(p, p - lr * T.true_div(fm, T.sqrt(sm + 1e-8)))
for p, fm, sm in zip(self.parameters, self.adamfm, self.adamsm)
]
self.updates += [(fm, adam_beta1 * fm + (1 - adam_beta1) * g)
for fm, g in zip(self.adamfm, self.gradients)]
self.updates += [(sm,
adam_beta2 * sm + (1 - adam_beta2) * T.sqr(g))
for sm, g in zip(self.adamsm, self.gradients)]
self.updates += [(self.adamc, self.adamc + 1)]
else:
raise ValueError("Optimizer not correct")
self.train_inputs = [self.X, self.Y]
self.train_outputs = [self.loss]
#print(self.updates)
self.train_fn = theano.function(inputs=self.train_inputs,
outputs=self.train_outputs,
updates=self.updates)
error_plot = []
num_iter = x_ip.shape[0] // batch_size
for epoch in range(num_epochs):
for n_iter in range(num_iter):
x_batch = x_ip[n_iter * batch_size:n_iter * batch_size +
batch_size, :]
y_batch = y_ip[n_iter * batch_size:n_iter * batch_size +
batch_size, :]
self.train_fn(x_batch, y_batch)
if epoch % 2 == 0:
error_plot.append(self.train_fn(x_ip, y_ip))
plt.plot(error_plot)
