def ber(self, y): tp = T.and_(T.eq(y, 1), T.eq(self.y_pred, 1)).sum() tn = T.and_(T.eq(y, 0), T.eq(self.y_pred, 0)).sum() fp = T.and_(T.eq(y, 0), T.eq(self.y_pred, 1)).sum() fn = T.and_(T.eq(y, 1), T.eq(self.y_pred, 0)).sum() ber = 0.5 * (T.true_div(fp, tp + fp) + T.true_div(fn, tn + fn)) return ber
def call(self, inputs, mask=None): if not isinstance(inputs, list) or len(inputs) <= 1: raise TypeError('SpkLifeLongMemory must be called on a list of tensors ' '(at least 2). Got: ' + str(inputs)) # (None(batch), 1), index of speaker target_spk_l = inputs[0] target_spk_l = K.reshape(target_spk_l, (target_spk_l.shape[0], )) if K.dtype(target_spk_l) != 'int32': target_spk_l = K.cast(target_spk_l, 'int32') # (None(batch), embed_dim) spk_vector_l = inputs[1] # Start to update life-long memory based on the learned speech vector # First do normalization spk_vector_eps = K.switch(K.equal(spk_vector_l, 0.), np.spacing(1), spk_vector_l) # avoid zero spk_vector_eps = K.sqrt(K.sum(spk_vector_eps**2, axis=1)) spk_vector_eps = spk_vector_eps.dimshuffle((0, 'x')) spk_vector = T.true_div(spk_vector_l, K.repeat_elements(spk_vector_eps, self.vec_dim, axis=1)) # Store speech vector into life-long memory according to the speaker identity. life_long_mem = T.inc_subtensor(self.life_long_mem[target_spk_l, :], spk_vector) # Normalization for memory life_long_mem_eps = K.switch(K.equal(life_long_mem, 0.), np.spacing(1), life_long_mem) # avoid 0 life_long_mem_eps = K.sqrt(K.sum(life_long_mem_eps**2, axis=1)) life_long_mem_eps = life_long_mem_eps.dimshuffle((0, 'x')) life_long_mem = T.true_div(life_long_mem, K.repeat_elements(life_long_mem_eps, self.vec_dim, axis=1)) # (None(batch), spk_size, embed_dim) return life_long_mem
def theano_periodic_sinc(in_sig, bandwidth): eps = TT.constant(1e-10) denominator = TT.mul(TT.sin(TT.true_div(in_sig, bandwidth)), bandwidth) idx_modi = TT.lt(TT.abs_(denominator), eps) numerator = TT.switch(idx_modi, TT.cos(in_sig), TT.sin(in_sig)) denominator = TT.switch(idx_modi, TT.cos(TT.true_div(in_sig, bandwidth)), denominator) return TT.true_div(numerator, denominator)
def f1_score(self, y): n_total = y.shape[0] n_relevant_documents_predicted = T.sum(T.eq(T.ones(self.y_pred.shape), self.y_pred)) two_vector = T.add(T.ones(self.y_pred.shape), T.ones(self.y_pred.shape)) n_relevant_predicted_correctly = T.sum(T.eq(T.add(self.y_pred, y), two_vector)) precision = T.true_div(n_relevant_predicted_correctly, n_relevant_documents_predicted) recall = T.true_div(n_relevant_predicted_correctly, n_total) f1_score = T.mul(2.0, T.true_div(T.mul(precision, recall), T.add(precision, recall))) return [f1_score, precision, recall]
def BGMM1_lpdf(node, sample, kw): r, weights, mus, sigmas, low, high, draw_shape = node.inputs assert weights.ndim == 1 assert mus.ndim == 1 assert sigmas.ndim == 1 _sample = sample if sample.ndim != 1: sample = sample.flatten() erf = theano.tensor.erf effective_weights = 0.5 * weights * (erf((high - mus) / sigmas) - erf( (low - mus) / sigmas)) dist = (sample.dimshuffle(0, 'x') - mus) mahal = ((dist**2) / (sigmas**2)) # POSTCONDITION: mahal.shape == (n_samples, n_components) Z = tensor.sqrt(2 * numpy.pi * sigmas**2) rval = tensor.log( tensor.sum(tensor.true_div( tensor.exp(-.5 * mahal) * weights, Z * effective_weights.sum()), axis=1)) if not sample is _sample: rval = rval.reshape(_sample.shape) assert rval.ndim != 1 return rval
def BGMM1_lpdf(node, sample, kw): r, weights, mus, sigmas, low, high, draw_shape = node.inputs assert weights.ndim == 1 assert mus.ndim == 1 assert sigmas.ndim == 1 _sample = sample if sample.ndim != 1: sample = sample.flatten() erf = theano.tensor.erf effective_weights = 0.5 * weights * (erf((high - mus) / sigmas) - erf((low - mus) / sigmas)) dist = sample.dimshuffle(0, "x") - mus mahal = (dist ** 2) / (sigmas ** 2) # POSTCONDITION: mahal.shape == (n_samples, n_components) Z = tensor.sqrt(2 * numpy.pi * sigmas ** 2) rval = tensor.log( tensor.sum(tensor.true_div(tensor.exp(-0.5 * mahal) * weights, Z * effective_weights.sum()), axis=1) ) if not sample is _sample: rval = rval.reshape(_sample.shape) assert rval.ndim != 1 return rval
def minus_corr(u, v): um = T.sub(u, T.mean(u)) vm = T.sub(v, T.mean(v)) r_num = T.sum(T.mul(um, vm)) r_den = T.sqrt(T.mul(T.sum(T.sqr(um)), T.sum(T.sqr(vm)))) r = T.true_div(r_num, r_den) r = T.neg(r) return r
def division(self): """ Note: This maps to the `true_div` operation. """ assert len(self.children) == 2 n = self.get_input(0) d = self.get_input(1) return T.true_div(n, d)
def SlopeLin2(x, slope): """ Linear unit with different slopes :param slope: slope of negative quadrant :return: x if x > 0 else x/slope """ import theano.tensor as T return T.switch(T.gt(x, 0), x, T.true_div(x, slope))
def add_termination(self, name, pstc, decoded_input=None, encoded_input=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix """ # make sure one and only one of # (decoded_input, encoded_input) is specified if decoded_input is not None: assert (encoded_input is None) elif encoded_input is not None: assert (decoded_input is None) else: assert False if decoded_input: if self.mode is not 'direct': # rescale decoded_input by this neuron's radius source = TT.true_div(decoded_input, self.radius) # ignore radius in direct mode else: source = decoded_input name = self.get_unique_name(name, self.decoded_input) self.decoded_input[name] = filter.Filter(name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions)) elif encoded_input: name = self.get_unique_name(name, self.encoded_input) self.encoded_input[name] = filter.Filter(name=name, pstc=pstc, source=encoded_input, shape=(self.array_size, self.neurons_num))
def quadratic_weighted_kappa_loss(y_true, y_pred): min_rating = T.minimum(T.min(y_true), T.min(y_pred)) max_rating = T.maximum(T.max(y_true), T.max(y_pred)) hist_true = T.bincount(y_true, minlength=max_rating) hist_pred = T.bincount(y_pred, minlength=max_rating) num_ratings = (max_rating - min_rating) + 1 num_scored = float(len(y_true)) numerator = T.zeros(1) denominator = T.zeros(1) z = T.zeros(len(y_true)) for i_true in range(min_rating, max_rating + 1): for j_pred in range(min_rating, max_rating + 1): expected = T.true_div(T.mul(hist_true[i_true], hist_pred[j_pred]), num_scored) d = T.true_div(T.sqr(i_true - j_pred), T.sqr(num_ratings - 1.)) conf_mat_cell = T.sum(T.and_(T.eq(T.sub(y_true, i_true), z), T.eq(T.sub(y_pred, j_pred), z))) numerator = T.add(numerator, T.true_div(T.mul(d, conf_mat_cell), num_scored)) denominator = T.add(denominator, T.true_div(T.mul(d, expected), num_scored)) return T.true_div(numerator, denominator)
def add_filtered_input(self, pstc, decoded_input=None, encoded_input=None, learn_input=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix """ # make sure one and only one of # (decoded_input, encoded_input, learn_input) is specified if decoded_input is not None: assert (encoded_input is None) and (learn_input is None) elif encoded_input is not None: assert (decoded_input is None) and (learn_input is None) elif learn_input is not None: assert (decoded_input is None) and (encoded_input is None) else: assert False # make sure there's an accumulator for given pstc if pstc not in self.accumulators: self.accumulators[pstc] = Accumulator(self, pstc) # add this termination's contribution to # the set of terminations with the same pstc if decoded_input: # rescale decoded_input by this neuron's radius # to put us in the right range self.accumulators[pstc].add_decoded_input( TT.true_div(decoded_input, self.radius)) elif encoded_input: self.accumulators[pstc].add_encoded_input(encoded_input) elif learn_input: self.accumulators[pstc].add_learn_input(learn_input)
def get_recall(self, y_true): """ ~~~~~~~~~~ Untested ~~~~~~~~~~ Returns recall/sensitivity of binary predictions. :type y_true: Theano shared variable :param y_true: Target :rtype Theano shared variable :return recall/sensitivity """ true_positives = T.sum(T.mul(T.eq(self.y_pred, 1.), T.eq(y_true, 1.))) condition_positives = T.sum(T.eq(y_true, 1.)) return T.true_div(true_positives, condition_positives)
def get_precision(self, y_true): """ ~~~~~~~~~~ Untested ~~~~~~~~~~ Returns precision of binary predictions. :type y_true: Theano shared variable :param y_true: Target :rtype Theano shared variable :return precision """ true_positives = T.sum(T.mul(T.eq(self.y_pred, 1.), T.eq(y_true, 1.))) test_outcome_positives = T.sum(T.eq(self.y_pred, 1.)) return T.true_div(true_positives, test_outcome_positives)
def get_specificity(self, y_true): """ ~~~~~~~~~~ Untested ~~~~~~~~~~ Returns specificity of binary predictions. :type y_true: Theano shared variable :param y_true: Target :rtype Theano shared variable :return specificity """ true_negatives = T.sum(T.mul(T.eq(self.y_pred, 0.), T.eq(y_true, 0.))) condition_negatives = T.sum(T.eq(y_true, 0.)) return T.true_div(true_negatives, condition_negatives)
def call(self, inputs, mask=None): if not isinstance(inputs, list) or len(inputs) <= 1: raise TypeError( 'SpkLifeLongMemory must be called on a list of tensors ' '(at least 2). Got: ' + str(inputs)) # (None(batch), 1),说话人对应的idx target_spk_l = inputs[0] target_spk_l = K.reshape(target_spk_l, (target_spk_l.shape[0], )) if K.dtype(target_spk_l) != 'int32': target_spk_l = K.cast(target_spk_l, 'int32') # (None(batch), embed_dim) spk_vector_l = inputs[1] # 开始更新长时记忆单元,训练好的语音向量 # 先对语音向量进行 normalization spk_vector_eps = K.switch(K.equal(spk_vector_l, 0.), np.spacing(1), spk_vector_l) # 防止分母为0 spk_vector_eps = K.sqrt(K.sum(spk_vector_eps**2, axis=1)) spk_vector_eps = spk_vector_eps.dimshuffle((0, 'x')) spk_vector = T.true_div( spk_vector_l, K.repeat_elements(spk_vector_eps, self.vec_dim, axis=1)) # 根据说话人idx,将语音向量增量式写入长时记忆单元中 life_long_mem = T.inc_subtensor(self.life_long_mem[target_spk_l, :], spk_vector) # 对长时Memory进行整体 normalization life_long_mem_eps = K.switch(K.equal(life_long_mem, 0.), np.spacing(1), life_long_mem) # 防止分母为0 life_long_mem_eps = K.sqrt(K.sum(life_long_mem_eps**2, axis=1)) life_long_mem_eps = life_long_mem_eps.dimshuffle((0, 'x')) life_long_mem = T.true_div( life_long_mem, K.repeat_elements(life_long_mem_eps, self.vec_dim, axis=1)) # # 更新记忆参数,由于life_long_mem必须是numpy才行,因而不能在这里做,放到外面做。 # K.set_value(self.life_long_mem, life_long_mem) # # 从长时记忆单元中提取说话人的声纹记忆特征 # spk_life_long_memory = K.gather(self.life_long_mem, target_spk_l) # (None(batch), spk_size, embed_dim) return life_long_mem
def add_termination(self, name, pstc, decoded_input=None, encoded_input=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix """ # make sure one and only one of # (decoded_input, encoded_input) is specified if decoded_input is not None: assert encoded_input is None elif encoded_input is not None: assert decoded_input is None else: assert False if decoded_input: if self.mode is not "direct": # rescale decoded_input by this neuron's radius source = TT.true_div(decoded_input, self.radius) # ignore radius in direct mode else: source = decoded_input name = helpers.get_unique_name(name, self.decoded_input) self.decoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions) ) elif encoded_input: name = helpers.get_unique_name(name, self.encoded_input) self.encoded_input[name] = filter.Filter( name=name, pstc=pstc, source=encoded_input, shape=(self.array_size, self.neurons_num) )
def fit(self, weights, o_error, tpo): gradients = T.grad(o_error, weights) updates = [] for c, v, w, g in zip(self.t_cache, self.t_velocity, weights, gradients): new_velocity = T.sub(T.mul(tpo["momentum_rate"], v), T.mul(tpo["learn_rate"], g)) new_cache = T.add(T.mul(tpo["decay_rate"], c), T.mul(T.sub(1, tpo["decay_rate"]), T.sqr(g))) new_weights = T.sub( T.add(w, new_velocity), T.true_div(T.mul(g, tpo["learn_rate"]), T.sqrt(T.add(new_cache, 0.1**8)))) updates.append((w, new_weights)) updates.append((v, new_velocity)) updates.append((c, new_cache)) return updates
def add_filtered_input(self, pstc, decoded_input=None, encoded_input=None): """Create a new termination that takes the given input (a theano object) and filters it with the given pstc Adds its contributions to the set of decoded or encoded input with the same pstc Decoded inputs are represented signals, encoded inputs are neuron activities * weight matrix Can only have decoded OR encoded input != None :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix """ # make sure one and only one of (decoded_input, encoded_input) is specified assert (decoded_input is None or encoded_input is None) assert (decoded_input is not None or encoded_input is not None) if pstc not in self.accumulators: # make sure there's an accumulator for given pstc self.accumulators[pstc] = Accumulator(self, pstc) # add this termination's contribution to the set of terminations with the same pstc if decoded_input is not None: # rescale decoded_input by this neurons radius to put us in the right range self.accumulators[pstc].add_decoded_input(TT.true_div(decoded_input, self.radius)) else: self.accumulators[pstc].add_encoded_input(encoded_input)
def Tnanmean(x,axis=None,keepdims=False): """ Theano version of numpy nanmean. Parameters ---------- x : Theano.tensor or array_like The array to be nanmeaned axis : int or None, optional Axis to do nanmean along. If None or default, nanmean is done over entire array keepdims : Boolean, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. Returns ------- An array-like Nan is returned for slices that contain only NaNs. """ mask1 = T.switch(T.isnan(x),0,x) mask2 = T.switch(T.isnan(x),0,1) return T.true_div(mask1.sum(axis=axis,keepdims=keepdims),mask2.sum(axis=axis,keepdims=keepdims))
def add_filtered_input(self, pstc, decoded_input=None, encoded_input=None): """Create a new termination that takes the given input (a theano object) and filters it with the given pstc Adds its contributions to the set of decoded or encoded input with the same pstc Decoded inputs are represented signals, encoded inputs are neuron activities * weight matrix Can only have decoded OR encoded input != None :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population multiplied by this termination's transform matrix :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix """ # make sure one and only one of (decoded_input, encoded_input) is specified assert (decoded_input is None or encoded_input is None) assert (decoded_input is not None or encoded_input is not None) if pstc not in self.accumulators: # make sure there's an accumulator for given pstc self.accumulators[pstc] = Accumulator(self, pstc) # add this termination's contribution to the set of terminations with the same pstc if decoded_input is not None: # rescale decoded_input by this neurons radius to put us in the right range self.accumulators[pstc].add_decoded_input( TT.true_div(decoded_input, self.radius)) else: self.accumulators[pstc].add_encoded_input(encoded_input)
def Tnanmean(x, axis=None, keepdims=False): """ Theano version of numpy nanmean. Parameters ---------- x : Theano.tensor or array_like The array to be nanmeaned axis : int or None, optional Axis to do nanmean along. If None or default, nanmean is done over entire array keepdims : Boolean, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. Returns ------- An array-like Nan is returned for slices that contain only NaNs. """ mask1 = T.switch(T.isnan(x), 0, x) mask2 = T.switch(T.isnan(x), 0, 1) return T.true_div(mask1.sum(axis=axis, keepdims=keepdims), mask2.sum(axis=axis, keepdims=keepdims))
def div(x, y): z = T.true_div(x, y) if isinstance(get_shape(x), (tuple, list)): output_shape = auto_infer_shape(T.true_div, x, y) add_shape(z, output_shape) return z
def max_f1_score_loss(coding_dist, true_dist): """ f1 = (2 * N_WWW) / (N_D + N_W) """ def set_2dim(j, matrix): return T.switch(T.eq(matrix[j], 0), -100, T.log(matrix[j])) def set_1dim(i, matrix): matrix_2dim, updates=theano.scan(set_2dim,\ outputs_info=None,\ sequences=T.arange(matrix.shape[1]),\ non_sequences=[matrix[i]]) return matrix_2dim if true_dist.ndim == coding_dist.ndim: ''' #parameter tau=-2 #ce_loss: [batch_size,] ce_loss = T.log(T.sum(true_dist * (coding_dist + 1e-14), axis=1)) #N_D : >tau=1; <tau=0 N_d = ce_loss N_D, updates = theano.scan(lambda i, x, y: T.switch(T.lt(x[i], y), 0, 1),\ outputs_info=None,\ sequences=T.arange(N_d.shape[0]),\ non_sequences=[N_d,tau]) N_D = T.sum(N_D, axis=0) # N_D = T.nnet.sigmoid(N_D) #N_W: [1, class_numbers] N_W = T.sum(true_dist ,axis=0) #N_WWW: [1, class_numbers] matrix = true_dist * (coding_dist + 1e-14) matrix_log, updates = theano.scan(set_1dim,\ outputs_info=None,\ sequences=T.arange(matrix.shape[0]),\ non_sequences=[matrix]) #indictor function I = T.switch(T.lt(matrix_log, tau), 0, 1) N_WWW = T.sum(I, axis=0) # N_WWW = T.nnet.sigmoid(N_WWW) ''' classid = T.argmax(coding_dist, axis=1) #N_D def N_D_count_2dim(j, x, y): return T.switch(T.eq(j, y), 1, 0) def N_D_count_1dim(i, x, y): N_d, updates = theano.scan(N_D_count_2dim,\ outputs_info=None,\ sequences=T.arange(x.shape[1]),\ non_sequences=[x[i], y[i]]) return N_d predict, updates = theano.scan(N_D_count_1dim,\ outputs_info=None,\ sequences=T.arange(coding_dist.shape[0]),\ non_sequences=[coding_dist, classid]) N_D = T.reshape(T.sum(predict, axis=0), (1, -1)) N_D = T.nnet.sigmoid(N_D) #N_W: [1, class_numbers] N_W = T.reshape(T.sum(true_dist, axis=0), (1, -1)) #N_WWW: [1, class_numbers] matrix = true_dist * predict N_WWW = T.reshape(T.sum(matrix, axis=0), (1, -1)) N_WWW = T.nnet.sigmoid(N_WWW) #F1-SCORE F1 = T.true_div(2 * N_WWW, (N_D + N_W + 1e-14)) return F1, F1 else: print "true_dist.ndim != coding_dist.ndim"
def beam_search_backward(self, top, idx): max_idx = top[idx] idx = T.true_div(max_idx, self.target_vocab_size) max_idx = T.divmod(max_idx, self.target_vocab_size) return idx, max_idx
theta = np.array([2, 2.5]) tau = np.array([.01, .01]) activation = T.vector('activation') t = T.scalar('t') input_duration = T.scalar('input_duration') input_intensity = T.scalar('input_intensity') P = input_intensity * ((T.sgn(input_duration - t) + 1) / 2) corrected_sigmoid = \ 1 / (1 + T.exp(-T.mul(a,(T.dot(c,activation) + P) - theta))) \ - 1 / (1 + T.exp(np.multiply(a, theta))) #corrected_sigmoid = theano.function([t, activation, input_duration, input_intensity], corrected_s) #d_a = T.true_div(-activation + T.mul( 1 - T.mul(r, activation), corrected_s), tau) d_a = T.true_div(-activation + T.mul( k - T.mul(r, activation), corrected_sigmoid), tau) d_activation = theano.function(inputs=[activation, t, input_duration, input_intensity], outputs=d_a, on_unused_input='warn') J = theano.function(inputs=[activation, t, input_duration, input_intensity], outputs=T.jacobian(d_a, activation), on_unused_input='warn') activation_0 = np.array([0, 0]) t_0 = 0 t_1 = .125 dt = .0001 times = np.arange(t_0, t_1, dt) intens = 1 duration = .125 params = (duration, intens)
def add_termination(self, name, pstc, decoded_input=None, encoded_input=None, input_socket=None, transform=None, case=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population (just value, not a shared variable) :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix :param transform: the transform that needs to be applied (dot product) to the decoded output of the pre population :param case: used to generate an encoded input by applying the transform matrix onto the decoded pre output is a special way """ # make sure one and only one of # (decoded_input, encoded_input) is specified if decoded_input is not None: assert (encoded_input is None) elif encoded_input is not None: assert (decoded_input is None) else: assert False if decoded_input is not None and self.mode == 'direct': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) self.decoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) # decoded_input in this case will be the output of pre node elif decoded_input is not None and self.mode == 'spiking': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) source = TT.true_div(source, self.radius) self.decoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) elif encoded_input is not None: # encoded_input is NOT the shared variable at the origin pre_output = theano.shared(encoded_input) source = case(transform, pre_output) self.encoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.neurons_num), pre_output=pre_output)
class Ensemble: """An ensemble is a collection of neurons representing a vector space. """ def __init__(self, neurons, dimensions, dt, tau_ref=0.002, tau_rc=0.02, max_rate=(200, 300), intercept=(-1.0, 1.0), radius=1.0, encoders=None, seed=None, neuron_type='lif', array_size=1, eval_points=None, decoder_noise=0.1, noise_type='uniform', noise=None, mode='spiking', is_subensemble=False, decoders=None, bias=None, alpha=None): self.dt = dt np.random.seed(97) if seed is None: seed = np.random.randint(1000) self.seed = seed self.neurons_num = neurons self.dimensions = dimensions self.array_size = array_size self.radius = radius self.noise = noise self.noise_type = noise_type self.decoder_noise = decoder_noise self.is_subensemble = is_subensemble self.mode = mode # make sure that eval_points is the right shape if eval_points is not None: eval_points = np.array(eval_points) if len(eval_points.shape) == 1: eval_points.shape = [1, eval_points.shape[0]] self.eval_points = eval_points # make sure intercept is the right shape if isinstance(intercept, (int,float)): intercept = [intercept, 1] elif len(intercept) == 1: intercept.append(1) self.cache_key = cache.generate_ensemble_key(neurons=neurons, dimensions=dimensions, tau_rc=tau_rc, tau_ref=tau_ref, max_rate=max_rate, intercept=intercept, radius=radius, encoders=(encoders if not is_subensemble else None), decoder_noise=decoder_noise, eval_points=eval_points, noise=noise, seed=seed, dt=dt) # make dictionary for origins self.origin = {} # set up a dictionary for decoded_input self.decoded_input = {} # if we're creating a spiking ensemble if self.mode == 'spiking': # TODO: handle different neuron types, self.neurons = neuron.types[neuron_type]( size=(array_size, self.neurons_num), tau_rc=tau_rc, tau_ref=tau_ref) # compute alpha and bias self.srng = RandomStreams(seed=seed) self.max_rate = max_rate if is_subensemble: self.bias = bias self.alpha = alpha self.encoders = encoders else: max_rates = self.srng.uniform( size=(self.array_size, self.neurons_num), low=max_rate[0], high=max_rate[1]) threshold = self.srng.uniform( size=(self.array_size, self.neurons_num), low=intercept[0], high=intercept[1]) self.alpha, self.bias = theano.function( [], self.neurons.make_alpha_bias(max_rates, threshold))() # force to 32 bit for consistency / speed self.bias = self.bias.astype(FLOAT_TYPE) # compute encoders self.encoders = self.make_encoders(encoders=encoders) # combine encoders and gain for simplification self.encoders = (self.encoders.T * self.alpha.T).T self.shared_encoders = theano.shared(self.encoders, name='ensemble.shared_encoders') # set up a dictionary for encoded_input connections self.encoded_input = {} # list of learned terminations on ensemble self.learned_terminations = [] # make default origin self.add_origin('X', func=None, dt=dt, eval_points=self.eval_points, decoders=decoders) elif self.mode == 'direct': # make default origin self.add_origin('X', func=None, dimensions=self.dimensions*self.array_size) # reset neurons_num to 0 self.neurons_num = 0 def add_termination(self, name, pstc, decoded_input=None, encoded_input=None, input_socket=None, transform=None, case=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population (just value, not a shared variable) :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix :param transform: the transform that needs to be applied (dot product) to the decoded output of the pre population :param case: used to generate an encoded input by applying the transform matrix onto the decoded pre output is a special way """ # make sure one and only one of # (decoded_input, encoded_input) is specified if decoded_input is not None: assert (encoded_input is None) elif encoded_input is not None: assert (decoded_input is None) else: assert False if decoded_input is not None and self.mode == 'direct': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) self.decoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) # decoded_input in this case will be the output of pre node elif decoded_input is not None and self.mode == 'spiking': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) source = TT.true_div(source, self.radius) self.decoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) elif encoded_input is not None: # encoded_input is NOT the shared variable at the origin pre_output = theano.shared(encoded_input) source = case(transform, pre_output) self.encoded_input[name] = filter.Filter( name=name, pstc=pstc, source=source, shape=(self.array_size, self.neurons_num), pre_output=pre_output)
def R2loss(y_true, y_pred): y_pred = y_pred.flatten() y_true = y_true.flatten() tot = T.sum(T.sqr(T.sub(y_true, T.mean(y_true)))) res = T.sum(T.sqr(T.sub(y_true, y_pred))) return T.true_div(res, tot)
def inner(x): return T.switch(T.gt(x, 0), x, T.true_div(x, slope))
def l2_normalize(x, dim, epsilon=1e-12): square_sum = T.sum(T.sqr(x), axis=dim) x_inv_norm = T.true_div(1, T.sqrt(T.maximum(square_sum, epsilon))) x_inv_norm = x_inv_norm.dimshuffle(0, 'x') return T.mul(x, x_inv_norm)
def fit(self, X_train, y_train, error_function, optimizer, regularizer=None, sample_weight=None, batch_size=128, nb_epoch=100, shuffle=True, verbosity=0, memory=True): """Learns the weights of a factorization machine with mini-batch gradient descent. The weights that minimize the loss function (across epochs) are retained.""" # ************************************************************ # * Learning (Symbolic) # ************************************************************ # *** Loss Function *** error = error_function.apply(self.y, self.y_hat) mean_error = T.true_div(T.sum(T.mul(error, self.s)), T.sum(self.s)) loss = mean_error # regularization if regularizer is not None: loss = regularizer.regularize(loss, self.w0[0], self.w1, self.v) params = [self.w0, self.w1, self.v] updates = optimizer.update(loss, params, self.e) theano_train = theano.function(inputs=[self.X, self.y, self.s, self.e], on_unused_input='ignore', outputs=loss, updates=updates, allow_input_downcast=True) theano_cost = theano.function(inputs=[self.X, self.y, self.s], on_unused_input='ignore', outputs=loss, allow_input_downcast=True) # ************************************************************ # * Learning (Numeric) # ************************************************************ n = X_train.shape[0] if batch_size > n: batch_size = n if sample_weight is None: sample_weight = np.ones(n) else: sample_weight = (float(n) / np.sum(sample_weight)) * sample_weight min_loss = float('inf') min_loss_weights = self.get_weights() for epoch in utils.xrange(1, nb_epoch + 1): if shuffle: indices = np.arange(n) np.random.shuffle(indices) X_train, y_train = X_train[indices], y_train[indices] for start in itertools.count(0, batch_size): if start >= n: break stop = min(start + batch_size, n) theano_train(X_train[start:stop], y_train[start:stop], sample_weight[start:stop], epoch) current_loss = theano_cost(X_train, y_train, sample_weight) if not np.isfinite(current_loss): raise ArithmeticError("Non-finite loss function.") if current_loss < min_loss: min_loss = current_loss min_loss_weights = self.get_weights() if verbosity > 0 and (epoch % verbosity == 0): print('Epoch {}/{}'.format(epoch, nb_epoch)) print(' loss: {}, min_loss: {}'.format(current_loss, min_loss)) weights = min_loss_weights if memory else self.get_weights() self.set_weights(weights)
class Ensemble: """An ensemble is a collection of neurons representing a vector space. """ def __init__(self, neurons, dimensions, dt, tau_ref=0.002, tau_rc=0.02, max_rate=(200, 300), intercept=(-1.0, 1.0), radius=1.0, encoders=None, seed=None, neuron_type='lif', array_size=1, eval_points=None, decoder_noise=0.1, noise_type='uniform', noise=None, mode='spiking'): """Construct an ensemble composed of the specific neuron model, with the specified neural parameters. :param int neurons: number of neurons in this population :param int dimensions: number of dimensions in the vector space that these neurons represent :param float tau_ref: length of refractory period :param float tau_rc: RC constant; approximately how long until 2/3 of the threshold voltage is accumulated :param tuple max_rate: lower and upper bounds on randomly generated firing rates for each neuron :param tuple intercept: lower and upper bounds on randomly generated x offsets for each neuron :param float radius: the range of input values (-radius:radius) per dimension this population is sensitive to :param list encoders: set of possible preferred directions :param int seed: seed value for random number generator :param string neuron_type: type of neuron model to use, options = {'lif'} :param int array_size: number of sub-populations for network arrays :param list eval_points: specific set of points to optimize decoders over by default :param float decoder_noise: amount of noise to assume when computing decoder :param string noise_type: the type of noise added to the input current. Possible options = {'uniform', 'gaussian'}. Default is 'uniform' to match the Nengo implementation. :param float noise: noise parameter for noise added to input current, sampled at every timestep. If noise_type = uniform, this is the lower and upper bound on the distribution. If noise_type = gaussian, this is the variance. """ self.dt = dt if seed is None: seed = np.random.randint(1000) self.seed = seed self.neurons_num = neurons self.dimensions = dimensions self.array_size = array_size self.radius = radius self.noise = noise self.noise_type = noise_type self.decoder_noise = decoder_noise self.mode = mode # make sure that eval_points is the right shape if eval_points is not None: eval_points = np.array(eval_points) if len(eval_points.shape) == 1: eval_points.shape = [1, eval_points.shape[0]] self.eval_points = eval_points # make sure intercept is the right shape if isinstance(intercept, (int, float)): intercept = [intercept, 1] elif len(intercept) == 1: intercept.append(1) self.cache_key = cache.generate_ensemble_key( neurons=neurons, dimensions=dimensions, tau_rc=tau_rc, tau_ref=tau_ref, max_rate=max_rate, intercept=intercept, radius=radius, encoders=encoders, decoder_noise=decoder_noise, eval_points=eval_points, noise=noise, seed=seed, dt=dt) # make dictionary for origins self.origin = {} # set up a dictionary for decoded_input self.decoded_input = {} # if we're creating a spiking ensemble if self.mode == 'spiking': # TODO: handle different neuron types, self.neurons = neuron.types[neuron_type](size=(array_size, self.neurons_num), tau_rc=tau_rc, tau_ref=tau_ref) # compute alpha and bias self.srng = RandomStreams(seed=seed) self.max_rate = max_rate max_rates = self.srng.uniform(size=(self.array_size, self.neurons_num), low=max_rate[0], high=max_rate[1]) threshold = self.srng.uniform(size=(self.array_size, self.neurons_num), low=intercept[0], high=intercept[1]) self.alpha, self.bias = theano.function( [], self.neurons.make_alpha_bias(max_rates, threshold))() # force to 32 bit for consistency / speed self.bias = self.bias.astype('float32') # compute encoders self.encoders = self.make_encoders(encoders=encoders) # combine encoders and gain for simplification self.encoders = (self.encoders.T * self.alpha.T).T self.shared_encoders = theano.shared( self.encoders, name='ensemble.shared_encoders') # set up a dictionary for encoded_input connections self.encoded_input = {} # list of learned terminations on ensemble self.learned_terminations = [] # make default origin self.add_origin('X', func=None, dt=dt, eval_points=self.eval_points) elif self.mode == 'direct': # make default origin self.add_origin('X', func=None, dimensions=self.dimensions * self.array_size) # reset neurons_num to 0 self.neurons_num = 0 def add_termination(self, name, pstc, decoded_input=None, encoded_input=None, input_socket=None, transform=None, case=None): """Accounts for a new termination that takes the given input (a theano object) and filters it with the given pstc. Adds its contributions to the set of decoded, encoded, or learn input with the same pstc. Decoded inputs are represented signals, encoded inputs are decoded_output * weight matrix, learn input is activities * weight_matrix. Can only have one of decoded OR encoded OR learn input != None. :param float pstc: post-synaptic time constant :param decoded_input: theano object representing the decoded output of the pre population (just value, not a shared variable) :param encoded_input: theano object representing the encoded output of the pre population multiplied by a connection weight matrix :param learn_input: theano object representing the learned output of the pre population multiplied by a connection weight matrix :param transform: the transform that needs to be applied (dot product) to the decoded output of the pre population :param case: used to generate an encoded input by applying the transform matrix onto the decoded pre output is a special way """ # make sure one and only one of # (decoded_input, encoded_input) is specified if decoded_input is not None: assert (encoded_input is None) elif encoded_input is not None: assert (decoded_input is None) else: assert False if decoded_input is not None and self.mode is 'direct': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) self.decoded_input[name] = filter.Filter(name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) # decoded_input in this case will be the output of pre node elif decoded_input is not None and self.mode is 'spiking': # decoded_input is NOT the shared variable at the origin pre_output = theano.shared(decoded_input) source = TT.dot(transform, pre_output) source = TT.true_div(source, self.radius) self.decoded_input[name] = filter.Filter(name=name, pstc=pstc, source=source, shape=(self.array_size, self.dimensions), pre_output=pre_output) elif encoded_input is not None: # encoded_input is NOT the shared variable at the origin pre_output = theano.shared(encoded_input) source = case(transform, pre_output) self.encoded_input[name] = filter.Filter(name=name, pstc=pstc, source=source, shape=(self.array_size, self.neurons_num), pre_output=pre_output)
def train(self, x_ip, y_ip, lr, optimizer, num_epochs, batch_size, momentum=0.9, rmsprop_decay_rate=0.9, adam_beta1=0.9, adam_beta2=0.999): self.output = self.layers[0].forward(self.X, True) for p in self.layers[0].norm_params: self.norm_params.append(p) for p in self.layers[0].n_norm_params: self.n_norm_params.append(p) self.intermediate_vals = [self.output] for l in self.layers[1:]: self.intermediate_vals.append( l.forward(self.intermediate_vals[-1], True)) for p in l.norm_params: self.norm_params.append(p) for p in l.n_norm_params: self.n_norm_params.append(p) self.cache = [ theano.shared(np.zeros_like(p.get_value())) for p in self.parameters ] self.momentum_params = [ theano.shared(np.zeros_like(p.get_value())) for p in self.parameters ] self.adamfm = [ theano.shared(np.zeros_like(p.get_value())) for p in self.parameters ] self.adamsm = [ theano.shared(np.zeros_like(p.get_value())) for p in self.parameters ] self.loss = T.mean(T.sqr(self.Y - self.layers[-1].A)) self.gradients = [T.grad(self.loss, p) for p in self.parameters] self.updates = [(p, np) for p, np in zip(self.norm_params, self.n_norm_params)] self.adamc = theano.shared(1) if optimizer == 'SGD': self.updates += [(p, p - lr * g) for p, g in zip(self.parameters, self.gradients)] elif optimizer == 'SGD with Momentum': self.updates += [ (mp, -momentum * mp + lr * g) for mp, g in zip(self.momentum_params, self.gradients) ] self.updates += [ (p, p - mp) for p, mp in zip(self.parameters, self.momentum_params) ] elif optimizer == 'AdaGrad': self.updates += [(c, c + T.sqr(g)) for c, g in zip(self.cache, self.gradients)] #print(self.updates) self.updates += [ (p, p - (lr * g / T.sqrt(c + 1e-8))) for p, g, c in zip(self.parameters, self.gradients, self.cache) ] #print(self.updates) elif optimizer == 'RMSProp': self.updates += [ (c, rmsprop_decay_rate * c + (1 - rmsprop_decay_rate) * T.sqr(g)) for c, g in zip(self.cache, self.gradients) ] self.updates += [ (p, p - lr * T.true_div(g, T.sqrt(c + 1e-8))) for p, g, c in zip(self.parameters, self.gradients, self.cache) ] #self.updates += [(c, rmsprop_decay_rate*c + (1-rmsprop_decay_rate)*T.sqr(g)) for c,g in zip(self.cache, self.gradients)] elif optimizer == 'Adam': self.updates += [ (p, p - lr * T.true_div(fm, T.sqrt(sm + 1e-8))) for p, fm, sm in zip(self.parameters, self.adamfm, self.adamsm) ] self.updates += [(fm, adam_beta1 * fm + (1 - adam_beta1) * g) for fm, g in zip(self.adamfm, self.gradients)] self.updates += [(sm, adam_beta2 * sm + (1 - adam_beta2) * T.sqr(g)) for sm, g in zip(self.adamsm, self.gradients)] self.updates += [(self.adamc, self.adamc + 1)] else: raise ValueError("Optimizer not correct") self.train_inputs = [self.X, self.Y] self.train_outputs = [self.loss] #print(self.updates) self.train_fn = theano.function(inputs=self.train_inputs, outputs=self.train_outputs, updates=self.updates) error_plot = [] num_iter = x_ip.shape[0] // batch_size for epoch in range(num_epochs): for n_iter in range(num_iter): x_batch = x_ip[n_iter * batch_size:n_iter * batch_size + batch_size, :] y_batch = y_ip[n_iter * batch_size:n_iter * batch_size + batch_size, :] self.train_fn(x_batch, y_batch) if epoch % 2 == 0: error_plot.append(self.train_fn(x_ip, y_ip)) plt.plot(error_plot)