def test_eval_points(Simulator, nl_nodirect, plt, seed, rng): n = 100 d = 5 filter = 0.08 eval_points = np.logspace(np.log10(300), np.log10(5000), 11) eval_points = np.round(eval_points).astype('int') max_points = eval_points.max() n_trials = 1 rmses = np.nan * np.zeros((len(eval_points), n_trials)) for j in range(n_trials): points = rng.normal(size=(max_points, d)) points *= (rng.uniform(size=max_points) / norm(points, axis=-1))[:, None] rng_j = np.random.RandomState(348 + j) seed = 903824 + j # generate random input in unit hypersphere x = rng_j.normal(size=d) x *= rng_j.uniform() / norm(x) for i, n_points in enumerate(eval_points): model = nengo.Network(seed=seed) with model: model.config[nengo.Ensemble].neuron_type = nl_nodirect() u = nengo.Node(output=x) a = nengo.Ensemble(n * d, dimensions=d, eval_points=points[:n_points]) nengo.Connection(u, a, synapse=0) up = nengo.Probe(u) ap = nengo.Probe(a) with Timer() as timer: sim = Simulator(model) sim.run(10 * filter) t = sim.trange() xt = nengo.synapses.filtfilt(sim.data[up], filter, dt=sim.dt) yt = nengo.synapses.filtfilt(sim.data[ap], filter, dt=sim.dt) t0 = 5 * filter t1 = 7 * filter tmask = (t > t0) & (t < t1) rmses[i, j] = rms(yt[tmask] - xt[tmask]) print("done %d (%d) in %0.3f s" % (n_points, j, timer.duration)) # subtract out mean for each model rmses_norm = rmses - rmses.mean(0, keepdims=True) mean = rmses_norm.mean(1) low = rmses_norm.min(1) high = rmses_norm.max(1) plt.semilogx(eval_points, mean, 'k-') plt.semilogx(eval_points, high, 'r-') plt.semilogx(eval_points, low, 'b-') plt.xlim([eval_points[0], eval_points[-1]]) plt.xticks(eval_points, eval_points)
def similarity(v1, v2): # v1 and v2 are vectors eps = np.nextafter(0, 1) # smallest float above zero dot = np.dot(v1, v2) dot /= max(npext.norm(v1), eps) dot /= max(npext.norm(v2), eps) return dot
def cd_encoders_biases(n_encoders, trainX, trainY, rng=np.random, mask=None, norm_min=0.05, norm_tries=10): """Constrained difference (CD) method for encoders from data [1]_. Parameters ========== n_encoders : int Number of encoders to generate. trainX : (n_samples, n_dimensions) array-like Training features. trainY : (n_samples,) array-like Training labels. Returns ======= encoders : (n_encoders, n_dimensions) array Generated encoders. biases : (n_encoders,) array Generated biases. These are biases assuming `f = G[E * X + b]`, and are therefore more like Nengo's `intercepts`. References ========== .. [1] McDonnell, M. D., Tissera, M. D., Vladusich, T., Van Schaik, A., Tapson, J., & Schwenker, F. (2015). Fast, simple and accurate handwritten digit classification by training shallow neural network classifiers with the "Extreme learning machine" algorithm. PLoS ONE, 10(8), 1-20. doi:10.1371/journal.pone.0134254 """ assert trainX.shape[0] == trainY.size trainX = trainX.reshape(trainX.shape[0], -1) trainY = trainY.ravel() d = trainX.shape[1] classes = np.unique(trainY) assert mask is None or mask.shape == (n_encoders, d) inds = [(trainY == label).nonzero()[0] for label in classes] train_norm = npext.norm(trainX, axis=1).mean() encoders = np.zeros((n_encoders, d)) biases = np.zeros(n_encoders) for k in range(n_encoders): for _ in range(norm_tries): i, j = rng.choice(len(classes), size=2, replace=False) a, b = trainX[rng.choice(inds[i])], trainX[rng.choice(inds[j])] dab = a - b if mask is not None: dab *= mask[k] ndab = npext.norm(dab)**2 if ndab >= norm_min * train_norm: break else: raise ValueError("Cannot find valid encoder") encoders[k] = (2. / ndab) * dab biases[k] = np.dot(a + b, dab) / ndab return encoders, biases
def test_sqrt_beta(n, m, rng): num_samples = 1000 num_bins = 5 vectors = rng.randn(num_samples, n + m) vectors /= npext.norm(vectors, axis=1, keepdims=True) expectation, _ = np.histogram(npext.norm(vectors[:, :m], axis=1), bins=num_bins) dist = SqrtBeta(n, m) samples = dist.sample(num_samples, 1, rng=rng) histogram, _ = np.histogram(samples, bins=num_bins) assert np.all(np.abs(np.asfarray(histogram - expectation) / num_samples) < 0.16)
def test_sqrt_beta(n, m, rng): num_samples = 250 num_bins = 5 vectors = rng.randn(num_samples, n + m) vectors /= npext.norm(vectors, axis=1, keepdims=True) expectation, _ = np.histogram( npext.norm(vectors[:, :m], axis=1), bins=num_bins) dist = dists.SqrtBeta(n, m) samples = dist.sample(num_samples, 1, rng=rng) hist, _ = np.histogram(samples, bins=num_bins) assert np.all(np.abs(np.asfarray(hist - expectation) / num_samples) < 0.16)
def test_derivative(f, df): get_network = lambda **kwargs: Network( weights, f=f, df=df, biases=None, noise=0, **kwargs) bp_learner = BPLearner(get_network(), squared_cost, rms_error, eta=eta, alpha=alpha, name='BP') bp_learner.weight_norms = [] fas_learner = FASkipLearner(get_network(), squared_cost, rms_error, eta=eta, alpha=alpha, name='DFA') fas_learner.Bs = directBs learners = [bp_learner, fas_learner] for learner in learners: learner.train(1, batch_fn) for learner in learners: print(", ".join("||W%d|| = %0.3f" % (i, norm(w)) for i, w in enumerate(learner.network.weights))) return learners
def test_eval_points_scaling(Simulator, sample, radius, seed, rng, scale): eval_points = UniformHypersphere() if sample: eval_points = eval_points.sample(500, 3, rng=rng) model = nengo.Network(seed=seed) with model: a = nengo.Ensemble( 1, 3, encoders=np.ones((1, 3)), intercepts=nengo.dists.Choice([-1]), radius=radius, ) b = nengo.Ensemble(1, 3) con = nengo.Connection(a, b, eval_points=eval_points, scale_eval_points=scale) with Simulator(model) as sim: dists = npext.norm(sim.data[con].eval_points, axis=1) limit = radius if scale else 1.0 assert np.all(dists <= limit) assert np.any(dists >= 0.9 * limit)
def test_sqrt_beta(n, m): np.random.seed(33) num_samples = 250 num_bins = 5 vectors = np.random.randn(num_samples, n + m) vectors /= npext.norm(vectors, axis=1, keepdims=True) expectation, _ = np.histogram(npext.norm(vectors[:, :m], axis=1), bins=num_bins) dist = dists.SqrtBeta(n, m) samples = dist.sample(num_samples, 1) hist, _ = np.histogram(samples, bins=num_bins) assert np.all(np.abs(np.asfarray(hist - expectation) / num_samples) < 0.16)
def test_hypersphere_surface(dimensions, rng): n = 150 * dimensions dist = dists.UniformHypersphere(surface=True) samples = dist.sample(n, dimensions, rng=rng) assert samples.shape == (n, dimensions) assert np.allclose(npext.norm(samples, axis=1), 1) assert np.allclose(np.mean(samples, axis=0), 0, atol=0.25 / dimensions)
def test_hypersphere_surface(cls, dimensions, rng, allclose): n = 200 * dimensions dist = cls(surface=True) samples = dist.sample(n, dimensions, rng=rng) assert samples.shape == (n, dimensions) assert allclose(npext.norm(samples, axis=1), 1) assert allclose(np.mean(samples, axis=0), 0, atol=0.25 / dimensions)
def test_hypersphere_volume(min_magnitude, d, rng): n = 150 * d dist = dists.UniformHypersphere(min_magnitude=min_magnitude) samples = dist.sample(n, d, rng=rng) assert samples.shape == (n, d) assert np.allclose(np.mean(samples, axis=0), 0, atol=0.1) assert np.all(npext.norm(samples, axis=1) >= min_magnitude)
def test_state_norm(plt): # Choose a filter, timestep, and number of simulation timesteps sys = Alpha(0.1) dt = 0.000001 length = 2000000 # Modify the state-space to read out the state vector A, B, C, D = sys2ss(sys) old_C = C C = np.eye(len(A)) D = np.zeros((len(A), B.shape[1])) response = np.empty((length, len(C))) for i in range(len(C)): # Simulate the state vector response[:, i] = impulse((A, B, C[i, :], D[i, :]), dt, length) # Check that the power of each state equals the H2-norm of each state # The analog case is the same after scaling since dt is approx 0. actual = norm(response, axis=0) * dt assert np.allclose(actual, state_norm(cont2discrete(sys, dt))) assert np.allclose(actual, state_norm(sys) * np.sqrt(dt)) plt.figure() plt.plot(response[:, 0], label="$x_0$") plt.plot(response[:, 1], label="$x_1$") plt.plot(np.dot(response, old_C.T), label="$y$") plt.legend()
def similarity(data, vocab, normalize=False): """Return the similarity between simulation data and Semantic Pointers. Computes the dot products between all Semantic Pointers in the Vocabulary and the simulation data for each timestep. If ``normalize=True``, normalizes all vectors to compute the cosine similarity. Parameters ---------- data: (D,) or (T, D) array_like The *D*-dimensional data for *T* timesteps used for comparison. vocab: Vocabulary or array_like Vocabulary (or list of vectors) used to calculate the similarity values. normalize : bool, optional Whether to normalize all vectors, to compute the cosine similarity. """ if isinstance(data, SemanticPointer): data = data.v if isinstance(vocab, Vocabulary): vectors = vocab.vectors elif is_iterable(vocab): if isinstance(next(iter(vocab)), SemanticPointer): vocab = [p.v for p in vocab] vectors = np.array(vocab, copy=False, ndmin=2) else: raise ValidationError("%r object is not a valid vocabulary" % (type(vocab).__name__), attr='vocab') dots = np.dot(vectors, data.T) if normalize: # Zero-norm vectors should return zero, so avoid divide-by-zero error eps = np.nextafter(0, 1) # smallest float above zero dnorm = np.maximum(npext.norm(data.T, axis=0, keepdims=True), eps) vnorm = np.maximum(npext.norm(vectors, axis=1, keepdims=True), eps) if len(dots.shape) == 1: vnorm = np.squeeze(vnorm) dots /= dnorm dots /= vnorm return dots.T
def ciw_encoders( n_encoders, trainX, trainY, rng=np.random, normalize_data=True, normalize_encoders=True, ): """Computed Input Weights (CIW) method for encoders from data. Parameters ---------- n_encoders : int Number of encoders to generate. trainX : (n_samples, n_dimensions) array-like Training features. trainY : (n_samples,) array-like Training labels. Returns ------- encoders : (n_encoders, n_dimensions) array Generated encoders. References ---------- .. [1] McDonnell, M. D., Tissera, M. D., Vladusich, T., Van Schaik, A., Tapson, J., & Schwenker, F. (2015). Fast, simple and accurate handwritten digit classification by training shallow neural network classifiers with the "Extreme learning machine" algorithm. PLoS ONE, 10(8), 1-20. doi:10.1371/journal.pone.0134254 """ assert trainX.shape[0] == trainY.size trainX = trainX.reshape((trainX.shape[0], -1)) trainY = trainY.ravel() classes = np.unique(trainY) assert n_encoders % len(classes) == 0 n_enc_per_class = n_encoders / len(classes) # normalize if normalize_data: trainX = (trainX - trainX.mean()) / trainX.std() # trainX = (trainX - trainX.mean(axis=0)) / trainX.std() # trainX = (trainX - trainX.mean(axis=0)) / (trainX.std(axis=0) + 1e-8) # generate encoders = [] for label in classes: X = trainX[trainY == label] plusminus = rng.choice([-1, 1], size=(X.shape[0], n_enc_per_class)) samples = np.dot(plusminus.T, X) encoders.append(samples) encoders = np.vstack(encoders) if normalize_encoders: encoders /= npext.norm(encoders, axis=1, keepdims=True) return encoders
def test_hypersphere_surface(dimensions): n = 100 * dimensions dist = dists.UniformHypersphere(surface=True) samples = dist.sample(n, dimensions, np.random.RandomState(1)) assert samples.shape == (n, dimensions) assert np.allclose(npext.norm(samples, axis=1), 1) assert np.allclose( np.mean(samples, axis=0), np.zeros(dimensions), atol=0.1)
def test_sphere(d, rng): n = 200 x = sphere.sample(n, d, rng) assert x.shape == (n, d) assert np.allclose(norm(x, axis=1), 1) f = _furthest(x) assert (f > 1.5).all()
def test_hypersphere_surface(dimensions): n = 100 * dimensions dist = dists.UniformHypersphere(dimensions, surface=True) samples = dist.sample(n, np.random.RandomState(1)) assert samples.shape == (n, dimensions) assert np.allclose(npext.norm(samples, axis=1), 1) assert np.allclose( np.mean(samples, axis=0), np.zeros(dimensions), atol=0.1)
def error_layers_plots(dt, t, learners): vsynapse = Alpha(0.01, default_dt=dt) for learner in [l for l in learners if 'els' in l]: plt.figure() plt.subplot(211) dind = 0 e = vsynapse.filtfilt(learner['e']) els = [vsynapse.filtfilt(el) for el in learner['els']] plt.plot(t, e[:, dind]) [plt.plot(t, el[:, dind]) for el in els] plt.subplot(212) plt.plot(t, norm(e, axis=1)) [plt.plot(t, norm(el, axis=1)) for el in els] plt.show()
def build_ensemble(model, ens): # Create random number generator rng = np.random.RandomState(model.seeds[ens]) eval_points = gen_eval_points(ens, ens.eval_points, rng=rng) # Set up encoders if isinstance(ens.neuron_type, nengo.Direct): encoders = np.identity(ens.dimensions) elif isinstance(ens.encoders, Distribution): encoders = get_samples(ens.encoders, ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) if ens.normalize_encoders: encoders /= npext.norm(encoders, axis=1, keepdims=True) # Build the neurons gain, bias, max_rates, intercepts = get_gain_bias(ens, rng, model.intercept_limit) group = CxGroup(ens.n_neurons, label='%s' % ens) group.bias[:] = bias model.build(ens.neuron_type, ens.neurons, group) # set default filter just in case no other filter gets set group.configure_default_filter(model.inter_tau, dt=model.dt) if ens.noise is not None: raise NotImplementedError("Ensemble noise not implemented") # Scale the encoders if isinstance(ens.neuron_type, nengo.Direct): raise NotImplementedError("Direct neurons not implemented") # scaled_encoders = encoders else: # to keep scaling reasonable, we don't include the radius # scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] scaled_encoders = encoders * gain[:, np.newaxis] model.add_group(group) model.objs[ens]['in'] = group model.objs[ens]['out'] = group model.objs[ens.neurons]['in'] = group model.objs[ens.neurons]['out'] = group model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders, gain=gain, bias=bias)
def test_ball(d, rng): n = 200 x = ball.sample(n, d, rng) assert x.shape == (n, d) dist = norm(x, axis=1) assert (dist <= 1).all() f = _furthest(x) assert (f > dist + 0.5).all()
def similarity(data, vocab, normalize=False): """Return the similarity between some data and the vocabulary. Computes the dot products between all data vectors and each vocabulary vector. If ``normalize=True``, normalizes all vectors to compute the cosine similarity. Parameters ---------- data: array_like The data used for comparison. vocab: Vocabulary or array_like Vocabulary (or list of vectors) to use to calculate the similarity values. normalize : bool, optional (Default: False) Whether to normalize all vectors, to compute the cosine similarity. """ from nengo.spa.vocab import Vocabulary if isinstance(vocab, Vocabulary): vectors = vocab.vectors elif npext.is_iterable(vocab): vectors = np.array(vocab, copy=False, ndmin=2) else: raise ValidationError("%r object is not a valid vocabulary" % (type(vocab).__name__), attr='vocab') data = np.array(data, copy=False, ndmin=2) dots = np.dot(data, vectors.T) if normalize: # Zero-norm vectors should return zero, so avoid divide-by-zero error eps = np.nextafter(0, 1) # smallest float above zero dnorm = np.maximum(npext.norm(data, axis=1, keepdims=True), eps) vnorm = np.maximum(npext.norm(vectors, axis=1, keepdims=True), eps) dots /= dnorm dots /= vnorm.T return dots
def ciw_encoders(n_encoders, trainX, trainY, rng=np.random, normalize_data=True, normalize_encoders=True): """Computed Input Weights (CIW) method for encoders from data [1]_. Parameters ========== n_encoders : int Number of encoders to generate. trainX : (n_samples, n_dimensions) array-like Training features. trainY : (n_samples,) array-like Training labels. Returns ======= encoders : (n_encoders, n_dimensions) array Generated encoders. References ========== .. [1] McDonnell, M. D., Tissera, M. D., Vladusich, T., Van Schaik, A., Tapson, J., & Schwenker, F. (2015). Fast, simple and accurate handwritten digit classification by training shallow neural network classifiers with the "Extreme learning machine" algorithm. PLoS ONE, 10(8), 1-20. doi:10.1371/journal.pone.0134254 """ assert trainX.shape[0] == trainY.size trainX = trainX.reshape(trainX.shape[0], -1) trainY = trainY.ravel() classes = np.unique(trainY) assert n_encoders % len(classes) == 0 n_enc_per_class = n_encoders / len(classes) # normalize if normalize_data: trainX = (trainX - trainX.mean()) / trainX.std() # trainX = (trainX - trainX.mean(axis=0)) / trainX.std() # trainX = (trainX - trainX.mean(axis=0)) / (trainX.std(axis=0) + 1e-8) # generate encoders = [] for label in classes: X = trainX[trainY == label] plusminus = rng.choice([-1, 1], size=(X.shape[0], n_enc_per_class)) samples = np.dot(plusminus.T, X) encoders.append(samples) encoders = np.vstack(encoders) if normalize_encoders: encoders /= npext.norm(encoders, axis=1, keepdims=True) return encoders
def initial_w(shape, kind='', normkind=None, offset=0.0, scale=1.0, rng=np.random): assert len(shape) == 2 kind = kind.lower() if kind == 'zeros': w = np.zeros(shape) elif kind == 'identity': min_shape = min(shape) ceili = lambda s: int(np.ceil(float(s) / min_shape)) w = np.tile(np.eye(min_shape), (ceili(shape[0]), ceili(shape[1])))[:shape[0], :shape[1]] w -= w.mean() elif kind == 'binary': w = 2. * rng.randint(0, 2, size=shape) - 1. elif kind == 'uniform': w = rng.uniform(-1, 1, size=shape) elif kind == 'norm_uniform': w = rng.uniform(-1, 1, size=shape) w /= norm(w) elif kind in ['normal', 'gaussian']: w = rng.normal(size=shape) elif kind in ['norm_normal', 'norm_gaussian']: w = rng.normal(size=shape) w /= norm(w) elif kind == 'ortho': # w = orthogonalize(rng.uniform(-1, 1, size=shape)) w = orthogonalize(rng.normal(size=shape)) elif kind == 'orthonorm': # w = orthogonalize(rng.uniform(-1, 1, size=shape)) w = orthogonalize(rng.normal(size=shape)) w /= norm(w) else: raise ValueError("Unrecognized kind %r" % kind) # normalixe (let X ~ N(0, 1)) if normkind == 'left': # then WX ~ N(0, 1) w /= norm(w, axis=1, keepdims=True) elif normkind == 'right': # then XW ~ N(0, 1) w /= norm(w, axis=0, keepdims=True) if normkind == 'leftmean': # then WX ~ N(0, 1) w /= norm(w, axis=1).mean() elif normkind == 'rightmean': # then XW ~ N(0, 1) w /= norm(w, axis=0).mean() elif normkind: raise ValueError("Unrecognized norm kind %r" % normkind) if scale != 1.0: w *= scale if offset != 0.0: w += offset return w
def similarity(data, vocab, normalize=False): """Return the similarity between some data and the vocabulary. Computes the dot products between all data vectors and each vocabulary vector. If `normalize=True`, normalizes all vectors to compute the cosine similarity. Parameters ---------- data: array_like The data used for comparison. vocab: spa.Vocabulary, array_like Vocabulary (or list of vectors) to use to calculate the similarity values normalize : boolean (optional) Whether to normalize all vectors, to compute the cosine similarity. """ from nengo.spa.vocab import Vocabulary if isinstance(vocab, Vocabulary): vectors = vocab.vectors elif is_iterable(vocab): vectors = np.array(vocab, copy=False, ndmin=2) else: raise ValidationError("%r object is not a valid vocabulary" % (vocab.__class__.__name__), attr='vocab') data = np.array(data, copy=False, ndmin=2) dots = np.dot(data, vectors.T) if normalize: # Zero-norm vectors should return zero, so avoid divide-by-zero error eps = np.nextafter(0, 1) # smallest float above zero dnorm = np.maximum(npext.norm(data, axis=1, keepdims=True), eps) vnorm = np.maximum(npext.norm(vectors, axis=1, keepdims=True), eps) dots /= dnorm dots /= vnorm.T return dots
def test_eval_points_scaling(Simulator, sample, radius, seed, rng): eval_points = UniformHypersphere() if sample: eval_points = eval_points.sample(500, 3, rng=rng) model = nengo.Network(seed=seed) with model: a = nengo.Ensemble(1, 3, eval_points=eval_points, radius=radius) with Simulator(model) as sim: dists = npext.norm(sim.data[a].eval_points, axis=1) assert np.all(dists <= radius) assert np.any(dists >= 0.9 * radius)
def build_lif(model, ens): # Create a random number generator rng = np.random.RandomState(model.seeds[ens]) # Get the eval points eval_points = ensemble.gen_eval_points(ens, ens.eval_points, rng=rng) # Get the encoders if isinstance(ens.encoders, Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=rng) encoders = np.asarray(encoders, dtype=np.float64) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Get maximum rates and intercepts max_rates = ensemble.sample(ens.max_rates, ens.n_neurons, rng=rng) intercepts = ensemble.sample(ens.intercepts, ens.n_neurons, rng=rng) # Build the neurons if ens.gain is None and ens.bias is None: gain, bias = ens.neuron_type.gain_bias(max_rates, intercepts) elif ens.gain is not None and ens.bias is not None: gain = ensemble.sample(ens.gain, ens.n_neurons, rng=rng) bias = ensemble.sample(ens.bias, ens.n_neurons, rng=rng) else: raise NotImplementedError( "gain or bias set for {!s}, but not both. Solving for one given " "the other is not yet implemented.".format(ens) ) # Scale the encoders scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] # Store all the parameters model.params[ens] = BuiltEnsemble( eval_points=eval_points, encoders=encoders, scaled_encoders=scaled_encoders, max_rates=max_rates, intercepts=intercepts, gain=gain, bias=bias ) # Create the object which will handle simulation of the LIF ensemble. This # object will be responsible for adding items to the netlist and providing # functions to prepare the ensemble for simulation. The object may be # modified by later methods. model.object_operators[ens] = operators.EnsembleLIF(ens)
def test_encoders(n_dimensions, n_neurons=10, encoders=None): if encoders is None: encoders = np.random.normal(size=(n_neurons, n_dimensions)) encoders /= norm(encoders, axis=-1, keepdims=True) model = nengo.Network(label="_test_encoders") with model: ens = nengo.Ensemble(neurons=nengo.LIF(n_neurons), dimensions=n_dimensions, encoders=encoders, label="A") sim = nengo.Simulator(model) assert np.allclose(encoders, sim.data[ens].encoders)
def test_encoders(n_dimensions, n_neurons=10, encoders=None): if encoders is None: encoders = np.random.normal(size=(n_neurons, n_dimensions)) encoders /= norm(encoders, axis=-1, keepdims=True) args = {'label': 'A', 'neurons': nengo.LIF(n_neurons), 'dimensions': n_dimensions} model = nengo.Model('_test_encoders') ens = nengo.Ensemble(encoders=encoders, **args) sim = nengo.Simulator(model) assert np.allclose(encoders, sim.data[ens].encoders)
def test_sphere(d, ntm, rng): sphere = ScatteredHypersphere(surface=True, base=ntm) n = 1000 x = sphere.sample(n, d, rng) assert x.shape == (n, d) assert abs(np.mean(x)) < 0.1 / d _compare_samples(x, UniformHypersphere(surface=True).sample(n, d, rng)) assert np.allclose(norm(x, axis=1), 1) f = _furthest(x) assert (f > 1.5).all()
def sample(self, n, d=None, rng=np.random): assert d == self.n_dimensions if self.weighting == 'dimensions': # sample in d-dimensional space, but project down to # len(dist)-dimensional space for the weights to bias # the projection so that each distribution is projected # from a proportional number of dimensions samples = rng.randn(n, d) samples /= npext.norm(samples, axis=1, keepdims=True) weights = np.zeros((n, len(self.distributions))) start = 0 for i, end in enumerate(np.cumsum(self.dimensions)): weights[:, i] = npext.norm(samples[:, start:end], axis=1, keepdims=False) start = end elif self.weighting == 'distributions': samples = rng.randn(n, len(self.distributions)) samples /= npext.norm(samples, axis=1, keepdims=True) weights = samples else: # use the original weighting method if nothing is specified data = [ dist.sample(n, d=self.dimensions[i], rng=rng) * self.weights[i] for i, dist in enumerate(self.distributions) ] return np.hstack(data) data = [ dist.sample(n, d=self.dimensions[i], rng=rng) * weights[:, i].reshape((n, 1)) for i, dist in enumerate(self.distributions) ] return np.hstack(data)
def test_encoders(RefSimulator, dimensions, seed, n_neurons=10, encoders=None): if encoders is None: encoders = np.random.normal(size=(n_neurons, dimensions)) encoders = npext.array(encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) model = nengo.Network(label="_test_encoders", seed=seed) with model: ens = nengo.Ensemble(n_neurons=n_neurons, dimensions=dimensions, encoders=encoders, label="A") with RefSimulator(model) as sim: assert np.allclose(encoders, sim.data[ens].encoders)
def test_encoders(RefSimulator, dimensions, seed, n_neurons=10, encoders=None): if encoders is None: encoders = np.random.normal(size=(n_neurons, dimensions)) encoders = npext.array(encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) model = nengo.Network(label="_test_encoders", seed=seed) with model: ens = nengo.Ensemble(n_neurons=n_neurons, dimensions=dimensions, encoders=encoders, label="A") sim = RefSimulator(model) assert np.allclose(encoders, sim.data[ens].encoders)
def test_ball(d, ntm, rng): ball = ScatteredHypersphere(surface=False, base=ntm) n = 1000 x = ball.sample(n, d, rng) assert x.shape == (n, d) assert abs(np.mean(x)) < 0.1 / d _compare_samples(x, UniformHypersphere(surface=False).sample(n, d, rng)) dist = norm(x, axis=1) assert (dist <= 1).all() f = _furthest(x) assert (f > dist + 0.5).all()
def sample(self, n, d, rng=np.random): if d is None or d < 1: # check this, since other dists allow d = None raise ValueError("Dimensions must be a positive integer") samples = rng.randn(n, d) samples /= npext.norm(samples, axis=1, keepdims=True) if self.surface: return samples # Generate magnitudes for vectors from uniform distribution. # The (1 / d) exponent ensures that samples are uniformly distributed # in n-space and not all bunched up at the centre of the sphere. samples *= rng.rand(n, 1)**(1.0 / d) return samples
def test_leech_kissing(tmpdir): cache_file = os.path.join(str(tmpdir), "leech_kissing.npy") x = leech_kissing(cache_file) assert x.shape == (196560, 24) assert np.allclose(norm(x, axis=1, keepdims=True), 1) assert len(set(map(tuple, x))) == len(x) # no duplicates assert os.path.exists(cache_file) x_cached = leech_kissing(cache_file) assert np.allclose(x, x_cached) s = np.random.choice(len(x), 1000, replace=False) # The only vectors in x that have dot product with s > 0.5 are those # from s themselves, and so this is exactly 1 (per vector in s) assert np.all(np.sum(np.dot(x, x[s].T) > 0.5, axis=0) == 1)
def sample(self, n, d, rng=np.random): if d is None or d < 1: # check this, since other dists allow d = None raise ValueError("Dimensions must be a positive integer") samples = rng.randn(n, d) samples /= npext.norm(samples, axis=1, keepdims=True) if self.surface: return samples # Generate magnitudes for vectors from uniform distribution. # The (1 / d) exponent ensures that samples are uniformly distributed # in n-space and not all bunched up at the centre of the sphere. samples *= rng.rand(n, 1) ** (1.0 / d) return samples
def build_lif(model, ens): # Create a random number generator rng = np.random.RandomState(model.seeds[ens]) # Get the eval points eval_points = ensemble.gen_eval_points(ens, ens.eval_points, rng=rng) # Get the encoders if isinstance(ens.encoders, Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=rng) encoders = np.asarray(encoders, dtype=np.float64) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Get maximum rates and intercepts max_rates = ensemble.sample(ens.max_rates, ens.n_neurons, rng=rng) intercepts = ensemble.sample(ens.intercepts, ens.n_neurons, rng=rng) # Build the neurons if ens.gain is None and ens.bias is None: gain, bias = ens.neuron_type.gain_bias(max_rates, intercepts) elif ens.gain is not None and ens.bias is not None: gain = ensemble.sample(ens.gain, ens.n_neurons, rng=rng) bias = ensemble.sample(ens.bias, ens.n_neurons, rng=rng) else: raise NotImplementedError( "gain or bias set for {!s}, but not both. Solving for one given " "the other is not yet implemented.".format(ens)) # Scale the encoders scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] # Store all the parameters model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, scaled_encoders=scaled_encoders, max_rates=max_rates, intercepts=intercepts, gain=gain, bias=bias) # Create the object which will handle simulation of the LIF ensemble. This # object will be responsible for adding items to the netlist and providing # functions to prepare the ensemble for simulation. The object may be # modified by later methods. model.object_operators[ens] = operators.EnsembleLIF(ens)
def test_eval_points_scaling(Simulator, sample, radius, seed, rng, scale): eval_points = UniformHypersphere() if sample: eval_points = eval_points.sample(500, 3, rng=rng) model = nengo.Network(seed=seed) with model: a = nengo.Ensemble(1, 3, radius=radius) b = nengo.Ensemble(1, 3) con = nengo.Connection(a, b, eval_points=eval_points, scale_eval_points=scale) sim = Simulator(model) dists = npext.norm(sim.data[con].eval_points, axis=1) limit = radius if scale else 1.0 assert np.all(dists <= limit) assert np.any(dists >= 0.9 * limit)
def test_hypersphere_volume(cls, min_magnitude, d, rng, allclose): n = 250 * d dist = cls(min_magnitude=min_magnitude) samples = dist.sample(n, d, rng=rng) assert samples.shape == (n, d) assert allclose(np.mean(samples, axis=0), 0, atol=0.1) norms = npext.norm(samples, axis=1) assert np.all(norms >= min_magnitude) assert np.all(norms <= 1) # probability of not finding a point in [min_magnitude, r_tol_min], [r_tol_max, 1] q = 1e-5 r_min_d = min_magnitude ** d r_tol_min = (r_min_d + (1 - r_min_d) * (1 - q ** (1 / n))) ** (1 / d) assert norms.min() <= r_tol_min r_tol_max = (1 - (1 - r_min_d) * (1 - q ** (1 / n))) ** (1 / d) assert norms.max() >= r_tol_max
def test_state_norm(plt): # Choose a filter, timestep, and number of simulation timesteps sys = Alpha(0.1) dt = 0.000001 length = 2000000 assert np.allclose(dt * length, 2.0) # Check that the power of each state equals the H2-norm of each state # The analog case is the same after scaling since dt is approx 0. response = sys.X.impulse(length, dt) actual = norm(response, axis=0) * dt assert np.allclose(actual, state_norm(cont2discrete(sys, dt))) assert np.allclose(actual, state_norm(sys) * np.sqrt(dt)) step = int(0.002 / dt) plt.figure() plt.plot(response[::step, 0], label="$x_0$") plt.plot(response[::step, 1], label="$x_1$") plt.plot(np.dot(response[::step], sys.C.T), label="$y$") plt.legend()
def trials_error_plot(prestime, t, ystar, learners): pdt = 0.01 vsynapse = Alpha(0.02, default_dt=pdt) plt.figure() dinds = slice(0, 2) plt.subplot(211) plt.plot(t, ystar[:, dinds]) for learner in learners: y = vsynapse.filtfilt(learner['y'][:, dinds]) plt.plot(t, y) plt.ylabel('outputs') plt.subplot(212) esynapse = Alpha(5 * prestime, default_dt=pdt) for learner in learners: e = norm(esynapse.filtfilt(learner['e']), axis=1) plt.plot(t, e) plt.ylabel('errors')
def test_derivative(f, df): get_network = lambda **kwargs: Network( weights, f=f, df=df, biases=None, noise=0, **kwargs) bp_learner = BPLearner(get_network(), cost, error, eta=eta, alpha=alpha, name='BP') bp_learner.weight_norms = [] # fa_learner = FALearner( # get_network(), squared_cost, rms_error, eta=eta, alpha=alpha) # fa_learner.Bs = [initial_w((j, i), kind='ortho', scale=2) # for i, j in zip(dhids, dhids[1:] + [dout])] # fa_learner.bp_angles = [] # # fa_learner.pbp_angles = [] fas_learner = FASkipLearner(get_network(), cost, error, eta=eta, alpha=alpha, name='FA') genB = lambda shape: initial_w( shape, kind='ortho', normkind='rightmean', scale=0.2) fas_learner.Bs = [genB((dout, dhid)) for dhid in dhids] # learners = [bp_learner, fa_learner] learners = [bp_learner, fas_learner] for learner in learners: learner.train(epochs, batch_fn, test_set=test_set) for learner in learners: print(", ".join("||W%d|| = %0.3f" % (i, norm(w)) for i, w in enumerate(learner.network.weights))) return learners
def make_pool(self, ens): if isinstance(ens.encoders, Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=self.rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) if self.config[ens].compact: p = pool.CompactPool(ens.n_neurons) elif self.config[ens].fixed: p = pool.FixedPool(ens.n_neurons, bits_soma=self.config[ens].fixed_bits_soma, bits_syn=self.config[ens].fixed_bits_syn) else: p = pool.StdPool(ens.n_neurons) intercepts = nengo.builder.sample(ens.intercepts, ens.n_neurons, rng=self.rng) max_rates = nengo.builder.sample(ens.max_rates, ens.n_neurons, rng=self.rng) gain, bias = self.find_gain_bias(p.soma, intercepts, max_rates) p.set_bias(bias) print 'bias', p.get_bias() scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] self.pools[ens] = p self.model.params[ens] = BuiltEnsemble(intercepts=intercepts, max_rates=max_rates, gain=gain, bias=bias, encoders=encoders, scaled_encoders=scaled_encoders, eval_points=None, )
def build_ensemble(model, ens): """Builds an `.Ensemble` object into a model. A brief summary of what happens in the ensemble build process, in order: 1. Generate evaluation points and encoders. 2. Normalize encoders to unit length. 3. Determine bias and gain. 4. Create neuron input signal 5. Add operator for injecting bias. 6. Call build function for neuron type. 7. Scale encoders by gain and radius. 8. Add operators for multiplying decoded input signal by encoders and incrementing the result in the neuron input signal. 9. Call build function for injected noise. Some of these steps may be altered or omitted depending on the parameters of the ensemble, in particular the neuron type. For example, most steps are omitted for the `.Direct` neuron type. Parameters ---------- model : Model The model to build into. ens : Ensemble The ensemble to build. Notes ----- Sets ``model.params[ens]`` to a `.BuiltEnsemble` instance. """ # Create random number generator rng = np.random.RandomState(model.seeds[ens]) eval_points = gen_eval_points(ens, ens.eval_points, rng=rng) # Set up signal model.sig[ens]['in'] = Signal(np.zeros(ens.dimensions), name="%s.signal" % ens) model.add_op(Reset(model.sig[ens]['in'])) # Set up encoders if isinstance(ens.neuron_type, Direct): encoders = np.identity(ens.dimensions) elif isinstance(ens.encoders, Distribution): encoders = get_samples( ens.encoders, ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) if ens.normalize_encoders: encoders /= npext.norm(encoders, axis=1, keepdims=True) # Build the neurons gain, bias, max_rates, intercepts = get_gain_bias(ens, rng) if isinstance(ens.neuron_type, Direct): model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.dimensions), name='%s.neuron_in' % ens) model.sig[ens.neurons]['out'] = model.sig[ens.neurons]['in'] model.add_op(Reset(model.sig[ens.neurons]['in'])) else: model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_in" % ens) model.sig[ens.neurons]['out'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_out" % ens) model.sig[ens.neurons]['bias'] = Signal( bias, name="%s.bias" % ens, readonly=True) model.add_op(Copy(model.sig[ens.neurons]['bias'], model.sig[ens.neurons]['in'])) # This adds the neuron's operator and sets other signals model.build(ens.neuron_type, ens.neurons) # Scale the encoders if isinstance(ens.neuron_type, Direct): scaled_encoders = encoders else: scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] model.sig[ens]['encoders'] = Signal( scaled_encoders, name="%s.scaled_encoders" % ens, readonly=True) # Inject noise if specified if ens.noise is not None: model.build(ens.noise, sig_out=model.sig[ens.neurons]['in'], inc=True) # Create output signal, using built Neurons model.add_op(DotInc( model.sig[ens]['encoders'], model.sig[ens]['in'], model.sig[ens.neurons]['in'], tag="%s encoding" % ens)) # Output is neural output model.sig[ens]['out'] = model.sig[ens.neurons]['out'] model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders, gain=gain, bias=bias)
def sorted_neurons(ensemble, sim, iterations=100, seed=None): '''Sort neurons in an ensemble by encoder and intercept. Parameters ---------- ensemble: nengo.Ensemble The population of neurons to be sorted. The ensemble must have its encoders specified. iterations: int The number of times to iterate during the sort. seed: float A random number seed. Returns ------- indices: ndarray An array with sorted indices into the neurons in the ensemble Examples -------- You can use this to generate an array of sorted indices for plotting. This can be done after collecting the data. E.g. >>>indices = sorted_neurons(simulator, 'My neurons') >>>plt.figure() >>>rasterplot(sim.data['My neurons.spikes'][:,indices]) Algorithm --------- The algorithm is for each encoder in the initial set, randomly pick another encoder and check to see if swapping those two encoders would reduce the average difference between the encoders and their neighbours. Difference is measured as the dot product. Each encoder has four neighbours (N, S, E, W), except for the ones on the edges which have fewer (no wrapping). This algorithm is repeated `iterations` times, so a total of `iterations*N` swaps are considered. ''' # Normalize all the encoders encoders = np.array(sim.data[ensemble].encoders) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Make an array with the starting order of the neurons N = encoders.shape[0] indices = np.arange(N) rng = np.random.RandomState(seed) for k in range(iterations): target = rng.randint(0, N, N) # pick random swap targets for i in range(N): j = target[i] if i != j: # if not swapping with yourself # compute similarity score how we are (unswapped) sim1 = (_similarity(encoders, i, N) + _similarity(encoders, j, N)) # swap the encoder encoders[[i, j], :] = encoders[[j, i], :] indices[[i, j]] = indices[[j, i]] # compute similarity score how we are (swapped) sim2 = (_similarity(encoders, i, N) + _similarity(encoders, j, N)) # if we were better unswapped if sim1 > sim2: # swap them back encoders[[i, j], :] = encoders[[j, i], :] indices[[i, j]] = indices[[j, i]] return indices
def from_object(cls, ens, out_conns, dt, rng): assert isinstance(ens.neuron_type, nengo.neurons.LIF) assert isinstance(ens, nengo.Ensemble) if ens.seed is None: rng = np.random.RandomState(rng.tomaxint()) else: rng = np.random.RandomState(ens.seed) # Generate evaluation points if isinstance(ens.eval_points, dists.Distribution): n_points = ens.n_eval_points if n_points is None: n_points = nengo.utils.builder.default_n_eval_points( ens.n_neurons, ens.dimensions) eval_points = ens.eval_points.sample(n_points, ens.dimensions, rng) eval_points *= ens.radius else: if (ens.eval_points is not None and ens.eval_points.shape[0] != ens.n_eval_points): warnings.warn("Number of eval points doesn't match " "n_eval_points. Ignoring n_eval_points.") eval_points = np.array(ens.eval_points, dtype=np.float64) # Determine max_rates and intercepts if isinstance(ens.max_rates, dists.Distribution): max_rates = ens.max_rates.sample(ens.n_neurons, rng=rng) else: max_rates = np.array(max_rates) if isinstance(ens.intercepts, dists.Distribution): intercepts = ens.intercepts.sample(ens.n_neurons, rng=rng) else: intercepts = np.array(intercepts) # Generate gains, bias gain, bias = ens.neuron_type.gain_bias(max_rates, intercepts) # Generate encoders if isinstance(ens.encoders, dists.Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Generate decoders for outgoing connections decoders = list() tfses = utils.connections.OutgoingEnsembleConnections(out_conns) def build_decoder(function, evals, solver): """Internal function for building a single decoder.""" if evals is None: evals = npext.array(eval_points, min_dims=2) else: evals = npext.array(evals, min_dims=2) assert solver is None or not solver.weights x = np.dot(evals, encoders.T / ens.radius) activities = ens.neuron_type.rates(x, gain, bias) if function is None: targets = evals else: (value, _) = checked_call(function, evals[0]) function_size = np.asarray(value).size targets = np.zeros((len(evals), function_size)) for i, ep in enumerate(evals): targets[i] = function(ep) if solver is None: solver = nengo.solvers.LstsqL2() return solver(activities, targets, rng=rng)[0] decoder_builder = utils.decoders.DecoderBuilder(build_decoder) # Build each of the decoders in turn for tfse in tfses.transforms_functions: decoders.append(decoder_builder.get_transformed_decoder( tfse.function, tfse.transform, tfse.eval_points, tfse.solver)) # Build list of learning rule, connection-index tuples learning_rules = list() for c in tfses: for l in utils.connections.get_learning_rules(c): learning_rules.append((l, tfses[c])) # By default compress all decoders decoders_to_compress = [True for d in decoders] # Turn off compression for all decoders associated with learning rules for l in learning_rules: decoders_to_compress[l[1]] = False # Compress and merge the decoders (decoder_headers, decoders) =\ utils.decoders.get_combined_compressed_decoders( decoders, compress=decoders_to_compress) decoders /= dt return cls(ens.n_neurons, gain, bias, encoders, decoders, ens.neuron_type.tau_rc, ens.neuron_type.tau_ref, eval_points, decoder_headers, learning_rules)
for images in [train_images, test_images]: images[:] = 2 * images - 1 # normalize to -1 to 1 # --- set up network parameters n_vis = train_images.shape[1] n_hid = 500 rng = np.random if 1: rf_shape = (9, 9) mask = create_mask(n_hid, (28, 28), rf_shape) weights = rng.normal(size=(n_hid, n_vis)) * mask weights = weights.T mask = mask.T weights /= norm(weights, axis=0, keepdims=True) neurons = nengo.LIF() gain, bias = neurons.gain_bias(200, -0.5) def encode(x): return neurons.rates(np.dot(x, weights), gain, bias) # --- determine initial decoders x = train_images[:1000] decoders, _ = nengo.decoders.LstsqL2()(encode(x), x) # x = train_images[:1000] # A = encode(x) # dshape = (n_hid, n_vis)
def build_ensemble(model, ens): # Create random number generator rng = np.random.RandomState(model.seeds[ens]) # Generate eval points if isinstance(ens.eval_points, Distribution): n_points = ens.n_eval_points if n_points is None: n_points = default_n_eval_points(ens.n_neurons, ens.dimensions) eval_points = ens.eval_points.sample(n_points, ens.dimensions, rng) # eval_points should be in the ensemble's representational range eval_points *= ens.radius else: if (ens.n_eval_points is not None and ens.eval_points.shape[0] != ens.n_eval_points): warnings.warn("Number of eval_points doesn't match " "n_eval_points. Ignoring n_eval_points.") eval_points = np.array(ens.eval_points, dtype=np.float64) # Set up signal model.sig[ens]['in'] = Signal(np.zeros(ens.dimensions), name="%s.signal" % ens) model.add_op(Reset(model.sig[ens]['in'])) # Set up encoders if isinstance(ens.neuron_type, Direct): encoders = np.identity(ens.dimensions) elif isinstance(ens.encoders, Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Determine max_rates and intercepts max_rates = sample(ens.max_rates, ens.n_neurons, rng=rng) intercepts = sample(ens.intercepts, ens.n_neurons, rng=rng) # Build the neurons if ens.gain is not None and ens.bias is not None: gain = sample(ens.gain, ens.n_neurons, rng=rng) bias = sample(ens.bias, ens.n_neurons, rng=rng) elif ens.gain is not None or ens.bias is not None: # TODO: handle this instead of error raise NotImplementedError("gain or bias set for %s, but not both. " "Solving for one given the other is not " "implemented yet." % ens) else: gain, bias = ens.neuron_type.gain_bias(max_rates, intercepts) if isinstance(ens.neuron_type, Direct): model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.dimensions), name='%s.neuron_in' % ens) model.sig[ens.neurons]['out'] = model.sig[ens.neurons]['in'] model.add_op(Reset(model.sig[ens.neurons]['in'])) else: model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_in" % ens) model.sig[ens.neurons]['out'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_out" % ens) model.add_op(Copy(src=Signal(bias, name="%s.bias" % ens), dst=model.sig[ens.neurons]['in'])) # This adds the neuron's operator and sets other signals model.build(ens.neuron_type, ens.neurons) # Scale the encoders if isinstance(ens.neuron_type, Direct): scaled_encoders = encoders else: scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] model.sig[ens]['encoders'] = Signal( scaled_encoders, name="%s.scaled_encoders" % ens) # Create output signal, using built Neurons model.add_op(DotInc( model.sig[ens]['encoders'], model.sig[ens]['in'], model.sig[ens.neurons]['in'], tag="%s encoding" % ens)) # Output is neural output model.sig[ens]['out'] = model.sig[ens.neurons]['out'] model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders, gain=gain, bias=bias)
def test_eval_points(Simulator, nl_nodirect, plt, seed, rng, logger): n = 100 d = 5 filter = 0.08 eval_points = np.logspace(np.log10(300), np.log10(5000), 11) eval_points = np.round(eval_points).astype('int') max_points = eval_points.max() n_trials = 1 rmses = np.nan * np.zeros((len(eval_points), n_trials)) for j in range(n_trials): points = rng.normal(size=(max_points, d)) points *= (rng.uniform(size=max_points) / norm(points, axis=-1))[:, None] rng_j = np.random.RandomState(348 + j) seed = 903824 + j # generate random input in unit hypersphere x = rng_j.normal(size=d) x *= rng_j.uniform() / norm(x) for i, n_points in enumerate(eval_points): model = nengo.Network(seed=seed) with model: model.config[nengo.Ensemble].neuron_type = nl_nodirect() u = nengo.Node(output=x) a = nengo.Ensemble(n * d, dimensions=d, eval_points=points[:n_points]) nengo.Connection(u, a, synapse=0) up = nengo.Probe(u) ap = nengo.Probe(a) with Timer() as timer: sim = Simulator(model) sim.run(10 * filter) t = sim.trange() xt = nengo.synapses.filtfilt(sim.data[up], filter, dt=sim.dt) yt = nengo.synapses.filtfilt(sim.data[ap], filter, dt=sim.dt) t0 = 5 * filter t1 = 7 * filter tmask = (t > t0) & (t < t1) rmses[i, j] = rms(yt[tmask] - xt[tmask]) logger.info('trial %d', j) logger.info(' n_points: %d', n_points) logger.info(' duration: %0.3f s', timer.duration) # subtract out mean for each model rmses_norm = rmses - rmses.mean(0, keepdims=True) mean = rmses_norm.mean(1) low = rmses_norm.min(1) high = rmses_norm.max(1) plt.semilogx(eval_points, mean, 'k-') plt.semilogx(eval_points, high, 'r-') plt.semilogx(eval_points, low, 'b-') plt.xlim([eval_points[0], eval_points[-1]]) plt.xticks(eval_points, eval_points)
def make_ensemble(self, ens): """Build an Ensemble.""" self.model.ensembles.append(ens) if isinstance(ens.neuron_type, nengo.Direct): self.make_direct_ensemble(ens) return p = self.make_pool(ens) if isinstance(ens.encoders, Distribution): encoders = ens.encoders.sample(ens.n_neurons, ens.dimensions, rng=self.rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) intercepts = nengo.builder.ensemble.sample(ens.intercepts, ens.n_neurons, rng=self.rng) max_rates = nengo.builder.ensemble.sample(ens.max_rates, ens.n_neurons, rng=self.rng) if ens.gain is not None and ens.bias is not None: gain = nengo.builder.ensemble.sample(ens.gain, ens.n_neurons, rng=self.rng) bias = nengo.builder.ensemble.sample(ens.bias, ens.n_neurons, rng=self.rng) elif ens.gain is not None or ens.bias is not None: raise NotImplementedError("gain or bias set for %s, but not both. " "Solving for one given the other is not " "implemented yet." % ens) else: gain, bias = neuron_tuning.find_gain_bias(p, intercepts, max_rates) scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] self.model.pools[ens] = p self.model.decoders[ens] = {} if isinstance(ens.eval_points, Distribution): n_points = ens.n_eval_points if n_points is None: n_points = nengo.utils.builder.default_n_eval_points( ens.n_neurons, ens.dimensions) eval_points = ens.eval_points.sample(n_points, ens.dimensions, self.rng) eval_points *= ens.radius else: if (ens.n_eval_points is not None and ens.eval_points.shape[0] != ens.n_eval_points): warnings.warn("Number of eval_points doesn't match " "n_eval_points. Ignoring n_eval_points.") eval_points = np.array(ens.eval_points, dtype=np.float64) eval_points *= ens.radius J = gain * np.dot(eval_points, encoders.T / ens.radius) + bias activity = self.compute_activity(ens, J) self.model.params[ens] = BuiltEnsemble(intercepts=intercepts, max_rates=max_rates, gain=gain, bias=bias, encoders=encoders, scaled_encoders=scaled_encoders, eval_points=eval_points, activity=activity, neurons=ens.neurons, radius=ens.radius ) self.model.outputs[ens] = np.zeros(ens.n_neurons, dtype=float) self.model.input_filters[ens] = {} self.model.input_filters[ens.neurons] = {}
def create_encoders(neurons, dimensions, mean, variance): encoders = variance * np.random.randn(neurons, dimensions) + mean encoders /= npext.norm(encoders, axis=1, keepdims=True) return encoders
def build_ensemble(model, ens): # Create random number generator rng = np.random.RandomState(model.seeds[ens]) eval_points = gen_eval_points(ens, ens.eval_points, rng=rng) # Set up signal model.sig[ens]['in'] = Signal(np.zeros(ens.dimensions), name="%s.signal" % ens) model.add_op(Reset(model.sig[ens]['in'])) # Set up encoders if isinstance(ens.neuron_type, Direct): encoders = np.identity(ens.dimensions) elif isinstance(ens.encoders, Distribution): encoders = sample(ens.encoders, ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Build the neurons gain, bias, max_rates, intercepts = get_gain_bias(ens, rng) if isinstance(ens.neuron_type, Direct): model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.dimensions), name='%s.neuron_in' % ens) model.sig[ens.neurons]['out'] = model.sig[ens.neurons]['in'] model.add_op(Reset(model.sig[ens.neurons]['in'])) else: model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_in" % ens) model.sig[ens.neurons]['out'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_out" % ens) bias_sig = Signal(bias, name="%s.bias" % ens, readonly=True) model.add_op(Copy(src=bias_sig, dst=model.sig[ens.neurons]['in'])) # This adds the neuron's operator and sets other signals model.build(ens.neuron_type, ens.neurons) # Scale the encoders if isinstance(ens.neuron_type, Direct): scaled_encoders = encoders else: scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] model.sig[ens]['encoders'] = Signal( scaled_encoders, name="%s.scaled_encoders" % ens, readonly=True) # Inject noise if specified if ens.noise is not None: model.build(ens.noise, sig_out=model.sig[ens.neurons]['in'], inc=True) # Create output signal, using built Neurons model.add_op(DotInc( model.sig[ens]['encoders'], model.sig[ens]['in'], model.sig[ens.neurons]['in'], tag="%s encoding" % ens)) # Output is neural output model.sig[ens]['out'] = model.sig[ens.neurons]['out'] model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders, gain=gain, bias=bias)
def test_eval_points(Simulator, nl_nodirect): rng = np.random.RandomState(0) n = 100 d = 5 filter = 0.08 dt = 1e-3 eval_points = np.logspace(np.log10(300), np.log10(5000), 11) eval_points = np.round(eval_points).astype('int') max_points = eval_points.max() n_trials = 1 rmses = np.nan * np.zeros((len(eval_points), n_trials)) for j in range(n_trials): points = rng.normal(size=(max_points, d)) points *= (rng.uniform(size=max_points) / norm(points, axis=-1))[:, None] rng_j = np.random.RandomState(348 + j) seed = 903824 + j # generate random input in unit hypersphere x = rng_j.normal(size=d) x *= rng_j.uniform() / norm(x) for i, n_points in enumerate(eval_points): model = nengo.Network( 'test_eval_points(%d,%d)' % (i, j), seed=seed) with model: u = nengo.Node(output=x) a = nengo.Ensemble(nl_nodirect(n * d), d, eval_points=points[:n_points]) nengo.Connection(u, a, synapse=0) up = nengo.Probe(u) ap = nengo.Probe(a) with Timer() as timer: sim = Simulator(model, dt=dt) sim.run(10 * filter) t = sim.trange() xt = filtfilt(sim.data[up], filter / dt) yt = filtfilt(sim.data[ap], filter / dt) t0 = 5 * filter t1 = 7 * filter tmask = (t > t0) & (t < t1) rmses[i, j] = rms(yt[tmask] - xt[tmask]) print("done %d (%d) in %0.3f s" % (n_points, j, timer.duration)) # subtract out mean for each model rmses_norm = rmses - rmses.mean(0, keepdims=True) with Plotter(Simulator, nl_nodirect) as plt: mean = rmses_norm.mean(1) low = rmses_norm.min(1) high = rmses_norm.max(1) plt.semilogx(eval_points, mean, 'k-') plt.semilogx(eval_points, high, 'r-') plt.semilogx(eval_points, low, 'b-') plt.xlim([eval_points[0], eval_points[-1]]) plt.xticks(eval_points, eval_points) plt.savefig('test_decoders.test_eval_points.pdf') plt.close()
def build_ensemble(self, ens): # Create random number generator seed = self.next_seed() if ens.seed is None else ens.seed rng = np.random.RandomState(seed) # Generate eval points if ens.eval_points is None or is_integer(ens.eval_points): eval_points = self.generate_eval_points( ens=ens, n_points=ens.eval_points, rng=rng) else: eval_points = npext.array( ens.eval_points, dtype=np.float64, min_dims=2) # Set up signal self.model.sig_in[ens] = Signal(np.zeros(ens.dimensions), name="%s.signal" % ens.label) self.model.operators.append(Reset(self.model.sig_in[ens])) # Set up encoders if ens.encoders is None: if isinstance(ens.neurons, nengo.Direct): encoders = np.identity(ens.dimensions) else: sphere = dists.UniformHypersphere(ens.dimensions, surface=True) encoders = sphere.sample(ens.neurons.n_neurons, rng=rng) else: encoders = np.array(ens.encoders, dtype=np.float64) enc_shape = (ens.neurons.n_neurons, ens.dimensions) if encoders.shape != enc_shape: raise ShapeMismatch( "Encoder shape is %s. Should be (n_neurons, dimensions); " "in this case %s." % (encoders.shape, enc_shape)) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Determine max_rates and intercepts if isinstance(ens.max_rates, dists.Distribution): max_rates = ens.max_rates.sample( ens.neurons.n_neurons, rng=rng) else: max_rates = np.array(ens.max_rates) if isinstance(ens.intercepts, dists.Distribution): intercepts = ens.intercepts.sample( ens.neurons.n_neurons, rng=rng) else: intercepts = np.array(ens.intercepts) # Build the neurons if isinstance(ens.neurons, nengo.Direct): bn = self.build(ens.neurons, ens.dimensions) else: bn = self.build(ens.neurons, max_rates, intercepts) # Scale the encoders if isinstance(ens.neurons, nengo.Direct): scaled_encoders = encoders else: scaled_encoders = encoders * (bn.gain / ens.radius)[:, np.newaxis] # Create output signal, using built Neurons self.model.operators.append(DotInc( Signal(scaled_encoders, name="%s.scaled_encoders" % ens.label), self.model.sig_in[ens], self.model.sig_in[ens.neurons], tag="%s encoding" % ens.label)) # Output is neural output self.model.sig_out[ens] = self.model.sig_out[ens.neurons] for probe in ens.probes["decoded_output"]: self.build(probe, dimensions=ens.dimensions) for probe in ens.probes["spikes"] + ens.probes["voltages"]: self.build(probe, dimensions=ens.neurons.n_neurons) return BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders)
def build_ensemble(model, ens): # Create random number generator rng = np.random.RandomState(model.seeds[ens]) eval_points = gen_eval_points(ens, ens.eval_points, rng=rng) # Set up signal model.sig[ens]['in'] = Signal(np.zeros(ens.dimensions), name="%s.signal" % ens) model.add_op(Reset(model.sig[ens]['in'])) # Set up encoders if isinstance(ens.neuron_type, Direct): encoders = np.identity(ens.dimensions) elif isinstance(ens.encoders, Distribution): encoders = sample(ens.encoders, ens.n_neurons, ens.dimensions, rng=rng) else: encoders = npext.array(ens.encoders, min_dims=2, dtype=np.float64) encoders /= npext.norm(encoders, axis=1, keepdims=True) # Build the neurons if ens.gain is not None and ens.bias is not None: gain = sample(ens.gain, ens.n_neurons, rng=rng) bias = sample(ens.bias, ens.n_neurons, rng=rng) max_rates, intercepts = None, None # TODO: determine from gain & bias elif ens.gain is not None or ens.bias is not None: # TODO: handle this instead of error raise NotImplementedError("gain or bias set for %s, but not both. " "Solving for one given the other is not " "implemented yet." % ens) else: max_rates = sample(ens.max_rates, ens.n_neurons, rng=rng) intercepts = sample(ens.intercepts, ens.n_neurons, rng=rng) gain, bias = ens.neuron_type.gain_bias(max_rates, intercepts) if isinstance(ens.neuron_type, Direct): model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.dimensions), name='%s.neuron_in' % ens) model.sig[ens.neurons]['out'] = model.sig[ens.neurons]['in'] model.add_op(Reset(model.sig[ens.neurons]['in'])) else: model.sig[ens.neurons]['in'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_in" % ens) model.sig[ens.neurons]['out'] = Signal( np.zeros(ens.n_neurons), name="%s.neuron_out" % ens) model.add_op(Copy(src=Signal(bias, name="%s.bias" % ens), dst=model.sig[ens.neurons]['in'])) # This adds the neuron's operator and sets other signals model.build(ens.neuron_type, ens.neurons) # Scale the encoders if isinstance(ens.neuron_type, Direct): scaled_encoders = encoders else: scaled_encoders = encoders * (gain / ens.radius)[:, np.newaxis] model.sig[ens]['encoders'] = Signal( scaled_encoders, name="%s.scaled_encoders" % ens) # Inject noise if specified if ens.noise is not None: model.build(ens.noise, sig_out=model.sig[ens.neurons]['in'], inc=True) # Create output signal, using built Neurons model.add_op(DotInc( model.sig[ens]['encoders'], model.sig[ens]['in'], model.sig[ens.neurons]['in'], tag="%s encoding" % ens)) # Output is neural output model.sig[ens]['out'] = model.sig[ens.neurons]['out'] model.params[ens] = BuiltEnsemble(eval_points=eval_points, encoders=encoders, intercepts=intercepts, max_rates=max_rates, scaled_encoders=scaled_encoders, gain=gain, bias=bias)