def test_dtype(Simulator, request, seed, bits):
# Ensure dtype is set back to default after the test, even if it fails
default = nengo.rc.get("precision", "bits")
request.addfinalizer(lambda: nengo.rc.set("precision", "bits", default))
float_dtype = np.dtype(getattr(np, "float%s" % bits))
int_dtype = np.dtype(getattr(np, "int%s" % bits))
with nengo.Network() as model:
nengo_dl.configure_settings(dtype="float%s" % bits)
u = nengo.Node([0.5, -0.4])
a = nengo.Ensemble(10, 2)
nengo.Connection(u, a)
p = nengo.Probe(a)
with Simulator(model) as sim:
sim.step()
# check that the builder has created signals of the correct dtype
# (note that we may not necessarily use that dtype during simulation)
for sig in sim.tensor_graph.signals:
assert sig.dtype in (float_dtype,
int_dtype), ("Signal '%s' wrong dtype" % sig)
objs = (obj for obj in model.all_objects if sim.data[obj] is not None)
for obj in objs:
for x in (x for x in sim.data[obj] if isinstance(x, np.ndarray)):
assert x.dtype == float_dtype, obj
assert sim.data[p].dtype == float_dtype
def build_SNN(image_size, config, spiking=True):
with nengo.Network(seed=config["seed"]) as net:
# remove some unnecessary features to speed up the training
nengo_dl.configure_settings(stateful=False)
# input node
inp = nengo.Node(np.zeros(image_size[-1]))
out = nengo.Node(size_in=101)
if spiking:
# lmu cell
lmu = LMUCellSpike(units=800,
order=2,
theta=image_size[1],
input_d=image_size[-1])
conn = nengo.Connection(inp, lmu.x, synapse=None)
else:
lmu = LMUCell(units=800,
order=2,
theta=image_size[1],
input_d=image_size[-1])
conn = nengo.Connection(inp, lmu.x, synapse=None)
# dense linear readout
nengo.Connection(lmu.h,
out,
transform=nengo_dl.dists.Glorot(),
synapse=None)
p = nengo.Probe(out)
return net
def test_reuse_vars(Simulator):
def my_func(_, x):
# note: the control dependencies thing is due to some weird tensorflow
# issue with creating variables inside while loops
with tf.control_dependencies(None):
w = tf.get_variable("weights", initializer=tf.constant(2.0))
return x * tf.cast(w, x.dtype)
with nengo.Network() as net:
configure_settings(trainable=False)
inp = nengo.Node([1])
node = TensorNode(my_func, size_in=1)
p = nengo.Probe(node)
nengo.Connection(inp, node, synapse=None)
with Simulator(net, unroll_simulation=5) as sim:
sim.run_steps(5)
assert np.allclose(sim.data[p], 2)
with sim.tensor_graph.graph.as_default():
vars = tf.trainable_variables()
assert len(vars) == 1
assert vars[0].get_shape() == ()
assert sim.sess.run(vars[0]) == 2
def gen_init(self):
model = nengo.Network()
with model:
model.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
model.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
nengo_dl.configure_settings(trainable=False)
return model
def _build(self, num_classes, num_layers, num_filters, kernel_sizes):
channel_each_layer = ([1] + num_filters)
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude = 0.01)
# init state, make neural trainable
nengo_dl.configure_settings(trainable = True)
inp = nengo.Node([0] * self.input_size[0] * self.input_size[1])
x = nengo_dl.tensor_layer(inp, tf.layers.conv2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[0]),
filters = num_filters[0], kernel_size = kernel_sizes[0], padding = 'same')
x = nengo_dl.tensor_layer(x, neuron_type)
for i in range(1, num_layers):
x = nengo_dl.tensor_layer(x, tf.layers.conv2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[i]),
filters = num_filters[i], kernel_size = kernel_sizes[i], padding = 'same')
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[-1]),
pool_size = (self.input_size[0], self.input_size[1]),
strides = (self.input_size[0], self.input_size[1]))
x = nengo_dl.tensor_layer(x, tf.layers.dense, units = num_classes)
out_p = nengo.Probe(x)
out_p_filt = nengo.Probe(x, synapse = self.synapse)
return inp, out_p, out_p_filt, net
def test_training_arg(Simulator):
class TrainingLayer(tf.keras.layers.Layer):
def __init__(self, expected):
super().__init__()
self.expected = expected
def call(self, inputs, training=None):
tf.assert_equal(training, self.expected)
return tf.reshape(inputs, (1, 1))
with nengo.Network() as net:
node = TensorNode(TrainingLayer(expected=False),
shape_in=None,
shape_out=(1, ))
nengo.Probe(node)
with Simulator(net) as sim:
sim.predict(n_steps=10)
with Simulator(net) as sim:
sim.compile(optimizer=tf.optimizers.SGD(0), loss=tf.losses.mse)
node.tensor_func.expected = True
sim.fit(n_steps=10, y=np.zeros((1, 1, 1)))
with net:
configure_settings(learning_phase=True)
with Simulator(net) as sim:
sim.predict(n_steps=10)
def test_nengo_dl_noise(neuron_type, seed, plt, allclose):
pytest.importorskip("tensorflow")
install_dl_builders()
net, rates, _ = rate_nengo_dl_net(neuron_type)
n_noise = 1000 # number of noise samples per x point
with net:
nengo_dl.configure_settings(learning_phase=True) # run with `training=True`
with nengo_dl.Simulator(net, dt=net.dt, minibatch_size=n_noise, seed=seed) as sim:
input_data = {net.stim: np.tile(net.x[None, None, :], (n_noise, 1, 1))}
sim.step(data=input_data)
y = sim.data[net.probe][:, 0, :]
ymean = y.mean(axis=0)
y25 = np.percentile(y, 25, axis=0)
y75 = np.percentile(y, 75, axis=0)
dy25 = y25 - rates["ref"]
dy75 = y75 - rates["ref"]
# exponential models roughly fitted to 25/75th percentiles
x1mask = net.x > 1.5
x1 = net.x[x1mask]
if isinstance(neuron_type.nengo_dl_noise, AlphaRCNoise):
exp_model = 0.7 + 2.8 * np.exp(-0.22 * (x1 - 1))
atol = 0.12 * exp_model.max()
elif isinstance(neuron_type.nengo_dl_noise, LowpassRCNoise):
exp_model = 1.5 + 2.2 * np.exp(-0.22 * (x1 - 1))
atol = 0.2 * exp_model.max()
rtol = 0.2
mu_atol = 0.6 # depends on n_noise and variance of noise
# --- plots
plt.subplot(211)
plt.plot(net.x, rates["med"], "--", label="LIF(tau_ref += 0.5*dt)")
plt.plot(net.x, ymean, label="nengo_dl")
plt.plot(net.x, y25, ":", label="25th")
plt.plot(net.x, y75, ":", label="75th")
plt.plot(net.x, rates["ref"], "k--", label="LoihiLIF")
plt.legend()
plt.subplot(212)
plt.plot(net.x, ymean - rates["ref"], "b", label="mean")
plt.plot(net.x, mu_atol * np.ones_like(net.x), "b:")
plt.plot(net.x, -mu_atol * np.ones_like(net.x), "b:")
plt.plot(net.x, y25 - rates["ref"], ":", label="25th")
plt.plot(net.x, y75 - rates["ref"], ":", label="75th")
plt.plot(x1, exp_model, "k--")
plt.plot(x1, exp_model * (1 + rtol) + atol, "k:")
plt.plot(x1, exp_model * (1 - rtol) - atol, "k:")
plt.plot(x1, -exp_model, "k--")
plt.legend()
assert ymean.shape == rates["ref"].shape
assert allclose(ymean, rates["ref"], atol=mu_atol, record_rmse=False)
assert allclose(dy25[x1mask], -exp_model, atol=atol, rtol=rtol, record_rmse=False)
assert allclose(dy75[x1mask], exp_model, atol=atol, rtol=rtol, record_rmse=False)
def test_save_load_params(Simulator, tmpdir):
with nengo.Network(seed=0) as net:
inp = nengo.Node([0])
out = nengo.Node(size_in=1)
ens = nengo.Ensemble(10, 1)
nengo.Connection(inp, ens)
nengo.Connection(ens, out)
configure_settings(trainable=None)
net.config[ens].trainable = False
with Simulator(net) as sim:
weights_var = [
x[0] for x in sim.tensor_graph.base_vars.values()
if x[0].get_shape() == (1, 10)
][0]
enc_var = sim.tensor_graph.base_vars[sim.tensor_graph.sig_map[
sim.model.sig[ens]["encoders"]].key][0]
weights0, enc0 = sim.sess.run([weights_var, enc_var])
sim.save_params(os.path.join(str(tmpdir), "train"))
sim.save_params(os.path.join(str(tmpdir), "local"), include_local=True)
with pytest.raises(SimulatorClosed):
sim.save_params(None)
with pytest.raises(SimulatorClosed):
sim.load_params(None)
with nengo.Network(seed=1) as net2:
inp = nengo.Node([0])
out = nengo.Node(size_in=1)
ens = nengo.Ensemble(10, 1)
nengo.Connection(inp, ens)
nengo.Connection(ens, out)
configure_settings(trainable=None)
net2.config[ens].trainable = False
with Simulator(net2) as sim:
weights_var = [
x[0] for x in sim.tensor_graph.base_vars.values()
if x[0].get_shape() == (1, 10)
][0]
enc_var = sim.tensor_graph.base_vars[sim.tensor_graph.sig_map[
sim.model.sig[ens]["encoders"]].key][0]
weights1, enc1 = sim.sess.run([weights_var, enc_var])
assert not np.allclose(weights0, weights1)
assert not np.allclose(enc0, enc1)
sim.load_params(os.path.join(str(tmpdir), "train"))
weights2, enc2 = sim.sess.run([weights_var, enc_var])
assert np.allclose(weights0, weights2)
assert not np.allclose(enc0, enc2)
sim.load_params(os.path.join(str(tmpdir), "local"), include_local=True)
weights3, enc3 = sim.sess.run([weights_var, enc_var])
assert np.allclose(weights0, weights3)
assert np.allclose(enc0, enc3)
def test_merged_learning(Simulator, rule, weights, seed):
# a slightly more complicated network with mergeable learning rules, to
# make sure that works OK
dimensions = 2
with nengo.Network(seed=seed) as net:
configure_settings(planner=partial(graph_optimizer.tree_planner,
max_depth=10),
dtype="float64")
a = nengo.Ensemble(3, dimensions, label="a")
b = nengo.Ensemble(3, dimensions, label="b")
c = nengo.Ensemble(5, dimensions, label="c")
# for PES rules the post (error) shape also has to match for the rules
# to be mergeable
d = nengo.Ensemble(5 if rule == nengo.PES else 10,
dimensions,
label="d")
conn0 = nengo.Connection(
a,
c,
learning_rule_type=rule(learning_rate=0.1),
solver=nengo.solvers.LstsqL2(weights=weights),
)
conn1 = nengo.Connection(
b,
d,
learning_rule_type=rule(learning_rate=0.2),
solver=nengo.solvers.LstsqL2(weights=weights),
)
p0 = nengo.Probe(conn0.learning_rule, "delta")
p1 = nengo.Probe(conn1.learning_rule, "delta")
with nengo.Simulator(net) as sim:
sim.run_steps(10)
canonical = (sim.data[p0], sim.data[p1])
with Simulator(net, minibatch_size=2) as sim:
build_type = {
nengo.Voja: SimVoja,
nengo.Oja: SimOja,
nengo.BCM: SimBCM,
nengo.PES: SimPES,
}
assert (len([
x for x in sim.tensor_graph.plan if type(x[0]) == build_type[rule]
]) == 1)
sim.run_steps(10)
for i in range(sim.minibatch_size):
assert np.allclose(sim.data[p0][i], canonical[0])
assert np.allclose(sim.data[p1][i], canonical[1])
def run_snn(model,
x_test,
y_test,
params_load_path,
iteration,
timesteps=50,
scale_firing_rates=1000,
synapse=0.01,
batch_size=16):
"""
Run model in spiking setting
:param batch_size: batch size
:param model: model reference
:param x_test: testing features
:param y_test: testing labels
:param params_load_path: path to load parameters
:param iteration: number of current iteration
:param timesteps: number of timesteps
:param scale_firing_rates: firing rate scaling
:param synapse: synaptic smoothing
:return: accuracy, precision, recall, f1 and confusion matrix from the testing data
"""
converter = nengo_dl.Converter(
model,
swap_activations={tf.nn.relu: nengo.SpikingRectifiedLinear()},
scale_firing_rates=scale_firing_rates,
synapse=synapse
) # create a Nengo converter object and swap all relu activations with spiking relu
with converter.net:
nengo_dl.configure_settings(stateful=False)
output_layer = converter.outputs[model.get_layer(
'output_layer')] # output layer for simulator
x_test_tiled = np.tile(x_test,
(1, timesteps, 1)) # tile test data to timesteps
with nengo_dl.Simulator(converter.net) as simulator:
simulator.load_params(params_load_path)
# Get the statistics
accuracy, precision, recall, f1, confusion_matrix = get_metrics(
simulator, output_layer, x_test_tiled, y_test, batch_size,
f'{iteration}. CNN (SNN conversion)')
return {
'accuracy': accuracy,
'precision': precision,
'recall': recall,
'f1': f1,
'confusion_matrix': confusion_matrix
}
def run_profile(net, train=False, n_steps=150, do_profile=True, **kwargs):
"""
Run profiler on a benchmark network.
Parameters
----------
net : :class:`~nengo:nengo.Network`
The nengo Network to be profiled.
train : bool, optional
If True, profile the ``sim.train`` function. Otherwise, profile the
``sim.run`` function.
n_steps : int, optional
The number of timesteps to run the simulation.
do_profile : bool, optional
Whether or not to run profiling
Notes
-----
kwargs will be passed on to :class:`.Simulator`
"""
with net:
nengo_dl.configure_settings(trainable=None if train else False)
with nengo_dl.Simulator(net, **kwargs) as sim:
# note: we run a few times to try to eliminate startup overhead (only
# the data from the last run will be kept)
if train:
opt = tf.train.GradientDescentOptimizer(0.001)
x = np.random.randn(sim.minibatch_size, n_steps, net.inp.size_out)
y = np.random.randn(sim.minibatch_size, n_steps, net.p.size_in)
for _ in range(2):
sim.train({net.inp: x}, {net.p: y},
optimizer=opt,
n_epochs=1,
profile=do_profile)
start = time.time()
sim.train({net.inp: x}, {net.p: y},
optimizer=opt,
n_epochs=1,
profile=do_profile)
print("Execution time:", time.time() - start)
else:
for _ in range(2):
sim.run_steps(n_steps, profile=do_profile)
start = time.time()
sim.run_steps(n_steps, profile=do_profile)
print("Execution time:", time.time() - start)
def run_profile(net, train=False, n_steps=150, do_profile=True, **kwargs):
"""
Run profiler on a benchmark network.
Parameters
----------
net : `~nengo.Network`
The nengo Network to be profiled.
train : bool
If True, profile the ``sim.train`` function. Otherwise, profile the
``sim.run`` function.
n_steps : int
The number of timesteps to run the simulation.
do_profile : bool
Whether or not to run profiling
Notes
-----
kwargs will be passed on to `.Simulator`
"""
with net:
nengo_dl.configure_settings(inference_only=not train)
with nengo_dl.Simulator(net, **kwargs) as sim:
# note: we run a few times to try to eliminate startup overhead (only
# the data from the last run will be kept)
if train:
opt = tf.train.GradientDescentOptimizer(0.001)
x = np.random.randn(sim.minibatch_size, n_steps, net.inp.size_out)
y = np.random.randn(sim.minibatch_size, n_steps, net.p.size_in)
for _ in range(2):
sim.train({net.inp: x, net.p: y}, optimizer=opt, n_epochs=1,
profile=do_profile)
start = time.time()
sim.train({net.inp: x, net.p: y}, optimizer=opt, n_epochs=1,
profile=do_profile)
exec_time = time.time() - start
print("Execution time:", exec_time)
else:
for _ in range(2):
sim.run_steps(n_steps, profile=do_profile)
start = time.time()
sim.run_steps(n_steps, profile=do_profile)
exec_time = time.time() - start
print("Execution time:", exec_time)
return exec_time
def test_reuse_vars(Simulator, pytestconfig):
class MyLayer(tf.keras.layers.Layer):
def build(self, input_shape):
self.w = self.add_weight(initializer=tf.initializers.constant(2.0),
name="weights")
def call(self, x):
return x * tf.cast(self.w, x.dtype)
with nengo.Network() as net:
configure_settings(trainable=False)
inp = nengo.Node([1])
node = TensorNode(MyLayer(), shape_in=(1, ), pass_time=False)
nengo.Connection(inp, node, synapse=None)
node2 = Layer(
tf.keras.layers.Dense(
units=10,
use_bias=False,
kernel_initializer=tf.initializers.constant(3),
dtype=pytestconfig.getoption("--dtype"),
))(inp)
p = nengo.Probe(node)
p2 = nengo.Probe(node2)
with Simulator(net, unroll_simulation=5) as sim:
sim.run_steps(5)
assert np.allclose(sim.data[p], 2)
assert np.allclose(sim.data[p2], 3)
# note: when inference-only=True the weights will be marked as non-trainable
if sim.tensor_graph.inference_only:
assert len(sim.tensor_graph.saved_state) == 2
assert len(sim.keras_model.non_trainable_variables) == 2
assert len(sim.keras_model.trainable_variables) == 0
vars = sim.keras_model.non_trainable_variables
else:
assert len(sim.tensor_graph.saved_state) == 2
assert len(sim.keras_model.non_trainable_variables) == 0
assert len(sim.keras_model.trainable_variables) == 2
vars = sim.keras_model.trainable_variables
assert len(vars) == 2
assert vars[0].shape == ()
assert tf.keras.backend.get_value(vars[0]) == 2
assert vars[1].shape == (1, 10)
assert np.allclose(tf.keras.backend.get_value(vars[1]), 3)
def test_merged_learning(Simulator, rule, weights, seed):
# a slightly more complicated network with mergeable learning rules, to
# make sure that works OK
dimensions = 2
with nengo.Network(seed=seed) as net:
configure_settings(
planner=partial(graph_optimizer.tree_planner, max_depth=10))
a = nengo.Ensemble(3, dimensions, label="a")
b = nengo.Ensemble(3, dimensions, label="b")
c = nengo.Ensemble(5, dimensions, label="c")
# for PES rules the post (error) shape also has to match for the rules
# to be mergeable
d = nengo.Ensemble(5 if rule == nengo.PES else 10, dimensions,
label="d")
conn0 = nengo.Connection(
a, c, learning_rule_type=rule(),
solver=nengo.solvers.LstsqL2(weights=weights))
conn1 = nengo.Connection(
b, d, learning_rule_type=rule(),
solver=nengo.solvers.LstsqL2(weights=weights))
p0 = nengo.Probe(conn0.learning_rule, "delta")
p1 = nengo.Probe(conn1.learning_rule, "delta")
with nengo.Simulator(net) as sim:
sim.run_steps(10)
canonical = (sim.data[p0], sim.data[p1])
with Simulator(net) as sim:
build_type = {nengo.Voja: SimVoja, nengo.Oja: SimOja,
nengo.BCM: SimBCM, nengo.PES: SimPES}
assert len([x for x in sim.tensor_graph.plan
if type(x[0]) == build_type[rule]]) == 1
sim.run_steps(10)
assert np.allclose(sim.data[p0], canonical[0])
assert np.allclose(sim.data[p1], canonical[1])
def build_network(neuron_type, ens_params):
with nengo.Network() as net:
nengo_dl.configure_settings(trainable=False)
inp = nengo.Node([0] * 28 * 28)
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 64),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(10, 10, 128),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
return net, inp, x
def test_nengo_dl_neurons(neuron_type, inference_only, Simulator, plt, allclose):
install_dl_builders()
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
nengo_dl.configure_settings(inference_only=inference_only)
a = nengo.Ensemble(
n, 1, neuron_type=neuron_type, encoders=encoders, gain=gain, bias=bias
)
ap = nengo.Probe(a.neurons)
t_final = 1.0
with nengo_dl.Simulator(model, dt=dt) as dl_sim:
dl_sim.run(t_final)
with Simulator(model, dt=dt) as loihi_sim:
loihi_sim.run(t_final)
rates_dlsim = (dl_sim.data[ap] > 0).sum(axis=0) / t_final
rates_loihisim = (loihi_sim.data[ap] > 0).sum(axis=0) / t_final
zeros = np.zeros((1, gain.size))
rates_ref = neuron_type.rates(zeros, gain, bias, dt=dt).squeeze(axis=0)
plt.plot(bias, rates_loihisim, "r", label="loihi sim")
plt.plot(bias, rates_dlsim, "b-.", label="dl sim")
plt.plot(bias, rates_ref, "k--", label="rates_ref")
plt.legend(loc="best")
atol = 1.0 / t_final # the fundamental unit for our rates
assert rates_ref.shape == rates_dlsim.shape == rates_loihisim.shape
assert allclose(rates_dlsim, rates_ref, atol=atol, rtol=0, xtol=1)
assert allclose(rates_loihisim, rates_ref, atol=atol, rtol=0, xtol=1)
def linear_net():
"""
A simple network with an input, output, and no nonlinearity.
"""
with nengo.Network() as net:
a = nengo.Node([1])
# note: in theory this would be nengo.Node(size_in=1), but due to
# https://github.com/tensorflow/tensorflow/issues/23383
# TensorFlow will hang
b = nengo.Ensemble(1, 1, neuron_type=nengo.RectifiedLinear(),
gain=np.ones(1), bias=np.ones(1) * 1e-6)
configure_settings(trainable=None)
net.config[b.neurons].trainable = False
nengo.Connection(a, b.neurons, synapse=None)
p = nengo.Probe(b.neurons)
return net, a, p
def build_SNN_simple(image_size, config):
with nengo.Network(seed=config["seed"]) as net:
# remove some unnecessary features to speed up the training
nengo_dl.configure_settings(stateful=False)
n_ensembles = 10000
# input node
inp = nengo.Node(np.zeros(image_size[-1])) #
u = nengo.networks.EnsembleArray(
n_neurons=50,
n_ensembles=n_ensembles,
neuron_type=nengo.SpikingRectifiedLinear(),
)
nengo.Connection(inp, u.input, transform=np.zeros((n_ensembles, 512)))
out = nengo.Node(size_in=101)
nengo.Connection(u.output,
out,
transform=nengo_dl.dists.Glorot(),
synapse=None)
p = nengo.Probe(out)
return net
def linear_net():
"""
A simple network with an input, output, and no nonlinearity.
"""
with nengo.Network() as net:
a = nengo.Node([1])
# note: in theory this would be nengo.Node(size_in=1), but due to
# https://github.com/tensorflow/tensorflow/issues/23383
# TensorFlow will hang
b = nengo.Ensemble(1,
1,
neuron_type=nengo.RectifiedLinear(),
gain=np.ones(1),
bias=np.ones(1) * 1e-6)
configure_settings(trainable=None)
net.config[b.neurons].trainable = False
nengo.Connection(a, b.neurons, synapse=None)
p = nengo.Probe(b.neurons)
return net, a, p
def build_spaun(dimensions):
vocab.sp_dim = dimensions
cfg.mtr_arm_type = None
cfg.set_seed(1)
experiment.initialize('A', vis_data.get_image_ind,
vis_data.get_image_label,
cfg.mtr_est_digit_response_time, cfg.rng)
vocab.initialize(experiment.num_learn_actions, cfg.rng)
vocab.initialize_mtr_vocab(mtr_data.dimensions, mtr_data.sps)
vocab.initialize_vis_vocab(vis_data.dimensions, vis_data.sps)
with Spaun() as net:
nengo_dl.configure_settings(
trainable=False,
simplifications=[
graph_optimizer.remove_constant_copies,
graph_optimizer.remove_unmodified_resets,
# graph_optimizer.remove_zero_incs,
graph_optimizer.remove_identity_muls
])
return net
def test_reuse_vars(Simulator):
def my_func(_, x):
# note: the control dependencies thing is due to some weird tensorflow
# issue with creating variables inside while loops
with tf.control_dependencies(None):
w = tf.get_variable("weights", initializer=tf.constant(2.0))
return x * tf.cast(w, x.dtype)
with nengo.Network() as net:
configure_settings(trainable=False)
inp = nengo.Node([1])
node = TensorNode(my_func, size_in=1)
node2 = TensorNode(
lambda _, x: tf.layers.dense(
x, units=10, use_bias=False,
kernel_initializer=tf.constant_initializer(3)),
size_in=1, size_out=10)
p = nengo.Probe(node)
p2 = nengo.Probe(node2)
nengo.Connection(inp, node, synapse=None)
nengo.Connection(inp, node2, synapse=None)
with Simulator(net, unroll_simulation=5) as sim:
sim.run_steps(5)
assert np.allclose(sim.data[p], 2)
assert np.allclose(sim.data[p2], 3)
with sim.tensor_graph.graph.as_default():
vars = tf.trainable_variables()
assert len(vars) == 2
assert vars[0].get_shape() == ()
assert sim.sess.run(vars[0]) == 2
assert vars[1].get_shape() == (1, 10)
assert np.allclose(sim.sess.run(vars[1]), 3)
def __init__(self,
param_file,
dim=256,
maze_id_dim=256,
n_sensors=36,
hidden_size=1024,
net_seed=13,
n_steps=30):
self.net = nengo.Network(seed=net_seed)
with self.net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
# net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
# net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
self.net.config[nengo.Connection].synapse = None
neuron_type = nengo.LIF(amplitude=0.01)
# this is an optimization to improve the training speed,
# since we won't require stateful behaviour in this example
nengo_dl.configure_settings(stateful=False)
# the input node that will be used to feed in (context, location, goal)
inp = nengo.Node(np.zeros((n_sensors * 4 + maze_id_dim, )))
x = nengo_dl.Layer(tf.keras.layers.Dense(units=hidden_size))(inp)
x = nengo_dl.Layer(neuron_type)(x)
out = nengo_dl.Layer(tf.keras.layers.Dense(units=dim))(x)
self.out_p = nengo.Probe(out, label="out_p")
self.out_p_filt = nengo.Probe(out, synapse=0.1, label="out_p_filt")
self.sim = nengo_dl.Simulator(self.net, minibatch_size=1)
self.sim.load_params(param_file)
self.sim.compile(loss={self.out_p_filt: mse_loss})
self.n_steps = n_steps
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
inp = nengo.Node([0] * 28 * 28)
# add the first convolutional layer
x = nengo_dl.tensor_layer(
inp, tf.layers.conv2d, shape_in=(28, 28, 1), filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(
x, tf.layers.conv2d, shape_in=(26, 26, 32),
def _build_net(self):
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
self.inp = nengo.Node([0] * 28 * 28)
# ENCODER
# add the first convolutional layer
x = nengo_dl.tensor_layer(self.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# add a pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(24, 24, 64),
filters=64,
kernel_size=2,
strides=2)
# x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d, shape_in=(24, 24, 64), pool_size=2, strides=2)
# another convolutional layer
# (W - Fw + 2P) / Sw + 1
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# another pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(10, 10, 128),
filters=128,
kernel_size=2,
strides=2)
# x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d, shape_in=(10, 10, 128), pool_size=2, strides=2)
# latent
self.latent = x
# DECODER
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(5, 5, 128),
filters=64,
kernel_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(12, 12, 64),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(24, 24, 64),
filters=32,
kernel_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(26, 26, 32),
filters=1,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(self.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=1,
kernel_size=3)
# linear readout
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
x = nengo_dl.tensor_layer(x, tf.identity)
# x = x + out_stim
self.out_stim = nengo.Node([0] * 10)
self.stim_conns = attach_stim(self.out_stim, x,
(np.arange(10), np.arange(15)))
# we'll create two different output probes, one with a filter
# (for when we're simulating the network over time and
# accumulating spikes), and one without (for when we're
# training the network using a rate-based approximation)
self.out_p = nengo.Probe(x)
self.out_p_filt = nengo.Probe(x, synapse=0.1)
return net
def LearningModel(neurons, dimensions, learning_rule, function_to_learn,
convolve, seed):
global decoded
with nengo.Network() as model:
nengo_dl.configure_settings(stateful=False)
model.inp = nengo.Node(
# WhiteNoise( dist=Gaussian( 0, 0.05 ), seed=seed ),
WhiteSignal(sim_time, high=5, seed=seed),
size_out=dimensions[0])
model.pre = nengo.Ensemble(neurons[0],
dimensions=dimensions[0],
seed=seed)
model.post = nengo.Ensemble(neurons[1],
dimensions=dimensions[1],
seed=seed)
model.ground_truth = nengo.Ensemble(neurons[2],
dimensions=dimensions[2],
seed=seed)
nengo.Connection(model.inp, model.pre)
if convolve:
model.conv = nengo.networks.CircularConvolution(neurons[4],
dimensions[4],
seed=seed)
nengo.Connection(model.inp[:int(dimensions[0] / 2)],
model.conv.input_a,
synapse=None)
nengo.Connection(model.inp[int(dimensions[0] / 2):],
model.conv.input_b,
synapse=None)
nengo.Connection(model.conv.output,
model.ground_truth,
synapse=None)
else:
nengo.Connection(model.inp,
model.ground_truth,
function=function_to_learn,
synapse=None)
if learning_rule:
model.error = nengo.Ensemble(neurons[3],
dimensions=dimensions[3],
seed=seed)
if isinstance(learning_rule,
mPES) or (isinstance(learning_rule, PES)
and not decoded):
model.conn = nengo.Connection(model.pre.neurons,
model.post.neurons,
transform=np.random.random(
(model.post.n_neurons,
model.pre.n_neurons)),
learning_rule_type=learning_rule)
else:
model.conn = nengo.Connection(
model.pre,
model.post,
function=lambda x: np.random.random(dimensions[1]),
learning_rule_type=learning_rule)
nengo.Connection(model.error, model.conn.learning_rule)
nengo.Connection(model.post, model.error)
nengo.Connection(model.ground_truth, model.error, transform=-1)
class cyclic_inhibit:
def __init__(self, cycle_time):
self.out_inhibit = 0.0
self.cycle_time = cycle_time
def step(self, t):
if t % self.cycle_time == 0:
if self.out_inhibit == 0.0:
self.out_inhibit = 2.0
else:
self.out_inhibit = 0.0
return self.out_inhibit
model.inhib = nengo.Node(cyclic_inhibit(learn_block_time).step)
nengo.Connection(model.inhib,
model.error.neurons,
transform=[[-1]] * model.error.n_neurons)
else:
model.conn = nengo.Connection(model.pre,
model.post,
function=function_to_learn)
# -- probes
model.pre_probe = nengo.Probe(model.pre, synapse=0.01)
model.post_probe = nengo.Probe(model.post, synapse=0.01)
model.ground_truth_probe = nengo.Probe(model.ground_truth,
synapse=0.01)
# function_learning_model.error_probe = nengo.Probe( function_learning_model.error, synapse=0.03 )
return model
# )
# print("\nData Generation Complete\n")
with nengo.Network(seed=args.net_seed) as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
# net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
# net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
net.config[nengo.Connection].synapse = None
net.config[nengo.Connection].transform = nengo_dl.dists.Glorot()
neuron_type = nengo.LIF(amplitude=0.01)
# this is an optimization to improve the training speed,
# since we won't require stateful behaviour in this example
nengo_dl.configure_settings(stateful=False)
# the input node that will be used to feed in (context, location, goal)
inp = nengo.Node(np.zeros((args.dim * 2 + args.maze_id_dim, )))
if '.pkl' in args.param_file:
print("Loading values from pickle file")
policy_params = pickle.load(open(args.param_file, 'rb'))
policy_inp_params = policy_params[0]
policy_ens_params = policy_params[1]
policy_out_params = policy_params[2]
hidden_ens = nengo.Ensemble(
n_neurons=args.hidden_size,
dimensions=1,
def compare_backends(ctx, batch, n_neurons):
load = ctx.obj["load"]
reps = ctx.obj["reps"]
device = ctx.obj["device"]
save = ctx.obj["save"]
bench_names = ["integrator", "cconv", "basal_ganglia", "pes"]
n_range = [n_neurons]
d_range = [64, 128, 192]
neuron_types = [nengo.RectifiedLinear()]
backends = ["nengo_dl", "nengo_ocl", "nengo"]
sim_time = 5.0
params = list(
itertools.product(bench_names, n_range, d_range, neuron_types,
backends))
if load:
with open("compare_backends_%d_data_saved.pkl" % batch, "rb") as f:
results = pickle.load(f)
else:
results = [{
"times": [],
"benchmark": bench,
"n_neurons": n_neurons,
"dimensions": dimensions,
"neuron_type": neuron_type,
"backend": backend
} for bench, n_neurons, dimensions, neuron_type, backend in params]
if reps > 0:
for i, (bench, neurons_per_ens, dimensions, neuron_type,
backend) in enumerate(params):
print("%d/%d: %s %s %s %s %s" %
(i + 1, len(params), bench, neurons_per_ens, dimensions,
neuron_type, backend))
net = getattr(benchmarks,
bench)(dimensions=dimensions,
neurons_per_d=neurons_per_ens // dimensions,
neuron_type=neuron_type)
with net:
nengo_dl.configure_settings(inference_only=True)
if "nengo_dl" in backend:
sim = nengo_dl.Simulator(net,
unroll_simulation=25,
minibatch_size=batch,
device=device,
progress_bar=False)
elif backend == "nengo":
sim = nengo.Simulator(net, progress_bar=False, optimize=True)
elif backend == "nengo_ocl":
import nengo_ocl
sim = nengo_ocl.Simulator(net, progress_bar=False)
with sim:
# run once to eliminate startup overhead
sim.run(0.1, progress_bar=False)
for _ in range(reps):
start = time.time()
for b in range(1 if "nengo_dl" in backend else batch):
if b > 0:
sim.reset()
sim.run(sim_time, progress_bar=False)
results[i]["times"].append(
(time.time() - start) / sim_time)
print(" ", min(results[i]["times"]), max(results[i]["times"]),
np.mean(results[i]["times"]))
with open("compare_backends_%d_data.pkl" % batch, "wb") as f:
pickle.dump(results, f)
# plotting
subplots = int(np.ceil(np.sqrt(len(bench_names))))
f, axes = plt.subplots(subplots,
subplots,
sharey=True,
sharex=False,
figsize=(5 * subplots, 5 * subplots),
gridspec_kw={
"hspace": 0.2,
"top": 0.95,
"bottom": 0.05,
"left": 0.07,
"right": 0.95
})
n_bars = len(d_range)
neuron_type = nengo.RectifiedLinear()
colours = plt.rcParams["axes.prop_cycle"].by_key()["color"]
y_max = 2.5 * batch
for k, m in enumerate(bench_names):
subplot_idx = (k // subplots, k % subplots)
x_pos = np.arange(n_bars)
for j, b in enumerate(backends):
bottoms = np.zeros(n_bars)
c = 0
for n in n_range:
data = np.asarray([
bootstrap_ci(t)
for t in filter_results(results,
benchmark=m,
neuron_type=neuron_type,
n_neurons=n,
backend=b)
])
axes[subplot_idx].bar(
x_pos,
data[:, 0],
yerr=abs(np.transpose(data[:, 1:] - data[:, [0]])),
width=0.5,
bottom=bottoms,
color=colours[(j + 1) % len(backends)])
for i, d in enumerate(data[:, 0]):
if d > y_max:
axes[subplot_idx].annotate("%.1f" % d,
(x_pos[i], y_max * 0.9),
ha="center",
va="center",
rotation="vertical",
color="white")
bottoms += data[:, 0]
c += 1
x_pos += n_bars + 1
axes[subplot_idx].set_title("%s" % m)
if k == 0 and len(n_range) > 1:
axes[subplot_idx].legend(["N=%d" % n for n in n_range])
axes[subplot_idx].set_xticks(
np.concatenate([
np.arange(i * (n_bars + 1),
i * (n_bars + 1) + n_bars)
for i in range(len(backends))
]))
axes[subplot_idx].set_xticklabels(
[t for _ in range(len(backends)) for t in d_range])
for i, b in enumerate(backends):
axes[subplot_idx].annotate(
b, (((n_bars - 1) / 2 + (n_bars + 1) * i + 1) /
((n_bars + 1) * len(backends)), -0.1),
xycoords="axes fraction",
ha="center")
axes[subplot_idx].set_ylim([0, y_max])
axes[subplot_idx].set_xlim([-1, (n_bars + 1) * len(backends) - 1])
if k % subplots == 0:
axes[subplot_idx].set_ylabel("real time / simulated time")
if save:
plt.savefig("compare_backends_%d.%s" % (batch, save))
def spiking_mnist(ctx, n_epochs):
load = ctx.obj["load"]
reps = ctx.obj["reps"]
neuron_type = nengo.LIF(amplitude=0.01)
ens_params = dict(max_rates=nengo.dists.Choice([100]),
intercepts=nengo.dists.Choice([0]))
minibatch_size = 200
n_steps = 50
with nengo.Network() as net:
nengo_dl.configure_settings(trainable=False)
inp = nengo.Node([0] * 28 * 28)
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 64),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(10, 10, 128),
pool_size=2,
strides=2)
out = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
out_p = nengo.Probe(out)
spk_out_p = nengo.Probe(out, synapse=0.1)
if load:
with open("spiking_mnist_data_saved.pkl", "rb") as f:
results = pickle.load(f)
else:
results = {"pre": [], "post": [], "spiking": []}
urlretrieve("http://deeplearning.net/data/mnist/mnist.pkl.gz",
"mnist.pkl.gz")
with gzip.open("mnist.pkl.gz") as f:
train_data, _, test_data = pickle.load(f, encoding="latin1")
train_data = list(train_data)
test_data = list(test_data)
for data in (train_data, test_data):
one_hot = np.zeros((data[0].shape[0], 10))
one_hot[np.arange(data[0].shape[0]), data[1]] = 1
data[1] = one_hot
train_data = {
inp: train_data[0][:, None, :],
out_p: train_data[1][:, None, :]
}
test_data = {
inp: test_data[0][:, None, :],
out_p: test_data[1][:, None, :]
}
test_data_time = {
inp: np.tile(test_data[inp], (1, n_steps, 1)),
spk_out_p: np.tile(test_data[out_p], (1, n_steps, 1))
}
for _ in range(reps):
# construct the simulator
with nengo_dl.Simulator(net, minibatch_size=minibatch_size) as sim:
def objective(x, y):
return tf.nn.softmax_cross_entropy_with_logits_v2(logits=x,
labels=y)
opt = tf.train.RMSPropOptimizer(learning_rate=0.001)
def classification_error(outputs, targets):
return 100 * tf.reduce_mean(
tf.cast(
tf.not_equal(tf.argmax(outputs[:, -1], axis=-1),
tf.argmax(targets[:, -1], axis=-1)),
tf.float32))
# collect error before training
results["pre"].append(
sim.loss(test_data, {out_p: classification_error},
training=True))
print("error before training: %.2f%%" % results["pre"][-1])
# run training
sim.train(train_data,
opt,
objective={out_p: objective},
n_epochs=n_epochs)
# collect error after training
results["post"].append(
sim.loss(test_data, {out_p: classification_error},
training=True))
print("error after training: %.2f%%" % results["post"][-1])
# collect spiking error
results["spiking"].append(
sim.loss(test_data_time, {spk_out_p: classification_error},
training=False))
print("spiking neuron error: %.2f%%" % results["spiking"][-1])
with open("spiking_mnist_data.pkl", "wb") as f:
pickle.dump(results, f)
print("pre", bootstrap_ci(results["pre"]))
print("post", bootstrap_ci(results["post"]))
print("spiking", bootstrap_ci(results["spiking"]))
def compare_simplifications(ctx, dimensions):
load = ctx.obj["load"]
reps = ctx.obj["reps"]
device = ctx.obj["device"]
simplifications = [
graph_optimizer.remove_constant_copies,
graph_optimizer.remove_unmodified_resets,
graph_optimizer.remove_zero_incs, graph_optimizer.remove_identity_muls
]
params = list(itertools.product((False, True),
repeat=len(simplifications)))
if load:
with open("compare_simplifications_data.pkl", "rb") as f:
results = pickle.load(f)
else:
results = [
dict([("times", [])] + [(s.__name__, p[i])
for i, s in enumerate(simplifications)])
for j, p in enumerate(params)
]
# net = build_spaun(dimensions)
net = benchmarks.random_network(dimensions, 32, nengo.RectifiedLinear(),
1000, 100)
with net:
nengo_dl.configure_settings(inference_only=True)
model = nengo.builder.Model(dt=0.001,
builder=nengo_dl.builder.NengoBuilder())
model.build(net)
if reps > 0:
# pre-heat
with nengo_dl.Simulator(None,
model=model,
device=device,
progress_bar=False) as sim:
sim.run(reps, progress_bar=False)
for j, p in enumerate(params):
simps = []
for i, s in enumerate(p):
if s:
simps.append(simplifications[i])
with net:
nengo_dl.configure_settings(simplifications=simps)
print("%d/%d" % (j + 1, len(params)), [x.__name__ for x in simps])
with nengo_dl.Simulator(None,
model=model,
unroll_simulation=10,
device=device,
progress_bar=False) as sim:
sim.run(0.1, progress_bar=False)
sim_time = 1.0
for _ in range(reps):
start = time.time()
sim.run(sim_time, progress_bar=False)
results[j]["times"].append(
(time.time() - start) / sim_time)
print(" ", min(results[j]["times"]), max(results[j]["times"]),
np.mean(results[j]["times"]))
with open("compare_simplifications_data.pkl", "wb") as f:
pickle.dump(results, f)
def compare_optimizations(ctx, dimensions, unroll):
load = ctx.obj["load"]
reps = ctx.obj["reps"]
device = ctx.obj["device"]
save = ctx.obj["save"]
# optimizations to apply (simplifications, merging, sorting, unroll)
params = [
(False, False, False, False),
(False, False, False, True),
(False, True, False, True),
(False, True, True, True),
(True, True, True, True),
]
# params = list(itertools.product((False, True), repeat=4))
if load:
with open("compare_optimizations_%d_data_saved.pkl" % dimensions,
"rb") as f:
results = pickle.load(f)
else:
results = [{
"times": [],
"simplifications": simp,
"planner": plan,
"sorting": sort,
"unroll": unro
} for simp, plan, sort, unro in params]
if reps > 0:
with benchmarks.spaun(dimensions) as net:
nengo_dl.configure_settings(inference_only=True)
model = nengo.builder.Model(dt=0.001,
builder=nengo_dl.builder.NengoBuilder())
model.build(net)
print("neurons", net.n_neurons)
print("ensembles", len(net.all_ensembles))
print("connections", len(net.all_connections))
for i, (simp, plan, sort, unro) in enumerate(params):
print("%d/%d: %s %s %s %s" %
(i + 1, len(params), simp, plan, sort, unro))
with net:
config = dict()
config["simplifications"] = ([
graph_optimizer.remove_constant_copies,
graph_optimizer.remove_unmodified_resets,
# graph_optimizer.remove_zero_incs,
graph_optimizer.remove_identity_muls
] if simp else [])
config["planner"] = (graph_optimizer.tree_planner if plan else
graph_optimizer.greedy_planner)
config["sorter"] = (graph_optimizer.order_signals if sort else
graph_optimizer.noop_order_signals)
nengo_dl.configure_settings(**config)
with nengo_dl.Simulator(None,
model=model,
unroll_simulation=unroll if unro else 1,
device=device) as sim:
sim.run(0.1)
sim_time = 1.0
for _ in range(reps):
start = time.time()
sim.run(sim_time)
results[i]["times"].append(
(time.time() - start) / sim_time)
print(" ", min(results[i]["times"]), max(results[i]["times"]),
np.mean(results[i]["times"]))
with open("compare_optimizations_%d_data.pkl" % dimensions, "wb") as f:
pickle.dump(results, f)
data = np.asarray([bootstrap_ci(x) for x in filter_results(results)])
plt.figure()
alphas = np.linspace(0.5, 1, len(results))
colour = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
for i in range(len(results)):
plt.bar([i], [data[i, 0]],
yerr=abs(data[i, 1:] - data[i, [0]])[:, None],
log=True,
alpha=alphas[i],
color=colour)
labels = []
for r in results:
lab = "merging\n"
if r["unroll"]:
lab += "unrolling\n"
if r["planner"]:
lab += "planning\n"
if r["sorting"]:
lab += "sorting\n"
if r["simplifications"]:
lab += "simplifications\n"
labels.append(lab[:-1])
plt.xticks(np.arange(len(results)), labels, rotation="vertical")
plt.ylabel("real time / simulated time")
plt.tight_layout()
if save:
plt.savefig("compare_optimizations_%d.%s" % (dimensions, save))
def run_profile(
net, train=False, n_steps=150, do_profile=True, reps=1, dtype="float32", **kwargs
):
"""
Run profiler on a benchmark network.
Parameters
----------
net : `~nengo.Network`
The nengo Network to be profiled.
train : bool
If True, profile the `.Simulator.fit` function. Otherwise, profile the
`.Simulator.run` function.
n_steps : int
The number of timesteps to run the simulation.
do_profile : bool
Whether or not to run profiling
reps : int
Repeat the run this many times (only profile data from the last
run will be kept).
dtype : str
Simulation dtype (e.g. "float32")
Returns
-------
exec_time : float
Time (in seconds) taken to run the benchmark, taking the minimum over
``reps``.
Notes
-----
kwargs will be passed on to `.Simulator`
"""
exec_time = 1e10
n_batches = 1
with net:
nengo_dl.configure_settings(inference_only=not train, dtype=dtype)
with nengo_dl.Simulator(net, **kwargs) as sim:
if hasattr(net, "inp"):
x = {
net.inp: np.random.randn(
sim.minibatch_size * n_batches, n_steps, net.inp.size_out
)
}
elif hasattr(net, "inp_a"):
x = {
net.inp_a: np.random.randn(
sim.minibatch_size * n_batches, n_steps, net.inp_a.size_out
),
net.inp_b: np.random.randn(
sim.minibatch_size * n_batches, n_steps, net.inp_b.size_out
),
}
else:
x = None
if train:
y = {
net.p: np.random.randn(
sim.minibatch_size * n_batches, n_steps, net.p.size_in
)
}
sim.compile(tf.optimizers.SGD(0.001), loss=tf.losses.mse)
# run once to eliminate startup overhead
start = timeit.default_timer()
sim.fit(x, y, epochs=1, n_steps=n_steps)
print("Warmup time:", timeit.default_timer() - start)
for _ in range(reps):
if do_profile:
tf.profiler.experimental.start("profile")
start = timeit.default_timer()
sim.fit(x, y, epochs=1, n_steps=n_steps)
exec_time = min(timeit.default_timer() - start, exec_time)
if do_profile:
tf.profiler.experimental.stop()
else:
# run once to eliminate startup overhead
start = timeit.default_timer()
sim.predict(x, n_steps=n_steps)
print("Warmup time:", timeit.default_timer() - start)
for _ in range(reps):
if do_profile:
tf.profiler.experimental.start("profile")
start = timeit.default_timer()
sim.predict(x, n_steps=n_steps)
exec_time = min(timeit.default_timer() - start, exec_time)
if do_profile:
tf.profiler.experimental.stop()
exec_time /= n_batches
print("Execution time:", exec_time)
return exec_time
def __init__(self, units, order, theta, input_d, **kwargs):
super().__init__(**kwargs)
# compute the A and B matrices according to the LMU's mathematical
# derivation (see the paper for details)
Q = np.arange(order, dtype=np.float64)
R = (2 * Q + 1)[:, None] / theta
j, i = np.meshgrid(Q, Q)
A = np.where(i < j, -1, (-1.0) ** (i - j + 1)) * R
B = (-1.0) ** Q[:, None] * R
C = np.ones((1, order))
D = np.zeros((1,))
A, B, _, _, _ = cont2discrete((A, B, C, D), dt=1.0, method="zoh")
with self:
nengo_dl.configure_settings(trainable=None)
# create objects corresponding to the x/u/m/h variables in LMU
self.x = nengo.Node(size_in=input_d)
self.u = nengo.Node(size_in=1)
self.m = nengo.Node(size_in=order)
self.h = nengo_dl.TensorNode(
tf.nn.tanh, shape_in=(units,), pass_time=False
)
# compute u_t
# note that setting synapse=0 (versus synapse=None) adds a
# one-timestep delay, so we can think of any connections with
# synapse=0 as representing value_{t-1}
nengo.Connection(
self.x, self.u, transform=np.ones((1, input_d)), synapse=None
)
nengo.Connection(
self.h, self.u, transform=np.zeros((1, units)), synapse=0
)
nengo.Connection(
self.m, self.u, transform=np.zeros((1, order)), synapse=0
)
# compute m_t
# in this implementation we'll make A and B non-trainable, but they
# could also be optimized in the same way as the other parameters
conn = nengo.Connection(self.m, self.m, transform=A, synapse=0)
self.config[conn].trainable = False
conn = nengo.Connection(self.u, self.m, transform=B, synapse=None)
self.config[conn].trainable = False
# compute h_t
nengo.Connection(
self.x,
self.h,
transform=np.zeros((units, input_d)),
synapse=None,
)
nengo.Connection(
self.h, self.h, transform=np.zeros((units, units)), synapse=0
)
nengo.Connection(
self.m,
self.h,
transform=nengo_dl.dists.Glorot(distribution="normal"),
synapse=None,
)
def lmu(theta, input_d, native_nengo=False, dtype="float32"):
"""
A network containing a single Legendre Memory Unit cell and dense readout.
See [1]_ for more details.
Parameters
----------
theta : int
Time window parameter for LMU.
input_d : int
Dimensionality of input signal.
native_nengo : bool
If True, build the LMU out of Nengo objects. Otherwise, build the LMU
directly in TensorFlow, and use a `.TensorNode` to wrap the whole cell.
dtype : str
Float dtype to use for internal parameters of LMU when ``native_nengo=False``
(``native_nengo=True`` will use the dtype of the Simulator).
Returns
-------
net : `nengo.Network`
Benchmark network
References
----------
.. [1] Aaron R. Voelker, Ivana Kajić, and Chris Eliasmith. Legendre memory units:
continuous-time representation in recurrent neural networks.
In Advances in Neural Information Processing Systems. 2019.
https://papers.nips.cc/paper/9689-legendre-memory-units-continuous-time-representation-in-recurrent-neural-networks.
"""
if native_nengo:
# building LMU cell directly out of Nengo objects
class LMUCell(nengo.Network):
"""Implements an LMU cell as a Nengo network."""
def __init__(self, units, order, theta, input_d, **kwargs):
super().__init__(**kwargs)
# compute the A and B matrices according to the LMU's mathematical
# derivation (see the paper for details)
Q = np.arange(order, dtype=np.float64)
R = (2 * Q + 1)[:, None] / theta
j, i = np.meshgrid(Q, Q)
A = np.where(i < j, -1, (-1.0) ** (i - j + 1)) * R
B = (-1.0) ** Q[:, None] * R
C = np.ones((1, order))
D = np.zeros((1,))
A, B, _, _, _ = cont2discrete((A, B, C, D), dt=1.0, method="zoh")
with self:
nengo_dl.configure_settings(trainable=None)
# create objects corresponding to the x/u/m/h variables in LMU
self.x = nengo.Node(size_in=input_d)
self.u = nengo.Node(size_in=1)
self.m = nengo.Node(size_in=order)
self.h = nengo_dl.TensorNode(
tf.nn.tanh, shape_in=(units,), pass_time=False
)
# compute u_t
# note that setting synapse=0 (versus synapse=None) adds a
# one-timestep delay, so we can think of any connections with
# synapse=0 as representing value_{t-1}
nengo.Connection(
self.x, self.u, transform=np.ones((1, input_d)), synapse=None
)
nengo.Connection(
self.h, self.u, transform=np.zeros((1, units)), synapse=0
)
nengo.Connection(
self.m, self.u, transform=np.zeros((1, order)), synapse=0
)
# compute m_t
# in this implementation we'll make A and B non-trainable, but they
# could also be optimized in the same way as the other parameters
conn = nengo.Connection(self.m, self.m, transform=A, synapse=0)
self.config[conn].trainable = False
conn = nengo.Connection(self.u, self.m, transform=B, synapse=None)
self.config[conn].trainable = False
# compute h_t
nengo.Connection(
self.x,
self.h,
transform=np.zeros((units, input_d)),
synapse=None,
)
nengo.Connection(
self.h, self.h, transform=np.zeros((units, units)), synapse=0
)
nengo.Connection(
self.m,
self.h,
transform=nengo_dl.dists.Glorot(distribution="normal"),
synapse=None,
)
with nengo.Network(seed=0) as net:
# remove some unnecessary features to speed up the training
nengo_dl.configure_settings(
trainable=None, stateful=False, keep_history=False,
)
# input node
net.inp = nengo.Node(np.zeros(input_d))
# lmu cell
lmu_cell = LMUCell(units=212, order=256, theta=theta, input_d=input_d)
conn = nengo.Connection(net.inp, lmu_cell.x, synapse=None)
net.config[conn].trainable = False
# dense linear readout
out = nengo.Node(size_in=10)
nengo.Connection(
lmu_cell.h, out, transform=nengo_dl.dists.Glorot(), synapse=None
)
# record output. note that we set keep_history=False above, so this will
# only record the output on the last timestep (which is all we need
# on this task)
net.p = nengo.Probe(out)
else:
# putting everything in a tensornode
# define LMUCell
class LMUCell(tf.keras.layers.AbstractRNNCell):
"""Implement LMU as Keras RNN cell."""
def __init__(self, units, order, theta, **kwargs):
super().__init__(**kwargs)
self.units = units
self.order = order
self.theta = theta
Q = np.arange(order, dtype=np.float64)
R = (2 * Q + 1)[:, None] / theta
j, i = np.meshgrid(Q, Q)
A = np.where(i < j, -1, (-1.0) ** (i - j + 1)) * R
B = (-1.0) ** Q[:, None] * R
C = np.ones((1, order))
D = np.zeros((1,))
self._A, self._B, _, _, _ = cont2discrete(
(A, B, C, D), dt=1.0, method="zoh"
)
@property
def state_size(self):
"""Size of RNN state variables."""
return self.units, self.order
@property
def output_size(self):
"""Size of cell output."""
return self.units
def build(self, input_shape):
"""Set up all the weight matrices used inside the cell."""
super().build(input_shape)
input_dim = input_shape[-1]
self.input_encoders = self.add_weight(
shape=(input_dim, 1), initializer=tf.initializers.ones(),
)
self.hidden_encoders = self.add_weight(
shape=(self.units, 1), initializer=tf.initializers.zeros(),
)
self.memory_encoders = self.add_weight(
shape=(self.order, 1), initializer=tf.initializers.zeros(),
)
self.input_kernel = self.add_weight(
shape=(input_dim, self.units), initializer=tf.initializers.zeros(),
)
self.hidden_kernel = self.add_weight(
shape=(self.units, self.units), initializer=tf.initializers.zeros(),
)
self.memory_kernel = self.add_weight(
shape=(self.order, self.units),
initializer=tf.initializers.glorot_normal(),
)
self.AT = self.add_weight(
shape=(self.order, self.order),
initializer=tf.initializers.constant(self._A.T),
trainable=False,
)
self.BT = self.add_weight(
shape=(1, self.order),
initializer=tf.initializers.constant(self._B.T),
trainable=False,
)
def call(self, inputs, states):
"""Compute cell output and state updates."""
h_prev, m_prev = states
# compute u_t from the above diagram
u = (
tf.matmul(inputs, self.input_encoders)
+ tf.matmul(h_prev, self.hidden_encoders)
+ tf.matmul(m_prev, self.memory_encoders)
)
# compute updated memory state vector (m_t in diagram)
m = tf.matmul(m_prev, self.AT) + tf.matmul(u, self.BT)
# compute updated hidden state vector (h_t in diagram)
h = tf.nn.tanh(
tf.matmul(inputs, self.input_kernel)
+ tf.matmul(h_prev, self.hidden_kernel)
+ tf.matmul(m, self.memory_kernel)
)
return h, [h, m]
with nengo.Network(seed=0) as net:
# remove some unnecessary features to speed up the training
# we could set use_loop=False as well here, but leaving it for parity
# with native_nengo
nengo_dl.configure_settings(stateful=False)
net.inp = nengo.Node(np.zeros(theta))
rnn = nengo_dl.Layer(
tf.keras.layers.RNN(
LMUCell(units=212, order=256, theta=theta, dtype=dtype),
return_sequences=False,
)
)(net.inp, shape_in=(theta, input_d))
out = nengo.Node(size_in=10)
nengo.Connection(rnn, out, transform=nengo_dl.dists.Glorot(), synapse=None)
net.p = nengo.Probe(out)
return net
def run_network(activation,
params_file="./keras_to_loihi_params",
n_steps=30,
scale_firing_rates=1,
synapse=None,
n_test=100,
n_plots=1,
plot_idx=-1):
# convert the keras model to a nengo network
nengo_converter = nengo_dl.Converter(
model,
scale_firing_rates=scale_firing_rates,
swap_activations={tf.nn.relu: activation},
synapse=synapse,
)
print_neurons_type(nengo_converter)
# get input/output objects
nengo_input = nengo_converter.inputs[inp]
nengo_output = nengo_converter.outputs[output]
# add probes to layers to record activity
with nengo_converter.net:
probes = collections.OrderedDict([
[L1_layer, nengo.Probe(nengo_converter.layers[L1])],
[L2_layer, nengo.Probe(nengo_converter.layers[L2])],
[L3_layer, nengo.Probe(nengo_converter.layers[L3])],
])
# repeat inputs for some number of timesteps
tiled_test_data = np.tile(test_data[:n_test], (1, n_steps, 1))
# set some options to speed up simulation
with nengo_converter.net:
nengo_dl.configure_settings(stateful=False)
# build network, load in trained weights, run inference on test images
with nengo_dl.Simulator(nengo_converter.net,
minibatch_size=1,
progress_bar=False) as nengo_sim:
nengo_sim.load_params(params_file)
data = nengo_sim.predict({nengo_input: tiled_test_data})
# compute accuracy on test data, using output of network on last timestep
test_predictions = np.argmax(data[nengo_output][:, -1], axis=-1)
correct = test_truth[:n_test, 0, 0]
print("Test accuracy: %.2f%%" %
(100 * np.mean(test_predictions == correct)))
predicted = np.array(test_predictions, dtype=int)
correct = np.array(correct, dtype=int)
# Plot normalized confusion matrix
plot_confusion_matrix(correct,
predicted,
classes=class_names,
normalize=True,
title='Normalized confusion matrix')
plt.savefig(outdir + f'/{plot_idx}_confusion_matrix.jpg')
# plot the results
mean_rates = []
for i in range(n_plots):
plt.figure(figsize=(12, 6))
plt.subplot(1, 2, 1)
# TODO: add a plot of current input signal
# plt.title("Input signal")
# plt.axis("off")
n_layers = len(probes)
mean_rates_i = []
for j, layer in enumerate(probes.keys()):
probe = probes[layer]
plt.subplot(n_layers, 3, (j * 3) + 2)
plt.suptitle("Neural activities")
outputs = data[probe][i]
# look at only at non-zero outputs
nonzero = (outputs > 0).any(axis=0)
outputs = outputs[:, nonzero] if sum(nonzero) > 0 else outputs
# undo neuron amplitude to get real firing rates
outputs /= nengo_converter.layers[
layer].ensemble.neuron_type.amplitude
rates = outputs.mean(axis=0)
mean_rate = rates.mean()
mean_rates_i.append(mean_rate)
print('"%s" mean firing rate (example %d): %0.1f' %
(layer.name, i, mean_rate))
if is_spiking_type(activation):
outputs *= 0.001
plt.ylabel("# of Spikes")
else:
plt.ylabel("Firing rates (Hz)")
# plot outputs of first 100 neurons
plt.plot(outputs[:, :100])
mean_rates.append(mean_rates_i)
plt.xlabel("Timestep")
plt.subplot(1, 3, 3)
plt.title("Output predictions")
plt.plot(tf.nn.softmax(data[nengo_output][i]))
plt.legend([str(j) for j in range(10)], loc="upper left")
plt.xlabel("Timestep")
plt.ylabel("Probability")
plt.tight_layout()
# take mean rates across all plotted examples
mean_rates = np.array(mean_rates).mean(axis=0)
return mean_rates
