def test_conv_onchip(Simulator, plt):
"""Tests a fully on-chip conv connection. """
from nengo._vendor.npconv2d.conv2d import conv2d
kernel = np.array([[-1, 2, -1], [-1, 2, -1], [-1, 2, -1]], dtype=float)
kernel /= kernel.max()
image = np.array([[1, 2, 1, 2, 0], [2, 3, 2, 1, 1], [1, 2, 1, 2, 3],
[2, 3, 2, 1, 1], [1, 2, 1, 2, 0]],
dtype=float)
image /= image.max()
input_scale = 119.
bias = input_scale * image.ravel()
neuron_type = nengo.SpikingRectifiedLinear()
y_ref = LoihiSpikingRectifiedLinear().rates(image.ravel(), input_scale, 0)
y_ref = conv2d(y_ref.reshape(1, 5, 5, 1),
kernel.reshape(3, 3, 1, 1),
pad='VALID')
y_ref = LoihiSpikingRectifiedLinear().rates(y_ref.ravel(), 1.,
0.).reshape(3, 3)
with nengo.Network() as net:
a = nengo.Ensemble(bias.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([0]),
bias=bias)
transform = nengo_transforms.Convolution(n_filters=1,
input_shape=(5, 5, 1),
init=kernel.reshape(
3, 3, 1, 1))
b = nengo.Ensemble(transform.output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([1]),
bias=nengo.dists.Choice([0]))
nengo.Connection(a.neurons, b.neurons, transform=transform)
bp = nengo.Probe(b.neurons, synapse=nengo.Alpha(0.02))
with Simulator(net) as sim:
sim.run(0.3)
y_ref = y_ref / input_scale
y = sim.data[bp][-1].reshape(3, -1) / input_scale
plt.subplot(121)
plt.imshow(y_ref)
plt.colorbar()
plt.subplot(122)
plt.imshow(y)
plt.colorbar()
assert np.allclose(y, y_ref, atol=0.02, rtol=0.1)
def __init__(self, pairs_per_dim=1, dt=0.001, rate=None):
super(OnOffDecodeNeurons, self).__init__(dt=dt)
self.pairs_per_dim = pairs_per_dim
self.rate = (1. /
(self.dt * self.pairs_per_dim) if rate is None else rate)
self.scale = 1. / (self.dt * self.rate * self.pairs_per_dim)
self.neuron_type = LoihiSpikingRectifiedLinear()
gain = 0.5 * self.rate * np.ones(self.pairs_per_dim)
bias = gain # intercept of -1
self.gain = gain.repeat(2)
self.bias = bias.repeat(2)
def __init__(self, pairs_per_dim=1, dt=0.001, rate=None, is_input=False):
super().__init__(dt=dt)
self.pairs_per_dim = pairs_per_dim
self.is_input = is_input
self.rate = 1.0 / (self.dt * self.pairs_per_dim) if rate is None else rate
self.scale = 1.0 / (self.dt * self.rate * self.pairs_per_dim)
self.neuron_type = LoihiSpikingRectifiedLinear()
gain = 0.5 * self.rate * np.ones(self.pairs_per_dim)
bias = gain # intercept of -1
self.gain = gain.repeat(2)
self.bias = bias.repeat(2)
def test_conv_preslice(Simulator, plt):
from nengo._vendor.npconv2d.conv2d import conv2d
kernel = np.array([[-1, 2, -1], [-1, 2, -1], [-1, 2, -1]], dtype=float)
kernel /= kernel.max()
image = np.array([[1, 2, 1, 2, 0], [2, 3, 2, 1, 1], [1, 2, 1, 2, 3],
[2, 3, 2, 1, 1], [1, 2, 1, 2, 0]],
dtype=float)
image /= image.max()
image2 = np.column_stack([c * x for c in image.T for x in (1, -1)])
input_gain = 149.
neuron_type = nengo.SpikingRectifiedLinear()
y_ref = LoihiSpikingRectifiedLinear().rates(image.ravel(), input_gain, 0)
y_ref = conv2d(y_ref.reshape(1, 5, 5, 1),
kernel.reshape(3, 3, 1, 1),
pad='VALID')
y_ref = LoihiSpikingRectifiedLinear().rates(y_ref.ravel(), 1.,
0.).reshape(3, 3)
with nengo.Network() as net:
u = nengo.Node(image2.ravel())
a = nengo.Ensemble(50,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([input_gain]),
bias=nengo.dists.Choice([0]))
transform = nengo_transforms.Convolution(n_filters=1,
input_shape=(5, 5, 1),
init=kernel.reshape(
3, 3, 1, 1))
b = nengo.Ensemble(transform.output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([1]),
bias=nengo.dists.Choice([0]))
nengo.Connection(u, a.neurons, synapse=None)
nengo.Connection(a.neurons[::2], b.neurons, transform=transform)
bp = nengo.Probe(b.neurons, synapse=nengo.Alpha(0.02))
hw_opts = dict(snip_max_spikes_per_step=100)
with Simulator(net, hardware_options=hw_opts) as sim:
sim.run(0.3)
y_ref = y_ref / input_gain
y = sim.data[bp][-1].reshape(3, -1) / input_gain
plt.subplot(121)
plt.imshow(y_ref)
plt.colorbar()
plt.subplot(122)
plt.imshow(y)
plt.colorbar()
assert np.allclose(y, y_ref, atol=0.02, rtol=0.1)
def test_conv_connection(channels, channels_last, Simulator, seed, rng, plt,
allclose):
if channels_last:
plt.saveas = None
pytest.xfail("Blocked by CxBase cannot be > 256 bug")
# load data
with open(os.path.join(test_dir, 'mnist10.pkl'), 'rb') as f:
test10 = pickle.load(f)
test_x = test10[0][0].reshape(28, 28)
test_x = 1.999 * test_x - 0.999 # range (-1, 1)
input_shape = nengo_transforms.ChannelShape(
(test_x.shape + (channels, )) if channels_last else
((channels, ) + test_x.shape),
channels_last=channels_last)
filters = Gabor(freq=Uniform(0.5, 1)).generate(8, (7, 7), rng=rng)
filters = filters[None, :, :, :] # single channel
filters = np.transpose(filters, (2, 3, 0, 1))
strides = (2, 2)
tau_rc = 0.02
tau_ref = 0.002
tau_s = 0.005
dt = 0.001
neuron_type = LoihiLIF(tau_rc=tau_rc, tau_ref=tau_ref)
pres_time = 0.1
with nengo.Network(seed=seed) as model:
nengo_loihi.add_params(model)
u = nengo.Node(test_x.ravel(), label="u")
a = nengo.Ensemble(input_shape.size,
1,
neuron_type=LoihiSpikingRectifiedLinear(),
max_rates=nengo.dists.Choice([40 / channels]),
intercepts=nengo.dists.Choice([0]),
label='a')
model.config[a].on_chip = False
if channels == 1:
nengo.Connection(u, a.neurons, transform=1, synapse=None)
elif channels == 2:
# encode image into spikes using two channels (on/off)
if input_shape.channels_last:
nengo.Connection(u, a.neurons[0::2], transform=1, synapse=None)
nengo.Connection(u,
a.neurons[1::2],
transform=-1,
synapse=None)
else:
k = input_shape.spatial_shape[0] * input_shape.spatial_shape[1]
nengo.Connection(u, a.neurons[:k], transform=1, synapse=None)
nengo.Connection(u, a.neurons[k:], transform=-1, synapse=None)
filters = np.concatenate([filters, -filters], axis=2)
else:
raise ValueError("Test not configured for more than two channels")
conv2d_transform = nengo_transforms.Convolution(
8,
input_shape,
strides=strides,
kernel_size=(7, 7),
channels_last=channels_last,
init=filters)
output_shape = conv2d_transform.output_shape
gain, bias = neuron_type.gain_bias(max_rates=100, intercepts=0)
gain = gain * 0.01 # account for `a` max_rates
b = nengo.Ensemble(output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([gain[0]]),
bias=nengo.dists.Choice([bias[0]]),
label='b')
nengo.Connection(a.neurons,
b.neurons,
synapse=tau_s,
transform=conv2d_transform)
bp = nengo.Probe(b.neurons)
with nengo.Simulator(model, dt=dt, optimize=False) as sim:
sim.run(pres_time)
ref_out = sim.data[bp].mean(axis=0).reshape(output_shape.shape)
# Currently, non-gpu TensorFlow does not support channels first in conv
use_nengo_dl = HAS_DL and channels_last
ndl_out = np.zeros_like(ref_out)
if use_nengo_dl:
with nengo_dl.Simulator(model, dt=dt) as sim_dl:
sim_dl.run(pres_time)
ndl_out = sim_dl.data[bp].mean(axis=0).reshape(output_shape.shape)
with nengo_loihi.Simulator(model, dt=dt, target='simreal') as sim_real:
sim_real.run(pres_time)
real_out = sim_real.data[bp].mean(axis=0).reshape(output_shape.shape)
with Simulator(model, dt=dt) as sim_loihi:
if "loihi" in sim_loihi.sims:
sim_loihi.sims["loihi"].snip_max_spikes_per_step = 800
sim_loihi.run(pres_time)
sim_out = sim_loihi.data[bp].mean(axis=0).reshape(output_shape.shape)
if not output_shape.channels_last:
ref_out = np.transpose(ref_out, (1, 2, 0))
ndl_out = np.transpose(ndl_out, (1, 2, 0))
real_out = np.transpose(real_out, (1, 2, 0))
sim_out = np.transpose(sim_out, (1, 2, 0))
out_max = max(ref_out.max(), sim_out.max())
# --- plot results
rows = 2
cols = 3
ax = plt.subplot(rows, cols, 1)
imshow(test_x, vmin=0, vmax=1, ax=ax)
ax = plt.subplot(rows, cols, 2)
tile(np.transpose(filters[0], (2, 0, 1)), cols=8, ax=ax)
ax = plt.subplot(rows, cols, 3)
plt.hist(ref_out.ravel(), bins=31)
plt.hist(sim_out.ravel(), bins=31)
ax = plt.subplot(rows, cols, 4)
tile(np.transpose(ref_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 5)
tile(np.transpose(ndl_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 6)
tile(np.transpose(sim_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
if use_nengo_dl:
assert allclose(ndl_out, ref_out, atol=1e-5, rtol=1e-5)
assert allclose(real_out, ref_out, atol=1, rtol=1e-3)
assert allclose(sim_out, ref_out, atol=10, rtol=1e-3)
def test_conv_preslice(on_chip, Simulator, plt):
conv2d = pytest.importorskip("nengo._vendor.npconv2d.conv2d")
kernel = np.array([[-1, 2, -1], [-1, 2, -1], [-1, 2, -1]], dtype=float)
kernel /= kernel.max()
image = np.array(
[
[1, 2, 1, 2, 0],
[2, 3, 2, 1, 1],
[1, 2, 1, 2, 3],
[2, 3, 2, 1, 1],
[1, 2, 1, 2, 0],
],
dtype=float,
)
image /= image.max()
image2 = np.column_stack([c * x for c in image.T for x in (1, -1)])
input_gain = 149.0
neuron_type = nengo.SpikingRectifiedLinear()
loihi_neuron = LoihiSpikingRectifiedLinear()
layer0_neuron = loihi_neuron if on_chip else neuron_type
y_ref = layer0_neuron.rates(image.ravel(), input_gain, 0)
y_ref = conv2d.conv2d(y_ref.reshape((1, 5, 5, 1)),
kernel.reshape((3, 3, 1, 1)),
pad="VALID")
y_ref = loihi_neuron.rates(y_ref.ravel(), 1.0, 0.0).reshape((3, 3))
with nengo.Network() as net:
nengo_loihi.add_params(net)
u = nengo.Node(image2.ravel())
a = nengo.Ensemble(
50,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([input_gain]),
bias=nengo.dists.Choice([0]),
)
net.config[a].on_chip = on_chip
transform = nengo_transforms.Convolution(n_filters=1,
input_shape=(5, 5, 1),
init=kernel.reshape(
(3, 3, 1, 1)))
b = nengo.Ensemble(
transform.output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([1]),
bias=nengo.dists.Choice([0]),
)
nengo.Connection(u, a.neurons, synapse=None)
nengo.Connection(a.neurons[::2], b.neurons, transform=transform)
bp = nengo.Probe(b.neurons, synapse=nengo.Alpha(0.02))
with Simulator(net) as sim:
assert sim.precompute is True
sim.run(0.3)
y_ref = y_ref / input_gain
y = sim.data[bp][-1].reshape((3, -1)) / input_gain
plt.subplot(121)
plt.imshow(y_ref)
plt.colorbar()
plt.subplot(122)
plt.imshow(y)
plt.colorbar()
assert np.allclose(y, y_ref, atol=0.02, rtol=0.1)
def test_conv_deepnet(
channels_last,
pop_type,
precompute,
Simulator,
request,
rng,
seed,
plt,
allclose,
):
"""Run a convolutional network with two layers on the chip.
Checks that network with block splitting on the target matches one without
on the emulator.
"""
# TODO: This case fails in NxSDK 0.9.0 but will be fixed in the next version.
# Remove this check once the next version is released.
if pop_type == 32:
pytest.skip("Pop32 multichip test requires latest NxSDK")
def set_partition(partition):
os.environ["PARTITION"] = partition
request.addfinalizer(lambda: set_partition(""))
# multichip pop_type = 16 works only on nahuku32 board currently
if pop_type == 16:
set_partition("nahuku32")
def conv_layer(x,
input_shape,
array_init=None,
label=None,
conn_args=None,
**conv_args):
conn_args = {} if conn_args is None else conn_args
if array_init is not None:
assert all(a not in conv_args
for a in ("init", "kernel_size", "n_filters"))
assert array_init.ndim == 4
conv_args["init"] = array_init
conv_args["kernel_size"] = array_init.shape[:2]
assert array_init.shape[2] == input_shape.n_channels
conv_args["n_filters"] = array_init.shape[3]
conv = nengo.Convolution(input_shape=input_shape, **conv_args)
# add an ensemble to implement the activation function
layer = nengo.Ensemble(conv.output_shape.size, 1, label=label)
# connect up the input object to the new layer
conn = nengo.Connection(x, layer.neurons, transform=conv)
return layer, conv, conn
channels = 1
n_filters0 = 1
n_filters1 = 4
n_filters2 = 4
# load data
with open(os.path.join(test_dir, "mnist10.pkl"), "rb") as f:
test10 = pickle.load(f)
test_x = test10[0][0].reshape(28, 28) # range (0, 1)
input_shape = make_channel_shape(test_x.shape, channels, channels_last)
filters0 = np.ones((1, 1, channels, n_filters0))
# use Gabor filters for first layer
filters1 = Gabor(freq=Uniform(0.5, 1),
sigma_x=Choice([0.9]),
sigma_y=Choice([0.9])).generate(n_filters1, (7, 7),
rng=rng)
assert n_filters0 == 1
filters1 = filters1[None, :, :, :] # single channel
filters1 = np.transpose(filters1,
(2, 3, 0, 1)) # rows, cols, in_chan, out_chan
# use random combinations of first-layer channels in 1x1 convolution
filters2 = rng.uniform(-0.2, 1,
size=(n_filters1, n_filters2)).clip(0, None)
filters2 *= 2 / filters2.sum(axis=0,
keepdims=True) # each filter sums to 2
filters2 = filters2[None, None, :, :] # rows, cols, in_chan, out_chan
tau_s = 0.001
max_rate = 100
amp = 1 / max_rate
f_split = 2 if pop_type == 32 else 4
# use Loihi neuron type so Nengo sim mimics Loihi neuron effects
neuron_type = LoihiSpikingRectifiedLinear(amplitude=amp)
pres_time = 0.2
with nengo.Network(seed=seed) as net:
nengo_loihi.add_params(net)
net.config[nengo.Ensemble].neuron_type = neuron_type
net.config[nengo.Ensemble].max_rates = Choice([max_rate])
net.config[nengo.Ensemble].intercepts = Choice([0])
net.config[nengo.Connection].synapse = tau_s
u = nengo.Node(test_x.ravel(), label="u")
layer0, conv0, conn0 = conv_layer(
u,
input_shape=input_shape,
array_init=filters0,
strides=(1, 1),
channels_last=channels_last,
label="layer0",
conn_args=dict(synapse=None),
)
net.config[layer0].on_chip = False
layer1, conv1, conn1 = conv_layer(
layer0.neurons,
input_shape=conv0.output_shape,
array_init=filters1,
strides=(2, 2),
channels_last=channels_last,
label="layer1",
)
net.config[layer1].block_shape = nengo_loihi.BlockShape(
make_shape((4, 4), f_split, channels_last), conv1)
net.config[conn1].pop_type = pop_type
layer2, conv2, conn2 = conv_layer(
layer1.neurons,
input_shape=conv1.output_shape,
array_init=filters2,
strides=(1, 1),
channels_last=channels_last,
label="layer2",
)
net.config[layer2].block_shape = nengo_loihi.BlockShape(
make_shape((4, 4), f_split, channels_last), conv2)
net.config[conn2].pop_type = pop_type
output_p = nengo.Probe(layer2.neurons)
output_shape = conv2.output_shape
with nengo.Simulator(net, optimize=False) as sim_nengo:
sim_nengo.run(pres_time)
ref_out = (sim_nengo.data[output_p] > 0).sum(axis=0).reshape(
output_shape.shape)
with Simulator(net, target="sim") as sim_emu:
sim_emu.run(pres_time)
emu_out = (sim_emu.data[output_p] > 0).sum(axis=0).reshape(
output_shape.shape)
# TODO: Remove the if condition when configurable timeout parameter
# is available in nxsdk
if (pop_type == 32 or
os.popen("sinfo -h --partition=nahuku32").read().find("idle") > 0):
with Simulator(
net,
precompute=precompute,
hardware_options={
"allocator": RoundRobin(),
"snip_max_spikes_per_step": 800,
},
) as sim_loihi:
sim_loihi.run(pres_time)
sim_out = ((sim_loihi.data[output_p] > 0).sum(axis=0).reshape(
output_shape.shape))
elif nengo_loihi.version.dev is None:
pytest.fail(
"Pop16 multichip test failed since Nahuku32 is unavailable")
else:
pytest.skip(
"Pop16 multichip test skipped since Nahuku32 is unavailable")
out_max = ref_out.max()
ref_out = ref_out / out_max
emu_out = emu_out / out_max
sim_out = sim_out / out_max
if channels_last:
# channels first, to display channels in separate plots
ref_out = np.transpose(ref_out, (2, 0, 1))
emu_out = np.transpose(emu_out, (2, 0, 1))
sim_out = np.transpose(sim_out, (2, 0, 1))
# --- plot results
rows = 2
cols = 3
ax = plt.subplot(rows, cols, 1)
imshow(test_x, vmin=0, vmax=1, ax=ax)
ax = plt.subplot(rows, cols, 2)
tile(np.transpose(filters1, (2, 3, 0, 1))[0],
rows=2,
cols=2,
grid=True,
ax=ax)
ax = plt.subplot(rows, cols, 3)
plt.hist((ref_out.ravel(), emu_out.ravel(), sim_out.ravel()), bins=21)
ax = plt.subplot(rows, cols, 4)
tile(ref_out, rows=2, cols=2, grid=True, ax=ax)
ax = plt.subplot(rows, cols, 5)
tile(emu_out, rows=2, cols=2, grid=True, ax=ax)
ax = plt.subplot(rows, cols, 6)
tile(sim_out, rows=2, cols=2, grid=True, ax=ax)
assert allclose(sim_out, ref_out, atol=0.15, rtol=1e-3)
assert allclose(sim_out, emu_out, atol=1e-3, rtol=1e-3)
def test_conv_connection(channels, channels_last, Simulator, seed, rng, plt,
allclose):
# load data
with open(os.path.join(test_dir, "mnist10.pkl"), "rb") as f:
test10 = pickle.load(f)
test_x = test10[0][0].reshape((28, 28))
test_x = 1.999 * test_x - 0.999 # range (-1, 1)
input_shape = make_channel_shape(test_x.shape, channels, channels_last)
filters = Gabor(freq=Uniform(0.5, 1)).generate(8, (7, 7), rng=rng)
filters = filters[None, :, :, :] # single channel
filters = np.transpose(filters, (2, 3, 0, 1))
strides = (2, 2)
tau_rc = 0.02
tau_ref = 0.002
tau_s = 0.005
neuron_type = LoihiLIF(tau_rc=tau_rc, tau_ref=tau_ref)
pres_time = 0.1
with nengo.Network(seed=seed) as model:
nengo_loihi.add_params(model)
u = nengo.Node(test_x.ravel(), label="u")
a = nengo.Ensemble(
input_shape.size,
1,
neuron_type=LoihiSpikingRectifiedLinear(),
max_rates=nengo.dists.Choice([40 / channels]),
intercepts=nengo.dists.Choice([0]),
label="a",
)
model.config[a].on_chip = False
if channels == 1:
nengo.Connection(u, a.neurons, transform=1, synapse=None)
elif channels == 2:
# encode image into spikes using two channels (on/off)
if input_shape.channels_last:
nengo.Connection(u, a.neurons[0::2], transform=1, synapse=None)
nengo.Connection(u,
a.neurons[1::2],
transform=-1,
synapse=None)
else:
k = input_shape.spatial_shape[0] * input_shape.spatial_shape[1]
nengo.Connection(u, a.neurons[:k], transform=1, synapse=None)
nengo.Connection(u, a.neurons[k:], transform=-1, synapse=None)
filters = np.concatenate([filters, -filters], axis=2)
else:
raise ValueError("Test not configured for more than two channels")
conv2d_transform = nengo_transforms.Convolution(
8,
input_shape,
strides=strides,
kernel_size=(7, 7),
channels_last=channels_last,
init=filters,
)
output_shape = conv2d_transform.output_shape
gain, bias = neuron_type.gain_bias(max_rates=100, intercepts=0)
gain = gain * 0.01 # account for `a` max_rates
b = nengo.Ensemble(
output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([gain[0]]),
bias=nengo.dists.Choice([bias[0]]),
label="b",
)
nengo.Connection(a.neurons,
b.neurons,
synapse=tau_s,
transform=conv2d_transform)
bp = nengo.Probe(b.neurons)
with nengo.Simulator(model, optimize=False) as sim_nengo:
sim_nengo.run(pres_time)
ref_out = sim_nengo.data[bp].mean(axis=0).reshape(output_shape.shape)
with Simulator(model, target="simreal") as sim_emu:
sim_emu.run(pres_time)
emu_out = sim_emu.data[bp].mean(axis=0).reshape(output_shape.shape)
with Simulator(model, hardware_options={"snip_max_spikes_per_step":
800}) as sim_loihi:
sim_loihi.run(pres_time)
sim_out = sim_loihi.data[bp].mean(axis=0).reshape(output_shape.shape)
if not output_shape.channels_last:
ref_out = np.transpose(ref_out, (1, 2, 0))
emu_out = np.transpose(emu_out, (1, 2, 0))
sim_out = np.transpose(sim_out, (1, 2, 0))
out_max = max(ref_out.max(), emu_out.max(), sim_out.max())
# --- plot results
rows = 2
cols = 3
ax = plt.subplot(rows, cols, 1)
imshow(test_x, vmin=0, vmax=1, ax=ax)
ax = plt.subplot(rows, cols, 2)
tile(np.transpose(filters[0], (2, 0, 1)), cols=8, ax=ax)
ax = plt.subplot(rows, cols, 3)
plt.hist(ref_out.ravel(), bins=31)
plt.hist(sim_out.ravel(), bins=31)
ax = plt.subplot(rows, cols, 4)
tile(np.transpose(ref_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 5)
tile(np.transpose(emu_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 6)
tile(np.transpose(sim_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
assert allclose(emu_out, ref_out, atol=10, rtol=1e-3)
assert allclose(sim_out, ref_out, atol=10, rtol=1e-3)
def demo(
backend="cpu",
collect_ground_truth=False,
motor_neuron_type=LoihiSpikingRectifiedLinear(),
neural_vision=True,
plot_mounted_camera_freq=None,
weights_name=None,
):
if backend == "loihi":
nengo_loihi.set_defaults()
seed = 0
np.random.seed(seed)
# target generation limits
dist_limit = [0.25, 3.5]
angle_limit = [-np.pi, np.pi]
accel_scale = 500
steer_scale = 500
# data collection parameters in steps (1ms per step)
max_time_to_target = 10000
net = nengo.Network(seed=seed)
# create our Mujoco interface
interface = Mujoco(
xml_file="rover.xml",
joint_names=["steering_wheel"],
dt=0.001,
render_params={
"cameras": [4, 1, 3, 2], # camera ids and order to render
"resolution": [32, 32],
"frequency": 1, # render images from cameras every time step
"plot_frequency": None, # do not plot images from cameras
},
track_input=True,
input_scale=np.array([steer_scale, accel_scale]),
)
# NOTE: why the slow rendering when defined before interface?
vision = RoverVision(seed=0)
# set up the target position
net.target = np.array([-0.0, 0.0, 0.2])
net.config[nengo.Connection].synapse = None
with net:
net.image_count = 0
net.val_img_count = -1
net.train_img_count = -1
# track position
net.target_track = []
net.rover_track = []
net.vision_track = []
net.localtarget_track = []
net.ideal_motor_track = []
if neural_vision:
visionsim, visionnet = vision.convert(
add_probes=False,
swap_activations={tf.nn.relu: LoihiSpikingRectifiedLinear()},
scale_firing_rates=400,
)
if weights_name is not None:
visionsim.load_params("%s/%s" % (current_dir, weights_name))
with visionsim:
visionsim.freeze_params(visionnet)
vision.input.size_in = vision.subpixels
vision.input.output = None
if not neural_vision or collect_ground_truth:
# rotation matrix for rotating error from world to rover frame
theta = 3 * np.pi / 2
R90 = np.array([
[np.cos(theta), -np.sin(theta), 0],
[np.sin(theta), np.cos(theta), 0],
[0, 0, 1],
])
def local_target(t):
rover_xyz = interface.get_position("base_link")
error = net.target - rover_xyz
# error in global coordinates, want it in local coordinates for rover
R_raw = interface.get_orientation("base_link").T # R.T = R^-1
# rotate it so y points forward toward the steering wheels
R = np.dot(R90, R_raw)
local_target = np.dot(R, error)
net.localtarget_track.append(local_target / np.pi)
# used to roughly normalize the local target signal to -1:1 range
output_signal = np.array([local_target[0], local_target[1]])
return output_signal
local_target = nengo.Node(output=local_target, size_out=2)
def check_exit_and_track_data(t, x):
if interface.exit:
raise ExitSim
rover_xyz = interface.get_position("base_link")
dist = np.linalg.norm(rover_xyz - net.target)
if dist < 0.2 or int(t / interface.dt) % max_time_to_target == 0:
# generate a new target 1-2.5m away from current position
while dist < 1 or dist > 2.5:
phi = np.random.uniform(low=angle_limit[0],
high=angle_limit[1])
radius = np.random.uniform(low=dist_limit[0],
high=dist_limit[1])
net.target = [
np.cos(phi) * radius,
np.sin(phi) * radius, 0.2
]
dist = np.linalg.norm(rover_xyz - net.target)
interface.set_mocap_xyz("target", net.target)
net.target_track.append(net.target)
# track data
net.rover_track.append(rover_xyz)
net.vision_track.append(np.copy(x))
check_exit_and_track_data = nengo.Node(
check_exit_and_track_data,
size_in=2,
label="check_exit_and_track_data")
mujoco_node = interface.make_node()
# -----------------------------------------------------------------------------
# --- set up ensemble to calculate acceleration
def accel_function(x):
return min(np.linalg.norm(-x), 1)
accel = nengo.Ensemble(
neuron_type=motor_neuron_type,
n_neurons=4096,
dimensions=2,
max_rates=nengo.dists.Uniform(50, 200),
radius=1,
label="accel",
)
nengo.Connection(
accel,
mujoco_node[1],
function=accel_function,
synapse=0 if motor_neuron_type == nengo.Direct() else 0.1,
)
# --- set up ensemble to calculate torque to apply to steering wheel
def steer_function(x):
error = x[1:] * np.pi # take out the error signal from vision
q = x[0] * 0.7 # scale normalized input back to original range
# arctan2 input set this way to account for different alignment
# of (x, y) axes of rover and the environment
# turn_des = np.arctan2(-error[0], abs(error[1]))
turn_des = np.arctan2(-error[0], error[1])
u = (turn_des - q) / 2 # divide by 2 to get in -1 to 1 ish range
return u
steer = nengo.Ensemble(
neuron_type=motor_neuron_type,
n_neurons=4096,
dimensions=3,
max_rates=nengo.dists.Uniform(50, 200),
radius=np.sqrt(2),
label="steer",
)
nengo.Connection(
steer,
mujoco_node[0],
function=lambda x: steer_function(x),
# if using Direct mode, a synapse will cause oscillating control
synapse=0 if motor_neuron_type == nengo.Direct() else 0.025,
)
# add a relay node to amalgamate input to steering ensemble
steer_input = nengo.Node(size_in=3, label="relay_motor_input")
nengo.Connection(steer_input, steer)
# -- connect relevant motor feedback (joint angle of steering wheel)
nengo.Connection(mujoco_node[0], steer_input[0])
# --- connect vision up to motor ensembles and mujoco node
vision_synapse = 0.05
if neural_vision:
# send image input in to vision system
nengo.Connection(mujoco_node[2:], vision.input)
# connect vision network prediction to steering ensemble
nengo.Connection(vision.output,
steer_input[1:],
synapse=vision_synapse)
# connect vision network prediction to accel ensemble
nengo.Connection(vision.output, accel, synapse=vision_synapse)
# connect vision network to data tracking node
nengo.Connection(vision.output,
check_exit_and_track_data,
synapse=vision_synapse)
else:
# if we're not using neural vision, then hook up the local_target node
# to our motor system to get the location of the target relative to rover
nengo.Connection(
local_target,
steer_input[1:],
synapse=vision_synapse,
transform=1 / np.pi,
)
nengo.Connection(
local_target,
accel,
synapse=vision_synapse,
transform=1 / np.pi,
)
nengo.Connection(local_target,
check_exit_and_track_data,
transform=1 / np.pi)
if collect_ground_truth:
# create a direct mode ensemble performing the same calculation for debugging
def calculate_ideal_motor_signals(t, x):
net.ideal_motor_track.append([
steer_function(x) * steer_scale,
accel_function(x[1:]) * accel_scale,
])
ground_truth = nengo.Node(output=calculate_ideal_motor_signals,
size_in=3,
size_out=0)
# connect local target prediction or node
if neural_vision:
nengo.Connection(
vision.output,
ground_truth[1:],
synapse=vision_synapse,
)
else:
nengo.Connection(
local_target,
ground_truth[1:],
synapse=vision_synapse,
transform=1 / np.pi,
)
# connect steering wheel angle feedback
nengo.Connection(mujoco_node[0], ground_truth[0])
if backend == "loihi":
nengo_loihi.add_params(net)
if neural_vision:
net.config[vision.conv0.ensemble].on_chip = False
net.control_track = interface.input_track
return net
self.probe_dense = nengo.Probe(self.output,
label="probe_dense",
synapse=synapse)
sim = nengo_dl.Simulator(net,
minibatch_size=self.minibatch_size,
seed=self.seed)
return sim, net
if __name__ == "__main__":
current_dir = os.path.abspath(".")
db_dir = "data"
mode = "predict" # should be ["predict"|"run"]
if mode == "run":
activation = LoihiSpikingRectifiedLinear(
) # can be any Nengo neuron type
n_steps = 100 # how many time steps to present the input for
synapse = 0.005
elif mode == "predict":
activation = LoihiRectifiedLinear()
n_steps = 2
synapse = None
scale_firing_rates = 400
weights = sys.argv[1] if len(sys.argv) > 1 else "data/reference_weights"
if weights[-4:] == ".npz":
weights = weights[:-4]
images, targets = dl_utils.load_data(
db_dir=db_dir,
db_name="rover",
def test_conv_connection(channels, Simulator, seed, rng, plt, allclose):
# channels_last = True
channels_last = False
if channels > 1:
pytest.xfail("Cannot send population spikes to chip")
# load data
with open(os.path.join(test_dir, 'mnist10.pkl'), 'rb') as f:
test10 = pickle.load(f)
test_x = test10[0][0].reshape(28, 28)
test_x = 1.999 * test_x - 0.999 # range (-1, 1)
test_x = test_x[:, :, None] # single channel
input_shape = ImageShape(test_x.shape[0],
test_x.shape[1],
channels,
channels_last=channels_last)
filters = Gabor(freq=Uniform(0.5, 1)).generate(8, (7, 7), rng=rng)
filters = filters[None, :, :, :] # single channel
filters = np.transpose(filters, (0, 2, 3, 1)) # filters last
strides = (2, 2)
tau_rc = 0.02
tau_ref = 0.002
tau_s = 0.005
dt = 0.001
neuron_type = LoihiLIF(tau_rc=tau_rc, tau_ref=tau_ref)
pres_time = 0.1
with nengo.Network(seed=seed) as model:
nengo_loihi.add_params(model)
u = nengo.Node(nengo.processes.PresentInput([test_x.ravel()],
pres_time),
label='u')
a = nengo.Ensemble(input_shape.size,
1,
neuron_type=LoihiSpikingRectifiedLinear(),
max_rates=nengo.dists.Choice([40 / channels]),
intercepts=nengo.dists.Choice([0]),
label='a')
model.config[a].on_chip = False
if channels == 1:
nengo.Connection(u, a.neurons, transform=1, synapse=None)
elif channels == 2:
# encode image into spikes using two channels (on/off)
if input_shape.channels_last:
nengo.Connection(u, a.neurons[0::2], transform=1, synapse=None)
nengo.Connection(u,
a.neurons[1::2],
transform=-1,
synapse=None)
else:
k = input_shape.rows * input_shape.cols
nengo.Connection(u, a.neurons[:k], transform=1, synapse=None)
nengo.Connection(u, a.neurons[k:], transform=-1, synapse=None)
filters = np.vstack([filters, -filters])
else:
raise ValueError("Test not configured for more than two channels")
conv2d_transform = Conv2D.from_kernel(filters,
input_shape,
strides=strides)
output_shape = conv2d_transform.output_shape
gain, bias = neuron_type.gain_bias(max_rates=100, intercepts=0)
gain = gain * 0.01 # account for `a` max_rates
b = nengo.Ensemble(output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([gain[0]]),
bias=nengo.dists.Choice([bias[0]]),
label='b')
nengo.Connection(a.neurons,
b.neurons,
synapse=tau_s,
transform=conv2d_transform)
bp = nengo.Probe(b.neurons)
with nengo.Simulator(model, dt=dt, optimize=False) as sim:
sim.run(pres_time)
ref_out = sim.data[bp].mean(axis=0).reshape(output_shape.shape())
# Currently, default TensorFlow does not support channels first in conv
use_nengo_dl = nengo_dl is not None and channels_last
ndl_out = np.zeros_like(ref_out)
if use_nengo_dl:
with nengo_dl.Simulator(model, dt=dt) as sim:
sim.run(pres_time)
ndl_out = sim.data[bp].mean(axis=0).reshape(output_shape.shape())
with nengo_loihi.Simulator(model, dt=dt, target='simreal') as sim:
sim.run(pres_time)
real_out = sim.data[bp].mean(axis=0).reshape(output_shape.shape())
with Simulator(model, dt=dt) as sim:
sim.run(pres_time)
sim_out = sim.data[bp].mean(axis=0).reshape(output_shape.shape())
if not output_shape.channels_last:
ref_out = np.transpose(ref_out, (1, 2, 0))
ndl_out = np.transpose(ndl_out, (1, 2, 0))
real_out = np.transpose(real_out, (1, 2, 0))
sim_out = np.transpose(sim_out, (1, 2, 0))
out_max = max(ref_out.max(), sim_out.max())
# --- plot results
rows = 2
cols = 3
ax = plt.subplot(rows, cols, 1)
imshow(test_x, vmin=0, vmax=1, ax=ax)
ax = plt.subplot(rows, cols, 2)
tile(np.transpose(filters[0], (2, 0, 1)), cols=8, ax=ax)
ax = plt.subplot(rows, cols, 3)
plt.hist(ref_out.ravel(), bins=31)
plt.hist(sim_out.ravel(), bins=31)
ax = plt.subplot(rows, cols, 4)
tile(np.transpose(ref_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 5)
tile(np.transpose(ndl_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
ax = plt.subplot(rows, cols, 6)
tile(np.transpose(sim_out, (2, 0, 1)), vmin=0, vmax=out_max, cols=8, ax=ax)
if use_nengo_dl:
assert allclose(ndl_out, ref_out, atol=1e-5, rtol=1e-5)
assert allclose(real_out, ref_out, atol=1, rtol=1e-3)
assert allclose(sim_out, ref_out, atol=10, rtol=1e-3)
with Simulator(model, dt=dt) as sim:
sim.run(1.0)
est_rates = sim.data[ap].mean(axis=0)
ref_rates = loihi_rates(neuron_type, x, gain, bias, dt=dt)
plt.plot(x, ref_rates, "k", label="predicted")
plt.plot(x, est_rates, "g", label="measured")
plt.legend(loc='best')
assert allclose(est_rates, ref_rates, atol=1, rtol=0, xtol=1)
@pytest.mark.parametrize('neuron_type', [
LoihiLIF(),
LoihiSpikingRectifiedLinear(),
])
def test_loihi_neurons(neuron_type, Simulator, plt, allclose):
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
a = nengo.Ensemble(n,
1,
# First time does not warn
with pytest.warns(None) as record:
NoiseBuilder.register(int)(MockNoiseBuilder)
assert len(record) == 0
# Second time warns
with pytest.warns(UserWarning, match="already has a builder"):
NoiseBuilder.register(int)(MockNoiseBuilder)
@pytest.mark.skipif(not reqs.HAS_NENGO_DL, reason="requires nengo-dl")
@pytest.mark.skipif(not reqs.HAS_NENGO_LOIHI, reason="requires nengo-loihi")
@pytest.mark.filterwarnings("ignore:divide by zero")
@pytest.mark.filterwarnings("ignore:invalid value")
@pytest.mark.filterwarnings("ignore:Non-finite values detected")
@pytest.mark.parametrize("neuron_type", [LoihiLIF(), LoihiSpikingRectifiedLinear()])
@pytest.mark.parametrize("inference_only", (True, False))
def test_nengo_dl_neurons(neuron_type, inference_only, Simulator, plt, allclose):
install_dl_builders()
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
def test_loihi_rates_other_type(allclose):
"""Test using a neuron type that has no Loihi-specific implementation"""
neuron_type = nengo.neurons.Sigmoid()
x = np.linspace(-7, 10)
gain, bias = 0.2, 0.4
dt = 0.002
ref_rates = nengo_rates(neuron_type, x, gain, bias)
rates = loihi_rates(neuron_type, x, gain, bias, dt)
assert ref_rates.shape == rates.shape
assert allclose(rates, ref_rates)
@pytest.mark.parametrize(
"neuron_type", [LoihiLIF(), LoihiSpikingRectifiedLinear()])
def test_loihi_neurons(neuron_type, Simulator, plt, allclose):
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
ens = nengo.Ensemble(n,
1,
neuron_type=neuron_type,
nengo.RegularSpiking(nengo.Sigmoid()),
],
)
def test_loihi_rates_other_type(neuron_type, allclose):
"""Test using a neuron type that has no Loihi-specific implementation"""
x = np.linspace(-7, 10)
gain, bias = 0.2, 0.4
dt = 0.002
ref_rates = nengo_rates(neuron_type, x, gain, bias)
rates = loihi_rates(neuron_type, x, gain, bias, dt)
assert ref_rates.shape == rates.shape
assert allclose(rates, ref_rates)
@pytest.mark.parametrize(
"neuron_type", [LoihiLIF(), LoihiSpikingRectifiedLinear()])
def test_loihi_neurons(neuron_type, Simulator, plt, allclose):
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
ens = nengo.Ensemble(n,
1,
neuron_type=neuron_type,
def test_conv_overlap_input(Simulator, plt):
"""Tests a fully on-chip conv connection. """
conv2d = pytest.importorskip("nengo._vendor.npconv2d.conv2d")
kernel = np.array([[-1, 2, -1], [-1, 2, -1], [-1, 2, -1]], dtype=float)
kernel /= kernel.max()
image = np.array(
[
[1, 2, 1, 2, 0],
[2, 3, 2, 1, 1],
[1, 2, 1, 2, 3],
[2, 3, 2, 1, 1],
[1, 2, 1, 2, 0],
],
dtype=float,
)
image /= image.max()
input_scale = 119.0
bias = input_scale * image.ravel()
neuron_type = nengo.SpikingRectifiedLinear()
y_ref = LoihiSpikingRectifiedLinear().rates(image.ravel(), input_scale, 0)
y_ref = conv2d.conv2d(y_ref.reshape((1, 5, 5, 1)),
kernel.reshape((3, 3, 1, 1)),
pad="VALID")
y_ref = LoihiSpikingRectifiedLinear().rates(y_ref.ravel(), 1.0,
0.0).reshape((3, 3))
with nengo.Network() as net:
a = nengo.Ensemble(
bias.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([0]),
bias=bias,
)
transform = nengo_transforms.Convolution(n_filters=1,
input_shape=(4, 5, 1),
init=kernel.reshape(
(3, 3, 1, 1)))
b0 = nengo.Ensemble(
transform.output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([1]),
bias=nengo.dists.Choice([0]),
)
b1 = nengo.Ensemble(
transform.output_shape.size,
1,
neuron_type=neuron_type,
gain=nengo.dists.Choice([1]),
bias=nengo.dists.Choice([0]),
)
nengo.Connection(a.neurons[:20], b0.neurons, transform=transform)
nengo.Connection(a.neurons[5:], b1.neurons, transform=transform)
b0p = nengo.Probe(b0.neurons, synapse=nengo.Alpha(0.02))
b1p = nengo.Probe(b1.neurons, synapse=nengo.Alpha(0.02))
with Simulator(net) as sim:
sim.run(0.3)
y_ref = y_ref / input_scale
y0 = sim.data[b0p][-1].reshape((2, -1)) / input_scale
y1 = sim.data[b1p][-1].reshape((2, -1)) / input_scale
plt.subplot(131)
plt.imshow(y_ref)
plt.colorbar()
plt.subplot(132)
plt.imshow(b0)
plt.colorbar()
plt.subplot(133)
plt.imshow(b1)
plt.colorbar()
assert np.allclose(y0, y_ref[:2], atol=0.02, rtol=0.1)
assert np.allclose(y1, y_ref[1:], atol=0.02, rtol=0.1)
def test_loihi_rates_other_type(allclose):
"""Test using a neuron type that has no Loihi-specific implementation"""
neuron_type = nengo.neurons.Sigmoid()
x = np.linspace(-7, 10)
gain, bias = 0.2, 0.4
dt = 0.002
ref_rates = neuron_type.rates(x, gain, bias)
rates = loihi_rates(neuron_type, x, gain, bias, dt)
assert allclose(rates, ref_rates)
@pytest.mark.parametrize('neuron_type', [
LoihiLIF(),
LoihiSpikingRectifiedLinear(),
])
def test_loihi_neurons(neuron_type, Simulator, plt, allclose):
dt = 0.0007
n = 256
encoders = np.ones((n, 1))
gain = np.zeros(n)
if isinstance(neuron_type, nengo.SpikingRectifiedLinear):
bias = np.linspace(0, 1001, n)
else:
bias = np.linspace(0, 30, n)
with nengo.Network() as model:
ens = nengo.Ensemble(n,
1,
if __name__ == "__main__":
backend = "loihi" # can be ["cpu"|"loihi"]
sim_runtime = 10 # simulated seconds
collect_ground_truth = True # for plotting comparison
weights = sys.argv[1] if len(sys.argv) > 1 else "data/reference_weights"
if weights[-4:] == ".npz":
weights = weights[:-4]
net = demo(
backend=backend,
collect_ground_truth=collect_ground_truth,
motor_neuron_type=LoihiSpikingRectifiedLinear(),
neural_vision=True,
plot_mounted_camera_freq=None, # how often to plot image from cameras
weights_name=weights,
)
try:
if backend == "loihi":
sim = nengo_loihi.Simulator(
net,
target="sim", # set equal to "loihi" to run on Loihi hardware
hardware_options=dict(snip_max_spikes_per_step=300),
)
print(sim.model.utilization_summary())
elif backend == "cpu":
sim = nengo.Simulator(net, progress_bar=False)