def test_tensor_layer(Simulator):
with nengo.Network() as net:
inp = nengo.Node(np.arange(12))
# check that connection arguments work
layer0 = tensor_layer(inp, tf.identity, transform=2)
assert isinstance(layer0, TensorNode)
p0 = nengo.Probe(layer0)
# check that arguments are passed to layer function
layer1 = tensor_layer(layer0,
lambda x, axis: tf.reduce_sum(x, axis=axis),
axis=1,
shape_in=(2, 6))
assert layer1.size_out == 6
p1 = nengo.Probe(layer1)
# check that ensemble layers work
layer2 = tensor_layer(layer1,
nengo.RectifiedLinear(),
gain=[1] * 6,
bias=[-20] * 6)
assert isinstance(layer2, nengo.ensemble.Neurons)
assert np.allclose(layer2.ensemble.gain, 1)
assert np.allclose(layer2.ensemble.bias, -20)
p2 = nengo.Probe(layer2)
# check that size_in can be inferred from transform
layer3 = tensor_layer(layer2, lambda x: x, transform=np.ones((1, 6)))
assert layer3.size_in == 1
# check that size_in can be inferred from shape_in
layer4 = tensor_layer(layer3,
lambda x: x,
transform=nengo.dists.Uniform(-1, 1),
shape_in=(2, ))
assert layer4.size_in == 2
with Simulator(net, minibatch_size=2) as sim:
sim.step()
x = np.arange(12) * 2
assert np.allclose(sim.data[p0], x)
x = np.sum(np.reshape(x, (2, 6)), axis=0)
assert np.allclose(sim.data[p1], x)
x = np.maximum(x - 20, 0)
assert np.allclose(sim.data[p2], x)
def convert_dense(self, model, pre_layer, input_shape, index,
onnx_model_graph):
onnx_model_graph_node = onnx_model_graph.node
node_info = onnx_model_graph_node[index]
dense_num = self.get_dense_num(node_info, onnx_model_graph)
neuron_type = self.get_neuronType(index, onnx_model_graph_node)
with model:
x = nengo_dl.tensor_layer(pre_layer,
tf.layers.dense,
units=dense_num)
if neuron_type != "softmax":
if neuron_type == "lif":
x = nengo_dl.tensor_layer(
x, nengo.LIF(amplitude=self.amplitude))
elif neuron_type == "lifrate":
x = nengo_dl.tensor_layer(
x, nengo.LIFRate(amplitude=self.amplitude))
elif neuron_type == "adaptivelif":
x = nengo_dl.tensor_layer(
x, nengo.AdaptiveLIF(amplitude=self.amplitude))
elif neuron_type == "adaptivelifrate":
x = nengo_dl.tensor_layer(
x, nengo.AdaptiveLIFRate(amplitude=self.amplitude))
elif neuron_type == "izhikevich":
x = nengo_dl.tensor_layer(
x, nengo.Izhikevich(amplitude=self.amplitude))
elif neuron_type == "softlifrate":
x = nengo_dl.tensor_layer(
x,
nengo_dl.neurons.SoftLIFRate(amplitude=self.amplitude))
elif neuron_type == None: #default neuron_type = LIF
x = nengo_dl.tensor_layer(
x, nengo.LIF(amplitude=self.amplitude))
output_shape = [dense_num, 1]
return model, output_shape, x
def test_tensor_layer(Simulator):
with nengo.Network() as net:
inp = nengo.Node(np.arange(12))
# check that connection arguments work
layer0 = tensor_layer(inp, tf.identity, transform=2)
assert isinstance(layer0, TensorNode)
p0 = nengo.Probe(layer0)
# check that arguments are passed to layer function
layer1 = tensor_layer(
layer0, lambda x, axis: tf.reduce_sum(x, axis=axis), axis=1,
shape_in=(2, 6))
assert layer1.size_out == 6
p1 = nengo.Probe(layer1)
# check that ensemble layers work
layer2 = tensor_layer(layer1, nengo.RectifiedLinear(), gain=[1] * 6,
bias=[-20] * 6)
assert isinstance(layer2, nengo.ensemble.Neurons)
assert np.allclose(layer2.ensemble.gain, 1)
assert np.allclose(layer2.ensemble.bias, -20)
p2 = nengo.Probe(layer2)
# check that size_in can be inferred from transform
layer3 = tensor_layer(layer2, lambda x: x,
transform=np.ones((1, 6)))
assert layer3.size_in == 1
# check that size_in can be inferred from shape_in
layer4 = tensor_layer(
layer3, lambda x: x, transform=nengo.dists.Uniform(-1, 1),
shape_in=(2,))
assert layer4.size_in == 2
with Simulator(net, minibatch_size=2) as sim:
sim.step()
x = np.arange(12) * 2
assert np.allclose(sim.data[p0], x)
x = np.sum(np.reshape(x, (2, 6)), axis=0)
assert np.allclose(sim.data[p1], x)
x = np.maximum(x - 20, 0)
assert np.allclose(sim.data[p2], x)
def convert_flatten(self, model, pre_layer, input_shape):
with model:
x = nengo_dl.tensor_layer(pre_layer, tf.layers.flatten)
output_shape = 1
for index in range(len(input_shape)):
output_shape *= input_shape[index]
output_shape = [output_shape, 1]
return model, output_shape, x
def convert_conv2d(self, model, pre_layer, input_shape, index,
onnx_model_graph):
onnx_model_graph_node = onnx_model_graph.node
node_info = onnx_model_graph_node[index]
neuron_type = self.get_neuronType(index, onnx_model_graph_node)
filters = self.get_filterNum(node_info, onnx_model_graph)
for index in range(len(node_info.attribute)):
if node_info.attribute[index].name == "kernel_shape":
kernel_size = node_info.attribute[index].ints[0]
elif node_info.attribute[index].name == "strides":
strides = node_info.attribute[index].ints[0]
elif node_info.attribute[index].name == "auto_pad":
padding = node_info.attribute[index].s.decode('ascii').lower()
if padding != "valid":
padding = "same"
if padding == "same":
output_shape = [input_shape[0], input_shape[1], filters]
else:
output_shape = [
int((input_shape[0] - kernel_size) / strides + 1),
int((input_shape[1] - kernel_size) / strides + 1), filters
]
with model:
x = nengo_dl.tensor_layer(pre_layer,
tf.layers.conv2d,
shape_in=(input_shape[0], input_shape[1],
input_shape[2]),
filters=filters,
kernel_size=kernel_size,
padding=padding)
if neuron_type == "lif":
x = nengo_dl.tensor_layer(x,
nengo.LIF(amplitude=self.amplitude))
elif neuron_type == "lifrate":
x = nengo_dl.tensor_layer(
x, nengo.LIFRate(amplitude=self.amplitude))
elif neuron_type == "adaptivelif":
x = nengo_dl.tensor_layer(
x, nengo.AdaptiveLIF(amplitude=self.amplitude))
elif neuron_type == "adaptivelifrate":
x = nengo_dl.tensor_layer(
x, nengo.AdaptiveLIFRate(amplitude=self.amplitude))
elif neuron_type == "izhikevich":
x = nengo_dl.tensor_layer(
x, nengo.Izhikevich(amplitude=self.amplitude))
elif neuron_type == "softlifrate":
x = nengo_dl.tensor_layer(
x, nengo_dl.neurons.SoftLIFRate(amplitude=self.amplitude))
elif neuron_type == None: #default neuron_type = LIF
x = nengo_dl.tensor_layer(x,
nengo.LIF(amplitude=self.amplitude))
return model, output_shape, x
def test_tensor_layer_deprecation(Simulator):
with nengo.Network() as net:
inp = nengo.Node([0])
with pytest.warns(DeprecationWarning, match="nengo_dl.Layer instead"):
out = tensor_layer(inp, lambda x: x + 1)
p = nengo.Probe(out)
with Simulator(net) as sim:
sim.run_steps(5)
assert np.allclose(sim.data[p], 1)
def _build(self, num_classes, num_layers, num_filters, kernel_sizes):
channel_each_layer = ([1] + num_filters)
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude = 0.01)
# init state, make neural trainable
nengo_dl.configure_settings(trainable = True)
inp = nengo.Node([0] * self.input_size[0] * self.input_size[1])
x = nengo_dl.tensor_layer(inp, tf.layers.conv2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[0]),
filters = num_filters[0], kernel_size = kernel_sizes[0], padding = 'same')
x = nengo_dl.tensor_layer(x, neuron_type)
for i in range(1, num_layers):
x = nengo_dl.tensor_layer(x, tf.layers.conv2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[i]),
filters = num_filters[i], kernel_size = kernel_sizes[i], padding = 'same')
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d,
shape_in = (self.input_size[0], self.input_size[1], channel_each_layer[-1]),
pool_size = (self.input_size[0], self.input_size[1]),
strides = (self.input_size[0], self.input_size[1]))
x = nengo_dl.tensor_layer(x, tf.layers.dense, units = num_classes)
out_p = nengo.Probe(x)
out_p_filt = nengo.Probe(x, synapse = self.synapse)
return inp, out_p, out_p_filt, net
def build_network(neuron_type, ens_params):
with nengo.Network() as net:
nengo_dl.configure_settings(trainable=False)
inp = nengo.Node([0] * 28 * 28)
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 64),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(10, 10, 128),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
return net, inp, x
def test_tensor_layer(Simulator):
with nengo.Network() as net:
inp = nengo.Node(np.arange(12))
layer0 = tensor_layer(inp, tf.identity, transform=2)
assert isinstance(layer0, TensorNode)
p0 = nengo.Probe(layer0)
layer1 = tensor_layer(layer0,
lambda x, axis: tf.reduce_sum(x, axis=axis),
axis=1,
shape_in=(2, 6))
assert layer1.size_out == 6
p1 = nengo.Probe(layer1)
layer2 = tensor_layer(layer1,
nengo.RectifiedLinear(),
gain=[1] * 6,
bias=[-20] * 6)
assert isinstance(layer2, nengo.ensemble.Neurons)
assert np.allclose(layer2.ensemble.gain, 1)
assert np.allclose(layer2.ensemble.bias, -20)
p2 = nengo.Probe(layer2)
with Simulator(net, minibatch_size=2) as sim:
sim.step()
x = np.arange(12) * 2
assert np.allclose(sim.data[p0], x)
x = np.sum(np.reshape(x, (2, 6)), axis=0)
assert np.allclose(sim.data[p1], x)
x = np.maximum(x - 20, 0)
assert np.allclose(sim.data[p2], x)
def convert_avgpool2d(self, model, pre_layer, input_shape, node_info):
for index in range(len(node_info.attribute)):
if node_info.attribute[index].name == "kernel_shape":
pool_size = node_info.attribute[index].ints[0]
elif node_info.attribute[index].name == "strides":
strides = node_info.attribute[index].ints[0]
output_shape = [
int(input_shape[0] / strides),
int(input_shape[1] / strides), input_shape[2]
]
with model:
x = nengo_dl.tensor_layer(pre_layer,
tf.layers.average_pooling2d,
shape_in=(input_shape[0], input_shape[1],
input_shape[2]),
pool_size=pool_size,
strides=strides)
return model, output_shape, x
def test_check_gradients_error(Simulator):
# check_gradients detects nans in gradient
with nengo.Network() as net:
x = nengo.Node([0])
y = tensor_layer(x, lambda x: 1 / x)
nengo.Probe(y)
with Simulator(net) as sim:
with pytest.raises(SimulationError):
sim.check_gradients()
# check_gradients detects errors in gradient (in this case caused by the
# fact that nengo.Alpha doesn't have a TensorFlow implementation)
with nengo.Network() as net:
x = nengo.Node([0])
nengo.Probe(x, synapse=nengo.Alpha(0.1))
with Simulator(net) as sim:
with pytest.raises(SimulationError):
sim.check_gradients()
def convert_batchnormalization2d(self, model, pre_layer, input_shape,
node_info):
for index in range(len(node_info.attribute)):
if node_info.attribute[index].name == "momentum":
momentum = round(node_info.attribute[index].f, 4)
if momentum == 0:
momentum = 0.99
elif node_info.attribute[index].name == "epsilon":
epsilon = round(node_info.attribute[index].f, 4)
if epsilon == 0:
epsilon = 0.001
with model:
x = nengo_dl.tensor_layer(pre_layer,
tf.layers.batch_normalization,
shape_in=(input_shape[0], input_shape[1],
input_shape[2]),
momentum=momentum,
epsilon=epsilon)
output_shape = input_shape
return model, output_shape, x
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
inp = nengo.Node([0] * 64 * 64)
# add the first convolutional layer
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(64, 64, 1),
filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(62, 62, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# add a pooling layer
def _build_net(self):
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
self.inp = nengo.Node([0] * 28 * 28, label='input')
# add the first convolutional layer
x = nengo_dl.tensor_layer(self.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x.label = 'conv1_pre'
x.conn_type = 'conv2d'
x = nengo_dl.tensor_layer(x, tf.identity)
x.label = 'conv1'
x.conn_type = 'identity'
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x.label = 'conv2_pre'
x.conn_type = 'conv2d'
x = nengo_dl.tensor_layer(x, tf.identity)
x.label = 'conv2'
x.conn_type = 'identity'
x = nengo_dl.tensor_layer(x, neuron_type)
# add a pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 64),
pool_size=2,
strides=2)
x.label = 'pool1_pre'
x.conn_type = 'average_pooling2d'
x = nengo_dl.tensor_layer(x, tf.identity)
x.label = 'pool1'
x.conn_type = 'identity'
# another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x.label = 'conv3_pre'
x.conn_type = 'conv2d'
x = nengo_dl.tensor_layer(x, tf.identity)
x.label = 'conv3'
x.conn_type = 'identity'
x = nengo_dl.tensor_layer(x, neuron_type)
x.label = 'conv3_act'
# another pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(10, 10, 128),
pool_size=2,
strides=2)
x.label = 'pool2_pre'
x.conn_type = 'average_pooling2d'
x = nengo_dl.tensor_layer(x, tf.identity)
x.label = 'pool2'
x.conn_type = 'identity'
# linear readout
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
x.label = 'output_pre'
x.conn_type = 'dense'
x = nengo_dl.tensor_layer(
x,
tf.identity) # TODO maybe use nengo passthrough nodes instead?
x.label = 'output'
# x = nengo_dl.tensor_layer(x, tf.identity)
# x.label = 'output_id'
# x = x + stim
self.stim = nengo.Node([0] * self.action_space_size, label='stim')
# self.stim_conns = attach_stim(self.stim, x, (np.arange(10), np.arange(15)))
# we'll create two different output probes, one with a filter
# (for when we're simulating the network over time and
# accumulating spikes), and one without (for when we're
# training the network using a rate-based approximation)
self.out_p = nengo.Probe(x)
self.out_p_filt = nengo.Probe(x, synapse=0.1)
# probe stimulated sites
net_nodes = [node.label for node in net.nodes]
for node in self.attach_at.keys():
self.probes.append(
nengo.Probe(net.nodes[net_nodes.index(node)]))
return net
def mnist(use_tensor_layer=True):
"""
A network designed to stress-test tensor layers (based on mnist net).
Parameters
----------
use_tensor_layer : bool
If True, use individual tensor_layers to build the network, as opposed
to a single TensorNode containing all layers.
Returns
-------
net : `nengo.Network`
benchmark network
"""
with nengo.Network() as net:
# create node to feed in images
net.inp = nengo.Node(np.ones(28 * 28))
if use_tensor_layer:
nengo_nl = nengo.RectifiedLinear()
ensemble_params = dict(max_rates=nengo.dists.Choice([100]),
intercepts=nengo.dists.Choice([0]))
amplitude = 1
synapse = None
x = nengo_dl.tensor_layer(net.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, nengo_nl, **ensemble_params)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
transform=amplitude,
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, nengo_nl, **ensemble_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 32),
synapse=synapse,
transform=amplitude,
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=128)
x = nengo_dl.tensor_layer(x, nengo_nl, **ensemble_params)
x = nengo_dl.tensor_layer(x,
tf.layers.dropout,
rate=0.4,
transform=amplitude)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
else:
nl = tf.nn.relu
# def softlif_layer(x, sigma=1, tau_ref=0.002, tau_rc=0.02,
# amplitude=1):
# # x -= 1
# z = tf.nn.softplus(x / sigma) * sigma
# z += 1e-10
# rates = amplitude / (tau_ref + tau_rc * tf.log1p(1 / z))
# return rates
@nengo_dl.reshaped((28, 28, 1))
def mnist_node(_, x): # pragma: no cover
x = tf.layers.conv2d(x,
filters=32,
kernel_size=3,
activation=nl)
x = tf.layers.conv2d(x,
filters=32,
kernel_size=3,
activation=nl)
x = tf.layers.average_pooling2d(x, pool_size=2, strides=2)
x = tf.contrib.layers.flatten(x)
x = tf.layers.dense(x, 128, activation=nl)
x = tf.layers.dropout(x, rate=0.4)
x = tf.layers.dense(x, 10)
return x
node = nengo_dl.TensorNode(mnist_node,
size_in=28 * 28,
size_out=10)
x = node
nengo.Connection(net.inp, node, synapse=None)
net.p = nengo.Probe(x)
return net
def build_network(neuron_type):
with nengo.Network() as net:
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power so we can save some computation
# by ignoring them. note that this doesn't affect the internal
# components of tensornodes, which will always be trainable or
# non-trainable depending on the code written in the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
inp = nengo.Node(nengo.processes.PresentInput(mnist.test.images, 0.2))
# add the first convolutional layer
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=16,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(24, 24, 16),
filters=8,
kernel_size=3)
# add a pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.max_pooling2d,
shape_in=(22, 22, 8),
pool_size=2,
strides=2)
# add a pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.max_pooling2d,
shape_in=(11, 11, 8),
pool_size=2,
strides=2)
# add a dense layer, with neural nonlinearity.
# note that for all-to-all connections like this we can use the
# normal nengo connection transform to implement the weights
# (instead of using a separate tensor_layer). we'll use a
# Glorot uniform distribution to initialize the weights.
x, conn = nengo_dl.tensor_layer(x,
neuron_type,
**ens_params,
transform=nengo_dl.dists.Glorot(),
shape_in=(128, ),
return_conn=True)
# we need to set the weights and biases to be trainable
# (since we set the default to be trainable=False)
# note: we used return_conn=True above so that we could access
# the connection object for this reason.
net.config[x].trainable = True
net.config[conn].trainable = True
# add a dropout layer
x = nengo_dl.tensor_layer(x, tf.layers.dropout, rate=0.3)
# the final 10 dimensional class output
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
return net, inp, x
def spiking_mnist(ctx, n_epochs):
load = ctx.obj["load"]
reps = ctx.obj["reps"]
neuron_type = nengo.LIF(amplitude=0.01)
ens_params = dict(max_rates=nengo.dists.Choice([100]),
intercepts=nengo.dists.Choice([0]))
minibatch_size = 200
n_steps = 50
with nengo.Network() as net:
nengo_dl.configure_settings(trainable=False)
inp = nengo.Node([0] * 28 * 28)
x = nengo_dl.tensor_layer(inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(24, 24, 64),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(10, 10, 128),
pool_size=2,
strides=2)
out = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
out_p = nengo.Probe(out)
spk_out_p = nengo.Probe(out, synapse=0.1)
if load:
with open("spiking_mnist_data_saved.pkl", "rb") as f:
results = pickle.load(f)
else:
results = {"pre": [], "post": [], "spiking": []}
urlretrieve("http://deeplearning.net/data/mnist/mnist.pkl.gz",
"mnist.pkl.gz")
with gzip.open("mnist.pkl.gz") as f:
train_data, _, test_data = pickle.load(f, encoding="latin1")
train_data = list(train_data)
test_data = list(test_data)
for data in (train_data, test_data):
one_hot = np.zeros((data[0].shape[0], 10))
one_hot[np.arange(data[0].shape[0]), data[1]] = 1
data[1] = one_hot
train_data = {
inp: train_data[0][:, None, :],
out_p: train_data[1][:, None, :]
}
test_data = {
inp: test_data[0][:, None, :],
out_p: test_data[1][:, None, :]
}
test_data_time = {
inp: np.tile(test_data[inp], (1, n_steps, 1)),
spk_out_p: np.tile(test_data[out_p], (1, n_steps, 1))
}
for _ in range(reps):
# construct the simulator
with nengo_dl.Simulator(net, minibatch_size=minibatch_size) as sim:
def objective(x, y):
return tf.nn.softmax_cross_entropy_with_logits_v2(logits=x,
labels=y)
opt = tf.train.RMSPropOptimizer(learning_rate=0.001)
def classification_error(outputs, targets):
return 100 * tf.reduce_mean(
tf.cast(
tf.not_equal(tf.argmax(outputs[:, -1], axis=-1),
tf.argmax(targets[:, -1], axis=-1)),
tf.float32))
# collect error before training
results["pre"].append(
sim.loss(test_data, {out_p: classification_error},
training=True))
print("error before training: %.2f%%" % results["pre"][-1])
# run training
sim.train(train_data,
opt,
objective={out_p: objective},
n_epochs=n_epochs)
# collect error after training
results["post"].append(
sim.loss(test_data, {out_p: classification_error},
training=True))
print("error after training: %.2f%%" % results["post"][-1])
# collect spiking error
results["spiking"].append(
sim.loss(test_data_time, {spk_out_p: classification_error},
training=False))
print("spiking neuron error: %.2f%%" % results["spiking"][-1])
with open("spiking_mnist_data.pkl", "wb") as f:
pickle.dump(results, f)
print("pre", bootstrap_ci(results["pre"]))
print("post", bootstrap_ci(results["post"]))
print("spiking", bootstrap_ci(results["spiking"]))
sim.run(1.0, data={a: np.random.randn(10, 1000, 1)})
print(sim.data[p])
# snippet 5 (section 3.2)
import tensorflow as tf
inputs = np.random.randn(50, 1000, 1)
targets = inputs**2
with nengo_dl.Simulator(net, minibatch_size=10) as sim:
sim.train(
data={a: inputs,
p: targets},
optimizer=tf.train.AdamOptimizer(),
n_epochs=2,
objective={p: nengo_dl.objectives.mse})
# snippet 6 (section 3.3)
with net:
def tensor_func(time, x):
return tf.layers.dense(x, 100, activation=tf.nn.relu)
t = nengo_dl.TensorNode(tensor_func, size_in=1)
nengo.Connection(a, t)
nengo.Connection(t, b.neurons)
# snippet 7 (section 3.3)
with net:
t = nengo_dl.tensor_layer(a, tf.layers.dense, units=100,
activation=tf.nn.relu)
nengo.Connection(t, b.neurons)
# num_test, _, _, _ = test_features.shape
# num_classes = len(np.unique(train_labels))
# #pre-processing
# train_features = train_features.astype('float32')/255
# test_features = test_features.astype('float32')/255
# # convert class labels to binary class labels
# train_labels = np_utils.to_categorical(train_labels, num_classes)
# test_labels = np_utils.to_categorical(test_labels, num_classes)
with nengo.Network() as net:
neuron_type = nengo.LIF(amplitude=0.001)
inp = nengo.Node([0]*3*32*32)
nengo_dl.configure_settings(trainable=False)
x = nengo_dl.tensor_layer(inp, tf.layers.conv2d, shape_in=(32,32,3), filters=32, kernel_size=3, use_bias=True)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.dropout, rate=0.1)
x = nengo_dl.tensor_layer(x, tf.layers.conv2d,shape_in=(30,30,32),filters=64, kernel_size=5, strides=2, use_bias=False)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.dropout,rate=0.2)
x = nengo_dl.tensor_layer(x, tf.layers.flatten)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=128)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.dropout,rate=0.3)
# snippet 5 (section 3.2)
import tensorflow as tf
inputs = np.random.randn(50, 1000, 1)
targets = inputs**2
with nengo_dl.Simulator(net, minibatch_size=10) as sim:
sim.train(inputs={a: inputs},
targets={p: targets},
optimizer=tf.train.AdamOptimizer(),
n_epochs=2,
objective="mse")
# snippet 6 (section 3.3)
with net:
def tensor_func(t, x):
return tf.layers.dense(x, 100, activation=tf.nn.relu)
t = nengo_dl.TensorNode(tensor_func, size_in=1)
nengo.Connection(a, t)
nengo.Connection(t, b.neurons)
# snippet 7 (section 3.3)
with net:
t = nengo_dl.tensor_layer(a,
tf.layers.dense,
units=100,
activation=tf.nn.relu)
nengo.Connection(t, b.neurons)
def mnist(use_tensor_layer=True):
"""
A network designed to stress-test tensor layers (based on mnist net).
Parameters
----------
use_tensor_layer : bool
If True, use individual tensor_layers to build the network, as opposed
to a single TensorNode containing all layers.
Returns
-------
net : `nengo.Network`
benchmark network
"""
with nengo.Network() as net:
# create node to feed in images
net.inp = nengo.Node(np.ones(28 * 28))
if use_tensor_layer:
nengo_nl = nengo.RectifiedLinear()
ensemble_params = dict(max_rates=nengo.dists.Choice([100]),
intercepts=nengo.dists.Choice([0]))
amplitude = 1
synapse = None
x = nengo_dl.tensor_layer(
net.inp, tf.layers.conv2d, shape_in=(28, 28, 1), filters=32,
kernel_size=3
)
x = nengo_dl.tensor_layer(x, nengo_nl,
**ensemble_params)
x = nengo_dl.tensor_layer(
x, tf.layers.conv2d, shape_in=(26, 26, 32),
transform=amplitude, filters=32, kernel_size=3
)
x = nengo_dl.tensor_layer(x, nengo_nl,
**ensemble_params)
x = nengo_dl.tensor_layer(
x, tf.layers.average_pooling2d, shape_in=(24, 24, 32),
synapse=synapse, transform=amplitude, pool_size=2, strides=2)
x = nengo_dl.tensor_layer(
x, tf.layers.dense, units=128
)
x = nengo_dl.tensor_layer(x, nengo_nl,
**ensemble_params)
x = nengo_dl.tensor_layer(x, tf.layers.dropout, rate=0.4,
transform=amplitude)
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
else:
nl = tf.nn.relu
# def softlif_layer(x, sigma=1, tau_ref=0.002, tau_rc=0.02,
# amplitude=1):
# # x -= 1
# z = tf.nn.softplus(x / sigma) * sigma
# z += 1e-10
# rates = amplitude / (tau_ref + tau_rc * tf.log1p(1 / z))
# return rates
@nengo_dl.reshaped((28, 28, 1))
def mnist_node(_, x): # pragma: no cover
x = tf.layers.conv2d(x, filters=32, kernel_size=3,
activation=nl)
x = tf.layers.conv2d(x, filters=32, kernel_size=3,
activation=nl)
x = tf.layers.average_pooling2d(x, pool_size=2, strides=2)
x = tf.contrib.layers.flatten(x)
x = tf.layers.dense(x, 128, activation=nl)
x = tf.layers.dropout(x, rate=0.4)
x = tf.layers.dense(x, 10)
return x
node = nengo_dl.TensorNode(mnist_node, size_in=28 * 28,
size_out=10)
x = node
nengo.Connection(net.inp, node, synapse=None)
net.p = nengo.Probe(x)
return net
def build_network(neuron_type,
drop_p,
l2_weight,
n_units=1024,
num_layers=4,
output_size=1):
with nengo.Network() as net:
use_dropout = False
if drop_p:
use_dropout = True
#net.config[nengo.Connection].synapse = None
#nengo_dl.configure_settings(trainable=False)
# input node
inp = nengo.Node([0])
shape_in = 1
x = inp
# the regularizer is a function, so why not reuse it
reg = tf.contrib.layers.l2_regularizer(l2_weight)
class DenseLayer(object):
i = 0
def pre_build(self, shape_in, shape_out):
self.W = tf.get_variable("weights" + str(DenseLayer.i),
shape=(shape_in[1], shape_out[1]),
regularizer=reg)
self.B = tf.get_variable("biases" + str(DenseLayer.i),
shape=(1, shape_out[1]),
regularizer=reg)
DenseLayer.i += 1
def __call__(self, t, x):
return x @ self.W + self.B
for n in range(num_layers):
# add a fully connected layer
a = nengo_dl.TensorNode(DenseLayer(),
size_in=shape_in,
size_out=n_units,
label='dense{}'.format(n))
nengo.Connection(x, a, synapse=None)
shape_in = n_units
x = a
# apply an activation function
x = nengo_dl.tensor_layer(x, neuron_type, **ens_params)
# add a dropout layer
x = nengo_dl.tensor_layer(x,
tf.layers.dropout,
rate=drop_p,
training=use_dropout)
# add an output layer
a = nengo_dl.TensorNode(DenseLayer(),
size_in=shape_in,
size_out=output_size)
nengo.Connection(x, a, synapse=None)
return net, inp, a
tf.not_equal(tf.argmax(outputs[:, -1], axis=-1),
tf.argmax(targets[:, -1], axis=-1)), tf.float32))
with nengo.Network(seed=1000) as net:
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
default_neuron_type = nengo.LIF(amplitude=0.01)
nengo_dl.configure_settings(trainable=False)
inp = nengo.Node([0] * 28 * 28 * 1)
x = inp
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=6,
kernel_size=5,
padding="same")
x = nengo_dl.tensor_layer(x, nengo.LIF(amplitude=0.01))
x = nengo_dl.tensor_layer(x,
tf.layers.average_pooling2d,
shape_in=(28, 28, 6),
pool_size=2,
strides=2)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(14, 14, 6),
filters=16,
kernel_size=5,
def _build_net(self):
with nengo.Network() as net:
# set some default parameters for the neurons that will make
# the training progress more smoothly
net.config[nengo.Ensemble].max_rates = nengo.dists.Choice([100])
net.config[nengo.Ensemble].intercepts = nengo.dists.Choice([0])
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
self.inp = nengo.Node([0] * 28 * 28)
# ENCODER
# add the first convolutional layer
x = nengo_dl.tensor_layer(self.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(26, 26, 32),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# add a pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(24, 24, 64),
filters=64,
kernel_size=2,
strides=2)
# x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d, shape_in=(24, 24, 64), pool_size=2, strides=2)
# another convolutional layer
# (W - Fw + 2P) / Sw + 1
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(12, 12, 64),
filters=128,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# another pooling layer
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d,
shape_in=(10, 10, 128),
filters=128,
kernel_size=2,
strides=2)
# x = nengo_dl.tensor_layer(x, tf.layers.average_pooling2d, shape_in=(10, 10, 128), pool_size=2, strides=2)
# latent
self.latent = x
# DECODER
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(5, 5, 128),
filters=64,
kernel_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(12, 12, 64),
filters=64,
kernel_size=3)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(24, 24, 64),
filters=32,
kernel_size=2,
strides=2)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x,
tf.layers.conv2d_transpose,
shape_in=(26, 26, 32),
filters=1,
kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(self.inp,
tf.layers.conv2d,
shape_in=(28, 28, 1),
filters=1,
kernel_size=3)
# linear readout
x = nengo_dl.tensor_layer(x, tf.layers.dense, units=10)
x = nengo_dl.tensor_layer(x, tf.identity)
# x = x + out_stim
self.out_stim = nengo.Node([0] * 10)
self.stim_conns = attach_stim(self.out_stim, x,
(np.arange(10), np.arange(15)))
# we'll create two different output probes, one with a filter
# (for when we're simulating the network over time and
# accumulating spikes), and one without (for when we're
# training the network using a rate-based approximation)
self.out_p = nengo.Probe(x)
self.out_p_filt = nengo.Probe(x, synapse=0.1)
return net
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
inp = nengo.Node([0] * 28 * 28)
# add the first convolutional layer
x = nengo_dl.tensor_layer(
inp, tf.layers.conv2d, shape_in=(28, 28, 1), filters=32,
kernel_size=3)
# apply the neural nonlinearity
x = nengo_dl.tensor_layer(x, neuron_type)
# add another convolutional layer
x = nengo_dl.tensor_layer(
x, tf.layers.conv2d, shape_in=(26, 26, 32),
filters=64, kernel_size=3)
x = nengo_dl.tensor_layer(x, neuron_type)
# add a pooling layer
x = nengo_dl.tensor_layer(
x, tf.layers.average_pooling2d, shape_in=(24, 24, 64),
pool_size=2, strides=2)
neuron_type = nengo.LIF(amplitude=0.01)
# we'll make all the nengo objects in the network
# non-trainable. we could train them if we wanted, but they don't
# add any representational power. note that this doesn't affect
# the internal components of tensornodes, which will always be
# trainable or non-trainable depending on the code written in
# the tensornode.
nengo_dl.configure_settings(trainable=False)
# the input node that will be used to feed in input images
inp = nengo.Node([0] * 28 * 28)
# fully connected encoder
x = nengo_dl.tensor_layer(inp,
tf.layers.dense,
shape_in=(28 * 28, ),
units=32) # TODO try less parameters
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.dense, shape_in=(32, ), units=16)
x = nengo_dl.tensor_layer(x, neuron_type)
x = nengo_dl.tensor_layer(x, tf.layers.dense, shape_in=(16, ), units=8)
latent = nengo_dl.tensor_layer(x, neuron_type)
# fully connected decoder
x = nengo_dl.tensor_layer(latent,
tf.layers.dense,
shape_in=(8, ),
units=16)
x = nengo_dl.tensor_layer(x, neuron_type)
