def main(n_input=1, n_output=10, time=1000):
# Network building.
network = Network(dt=1.0)
input_layer = RealInput(n=n_input)
output_layer = LIFNodes(n=n_output)
connection = Connection(source=input_layer, target=output_layer)
monitor = Monitor(obj=output_layer, state_vars=('v', ), time=time)
# Adding network components.
network.add_layer(input_layer, name='X')
network.add_layer(output_layer, name='Y')
network.add_connection(connection, source='X', target='Y')
network.add_monitor(monitor, name='X_monitor')
# Creating real-valued inputs and running simulation.
inpts = {'X': torch.ones(time, n_input)}
network.run(inpts=inpts, time=time)
# Plot voltage activity.
plt.plot(monitor.get('v').numpy().T)
plt.show()
def __init__(self,
encoder,
dt: float = 1.0,
lag: int = 10,
n_neurons: int = 100,
time: int = 100,
learning: bool = False):
super().__init__(dt=dt)
self.learning = learning
self.n_neurons = n_neurons
self.lag = lag
self.encoder = encoder
self.time = time
for i in range(lag):
self.add_layer(RealInput(n=encoder.e_size, traces=True),
name=f'input_{i+1}')
self.add_layer(LIFNodes(n=self.n_neurons, traces=True),
name=f'column_{i+1}')
self.add_monitor(Monitor(self.layers[f'column_{i+1}'], ['s'],
time=self.time),
name=f'monitor_{i+1}')
w = 0.3 * torch.rand(self.encoder.e_size, self.n_neurons)
self.add_connection(Connection(source=self.layers[f'input_{i+1}'],
target=self.layers[f'column_{i+1}'],
w=w),
source=f'input_{i+1}',
target=f'column_{i+1}')
for i in range(lag):
for j in range(lag):
w = torch.zeros(self.n_neurons, self.n_neurons)
self.add_connection(Connection(
source=self.layers[f'column_{i+1}'],
target=self.layers[f'column_{j+1}'],
w=w,
update_rule=Hebbian,
nu=args.nu),
source=f'column_{i+1}',
target=f'column_{j+1}')
import torch
import matplotlib.pyplot as plt
from bindsnet.network import Network
from bindsnet.datasets import FashionMNIST
from bindsnet.network.monitors import Monitor
from bindsnet.network.topology import Connection
from bindsnet.network.nodes import RealInput, IFNodes
from bindsnet.analysis.plotting import plot_spikes, plot_weights
# Network building.
network = Network()
input_layer = RealInput(n=784, sum_input=True)
output_layer = IFNodes(n=10, sum_input=True)
bias = RealInput(n=1, sum_input=True)
network.add_layer(input_layer, name='X')
network.add_layer(output_layer, name='Y')
network.add_layer(bias, name='Y_b')
input_connection = Connection(source=input_layer, target=output_layer, norm=150, wmin=-1, wmax=1)
bias_connection = Connection(source=bias, target=output_layer)
network.add_connection(input_connection, source='X', target='Y')
network.add_connection(bias_connection, source='Y_b', target='Y')
# State variable monitoring.
time = 25
for l in network.layers:
m = Monitor(network.layers[l], state_vars=['s'], time=time)
network.add_monitor(m, name=l)
# Load Fashion-MNIST data.
def main(seed=0, n_neurons=100, n_train=60000, n_test=10000, inhib=250, time=50, lr=1e-2, lr_decay=0.99, dt=1,
theta_plus=0.05, theta_decay=1e-7, progress_interval=10, update_interval=250, train=True, plot=False,
gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [
seed, n_neurons, n_train, inhib, time, lr, lr_decay,
theta_plus, theta_decay, progress_interval, update_interval
]
test_params = [
seed, n_neurons, n_train, n_test, inhib, time, lr, lr_decay,
theta_plus, theta_decay, progress_interval, update_interval
]
model_name = '_'.join([str(x) for x in params])
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
if train:
n_examples = n_train
else:
n_examples = n_test
n_sqrt = int(np.ceil(np.sqrt(n_neurons)))
n_classes = 10
# Build network.
if train:
network = Network(dt=dt)
input_layer = RealInput(n=784, traces=True, trace_tc=5e-2)
network.add_layer(input_layer, name='X')
output_layer = DiehlAndCookNodes(
n=n_neurons, traces=True, rest=0, reset=0, thresh=1, refrac=0,
decay=1e-2, trace_tc=5e-2, theta_plus=theta_plus, theta_decay=theta_decay
)
network.add_layer(output_layer, name='Y')
w = 0.3 * torch.rand(784, n_neurons)
input_connection = Connection(
source=network.layers['X'], target=network.layers['Y'], w=w, update_rule=PostPre,
nu=[0, lr], wmin=0, wmax=1, norm=78.4
)
network.add_connection(input_connection, source='X', target='Y')
w = -inhib * (torch.ones(n_neurons, n_neurons) - torch.diag(torch.ones(n_neurons)))
recurrent_connection = Connection(
source=network.layers['Y'], target=network.layers['Y'], w=w, wmin=-inhib, wmax=0
)
network.add_connection(recurrent_connection, source='Y', target='Y')
else:
path = os.path.join('..', '..', 'params', data, model)
network = load_network(os.path.join(path, model_name + '.pt'))
network.connections['X', 'Y'].update_rule = NoOp(
connection=network.connections['X', 'Y'], nu=network.connections['X', 'Y'].nu
)
network.layers['Y'].theta_decay = 0
network.layers['Y'].theta_plus = 0
# Load MNIST data.
dataset = MNIST(path=os.path.join('..', '..', 'data', 'MNIST'), download=True)
if train:
images, labels = dataset.get_train()
else:
images, labels = dataset.get_test()
images = images.view(-1, 784)
images = images / 255
# if train:
# for i in range(n_neurons):
# network.connections['X', 'Y'].w[:, i] = images[i] + images[i].mean() * torch.randn(784)
# Record spikes during the simulation.
spike_record = torch.zeros(update_interval, time, n_neurons)
# Neuron assignments and spike proportions.
if train:
assignments = -torch.ones_like(torch.Tensor(n_neurons))
proportions = torch.zeros_like(torch.Tensor(n_neurons, n_classes))
rates = torch.zeros_like(torch.Tensor(n_neurons, n_classes))
ngram_scores = {}
else:
path = os.path.join('..', '..', 'params', data, model)
path = os.path.join(path, '_'.join(['auxiliary', model_name]) + '.pt')
assignments, proportions, rates, ngram_scores = torch.load(open(path, 'rb'))
# Sequence of accuracy estimates.
curves = {'all': [], 'proportion': [], 'ngram': []}
if train:
best_accuracy = 0
spikes = {}
for layer in set(network.layers):
spikes[layer] = Monitor(network.layers[layer], state_vars=['s'], time=time)
network.add_monitor(spikes[layer], name='%s_spikes' % layer)
# Train the network.
if train:
print('\nBegin training.\n')
else:
print('\nBegin test.\n')
inpt_axes = None
inpt_ims = None
spike_ims = None
spike_axes = None
weights_im = None
assigns_im = None
perf_ax = None
start = t()
for i in range(n_examples):
if i % progress_interval == 0:
print(f'Progress: {i} / {n_examples} ({t() - start:.4f} seconds)')
start = t()
if i % update_interval == 0 and i > 0:
if train:
network.connections['X', 'Y'].update_rule.nu[1] *= lr_decay
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(images) - update_interval:i % len(images)]
# Update and print accuracy evaluations.
curves, predictions = update_curves(
curves, current_labels, n_classes, spike_record=spike_record, assignments=assignments,
proportions=proportions, ngram_scores=ngram_scores, n=2
)
print_results(curves)
if train:
if any([x[-1] > best_accuracy for x in curves.values()]):
print('New best accuracy! Saving network parameters to disk.')
# Save network to disk.
path = os.path.join('..', '..', 'params', data, model)
if not os.path.isdir(path):
os.makedirs(path)
network.save(os.path.join(path, model_name + '.pt'))
path = os.path.join(path, '_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores), open(path, 'wb'))
best_accuracy = max([x[-1] for x in curves.values()])
# Assign labels to excitatory layer neurons.
assignments, proportions, rates = assign_labels(spike_record, current_labels, n_classes, rates)
# Compute ngram scores.
ngram_scores = update_ngram_scores(spike_record, current_labels, n_classes, 2, ngram_scores)
print()
# Get next input sample.
image = images[i % n_examples].repeat([time, 1])
inpts = {'X': image}
# Run the network on the input.
network.run(inpts=inpts, time=time)
retries = 0
while spikes['Y'].get('s').sum() < 5 and retries < 3:
retries += 1
image *= 2
inpts = {'X': image}
network.run(inpts=inpts, time=time)
# Add to spikes recording.
spike_record[i % update_interval] = spikes['Y'].get('s').t()
# Optionally plot various simulation information.
if plot and i % update_interval == 0:
_input = images[i % n_examples].view(28, 28)
reconstruction = inpts['X'].view(time, 784).sum(0).view(28, 28)
_spikes = {layer: spikes[layer].get('s') for layer in spikes}
input_exc_weights = network.connections['X', 'Y'].w
square_weights = get_square_weights(input_exc_weights.view(784, n_neurons), n_sqrt, 28)
square_assignments = get_square_assignments(assignments, n_sqrt)
# inpt_axes, inpt_ims = plot_input(_input, reconstruction, label=labels[i], axes=inpt_axes, ims=inpt_ims)
spike_ims, spike_axes = plot_spikes(_spikes, ims=spike_ims, axes=spike_axes)
weights_im = plot_weights(square_weights, im=weights_im)
# assigns_im = plot_assignments(square_assignments, im=assigns_im)
# perf_ax = plot_performance(curves, ax=perf_ax)
plt.pause(1e-1)
network.reset_() # Reset state variables.
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.4f} seconds)')
i += 1
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(images) - update_interval:i % len(images)]
# Update and print accuracy evaluations.
curves, predictions = update_curves(
curves, current_labels, n_classes, spike_record=spike_record, assignments=assignments,
proportions=proportions, ngram_scores=ngram_scores, n=2
)
print_results(curves)
if train:
if any([x[-1] > best_accuracy for x in curves.values()]):
print('New best accuracy! Saving network parameters to disk.')
# Save network to disk.
if train:
path = os.path.join('..', '..', 'params', data, model)
if not os.path.isdir(path):
os.makedirs(path)
network.save(os.path.join(path, model_name + '.pt'))
path = os.path.join(path, '_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores), open(path, 'wb'))
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')
print('Average accuracies:\n')
for scheme in curves.keys():
print('\t%s: %.2f' % (scheme, float(np.mean(curves[scheme]))))
# Save accuracy curves to disk.
path = os.path.join('..', '..', 'curves', data, model)
if not os.path.isdir(path):
os.makedirs(path)
if train:
to_write = ['train'] + params
else:
to_write = ['test'] + params
to_write = [str(x) for x in to_write]
f = '_'.join(to_write) + '.pt'
torch.save((curves, update_interval, n_examples), open(os.path.join(path, f), 'wb'))
# Save results to disk.
path = os.path.join('..', '..', 'results', data, model)
if not os.path.isdir(path):
os.makedirs(path)
results = [
np.mean(curves['all']), np.mean(curves['proportion']), np.mean(curves['ngram']),
np.max(curves['all']), np.max(curves['proportion']), np.max(curves['ngram'])
]
if train:
to_write = params + results
else:
to_write = test_params + results
to_write = [str(x) for x in to_write]
if train:
name = 'train.csv'
else:
name = 'test.csv'
if not os.path.isfile(os.path.join(path, name)):
with open(os.path.join(path, name), 'w') as f:
if train:
f.write('random_seed,n_neurons,n_train,inhib,time,lr,lr_decay,theta_plus,theta_decay,'
'progress_interval,update_interval,mean_all_activity,mean_proportion_weighting,'
'mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n')
else:
f.write('random_seed,n_neurons,n_train,n_test,inhib,time,lr,lr_decay,theta_plus,theta_decay,'
'progress_interval,update_interval,mean_all_activity,mean_proportion_weighting,'
'mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n')
with open(os.path.join(path, name), 'a') as f:
f.write(','.join(to_write) + '\n')
for path in [params_path, curves_path, results_path, confusion_path]:
if not os.path.isdir(path):
os.makedirs(path)
criterion = torch.nn.CrossEntropyLoss(
) # Loss function on output firing rates.
sqrt = int(np.ceil(
np.sqrt(n_hidden))) # Ceiling(square root(no. hidden neurons)).
n_examples = n_train if train else n_test
if train:
# Network building.
network = Network()
# Groups of neurons.
input_layer = RealInput(n=784, sum_input=True)
hidden_layer = IFNodes(n=n_hidden, sum_input=True)
hidden_bias = RealInput(n=1, sum_input=True)
output_layer = IFNodes(n=10, sum_input=True)
output_bias = RealInput(n=1, sum_input=True)
network.add_layer(input_layer, name='X')
network.add_layer(hidden_layer, name='Y')
network.add_layer(hidden_bias, name='Y_b')
network.add_layer(output_layer, name='Z')
network.add_layer(output_bias, name='Z_b')
# Connections between groups of neurons.
input_connection = Connection(source=input_layer, target=hidden_layer)
hidden_bias_connection = Connection(source=hidden_bias,
target=hidden_layer)
hidden_connection = Connection(source=hidden_layer, target=output_layer)
confusion_path = os.path.join(top_level, 'confusion', data, model)
for path in [params_path, curves_path, results_path, confusion_path]:
if not os.path.isdir(path):
os.makedirs(path)
criterion = torch.nn.CrossEntropyLoss(
) # Loss function on output firing rates.
n_examples = n_train if train else n_test
if train:
# Network building.
network = Network()
# Groups of neurons.
input_layer = RealInput(n=50 * 72, sum_input=True)
output_layer = IFNodes(n=4, sum_input=True)
bias = RealInput(n=1, sum_input=True)
network.add_layer(input_layer, name='X')
network.add_layer(output_layer, name='Y')
network.add_layer(bias, name='Y_b')
# Connections between groups of neurons.
input_connection = Connection(source=input_layer,
target=output_layer,
norm=norm,
wmin=wmin,
wmax=wmax)
bias_connection = Connection(source=bias, target=output_layer)
network.add_connection(input_connection, source='X', target='Y')
network.add_connection(bias_connection, source='Y_b', target='Y')
def main(seed=0,
n_train=60000,
n_test=10000,
time=50,
lr=0.01,
lr_decay=0.95,
update_interval=500,
max_prob=1.0,
plot=False,
train=True,
gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [seed, n_train, time, lr, lr_decay, update_interval, max_prob]
model_name = '_'.join([str(x) for x in params])
if not train:
test_params = [
seed, n_train, n_test, time, lr, lr_decay, update_interval,
max_prob
]
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
criterion = torch.nn.CrossEntropyLoss(
) # Loss function on output firing rates.
n_examples = n_train if train else n_test
if train:
# Network building.
network = Network()
# Groups of neurons.
input_layer = RealInput(n=784, sum_input=True)
output_layer = IFNodes(n=10, sum_input=True)
bias = RealInput(n=1, sum_input=True)
network.add_layer(input_layer, name='X')
network.add_layer(output_layer, name='Y')
network.add_layer(bias, name='Y_b')
# Connections between groups of neurons.
input_connection = Connection(source=input_layer,
target=output_layer,
norm=150,
wmin=-1,
wmax=1)
bias_connection = Connection(source=bias, target=output_layer)
network.add_connection(input_connection, source='X', target='Y')
network.add_connection(bias_connection, source='Y_b', target='Y')
# State variable monitoring.
for l in network.layers:
m = Monitor(network.layers[l], state_vars=['s'], time=time)
network.add_monitor(m, name=l)
else:
network = load_network(os.path.join(params_path, model_name + '.pt'))
# Load MNIST data.
dataset = MNIST(path=data_path, download=True, shuffle=True)
if train:
images, labels = dataset.get_train()
else:
images, labels = dataset.get_test()
images, labels = images.view(-1, 784) / 255, labels
grads = {}
accuracies = []
predictions = []
ground_truth = []
best = -np.inf
spike_ims, spike_axes, weights_im = None, None, None
losses = torch.zeros(update_interval)
correct = torch.zeros(update_interval)
# Run training.
start = t()
for i in range(n_examples):
label = torch.Tensor([labels[i % len(labels)]]).long()
image = images[i % len(labels)]
# Run simulation for single datum.
inpts = {'X': image.repeat(time, 1), 'Y_b': torch.ones(time, 1)}
network.run(inpts=inpts, time=time)
# Retrieve spikes and summed inputs from both layers.
spikes = {
l: network.monitors[l].get('s')
for l in network.layers if '_b' not in l
}
summed_inputs = {l: network.layers[l].summed for l in network.layers}
# Compute softmax of output spiking activity and get predicted label.
output = summed_inputs['Y'].softmax(0).view(1, -1)
predicted = output.argmax(1).item()
correct[i % update_interval] = int(predicted == label[0].item())
predictions.append(predicted)
ground_truth.append(label)
# Compute cross-entropy loss between output and true label.
losses[i % update_interval] = criterion(output, label)
if train:
# Compute gradient of the loss WRT average firing rates.
grads['dl/df'] = summed_inputs['Y'].softmax(0)
grads['dl/df'][label] -= 1
# Compute gradient of the summed voltages WRT connection weights.
# This is an approximation; the summed voltages are not a
# smooth function of the connection weights.
grads['dl/dw'] = torch.ger(summed_inputs['X'], grads['dl/df'])
grads['dl/db'] = grads['dl/df']
# Do stochastic gradient descent calculation.
network.connections['X', 'Y'].w -= lr * grads['dl/dw']
network.connections['Y_b', 'Y'].w -= lr * grads['dl/db']
if i > 0 and i % update_interval == 0:
accuracies.append(correct.mean() * 100)
if train:
if accuracies[-1] > best:
print()
print(
'New best accuracy! Saving network parameters to disk.'
)
# Save network to disk.
network.save(os.path.join(params_path, model_name + '.pt'))
best = accuracies[-1]
print()
print(f'Progress: {i} / {n_examples} ({t() - start:.3f} seconds)')
print(f'Average cross-entropy loss: {losses.mean():.3f}')
print(f'Last accuracy: {accuracies[-1]:.3f}')
print(f'Average accuracy: {np.mean(accuracies):.3f}')
# Decay learning rate.
lr *= lr_decay
if train:
print(f'Best accuracy: {best:.3f}')
print(f'Current learning rate: {lr:.3f}')
start = t()
if plot:
w = network.connections['X', 'Y'].w
weights = [w[:, i].view(28, 28) for i in range(10)]
w = torch.zeros(5 * 28, 2 * 28)
for i in range(5):
for j in range(2):
w[i * 28:(i + 1) * 28,
j * 28:(j + 1) * 28] = weights[i + j * 5]
spike_ims, spike_axes = plot_spikes(spikes,
ims=spike_ims,
axes=spike_axes)
weights_im = plot_weights(w, im=weights_im, wmin=-1, wmax=1)
plt.pause(1e-1)
network.reset_() # Reset state variables.
accuracies.append(correct.mean() * 100)
if train:
lr *= lr_decay
for c in network.connections:
network.connections[c].update_rule.weight_decay *= lr_decay
if accuracies[-1] > best:
print()
print('New best accuracy! Saving network parameters to disk.')
# Save network to disk.
network.save(os.path.join(params_path, model_name + '.pt'))
best = accuracies[-1]
print()
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.3f} seconds)')
print(f'Average cross-entropy loss: {losses.mean():.3f}')
print(f'Last accuracy: {accuracies[-1]:.3f}')
print(f'Average accuracy: {np.mean(accuracies):.3f}')
if train:
print(f'Best accuracy: {best:.3f}')
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')
print(f'Average accuracy: {np.mean(accuracies):.3f}')
# Save accuracy curves to disk.
to_write = ['train'] + params if train else ['test'] + params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save((accuracies, update_interval, n_examples),
open(os.path.join(curves_path, f), 'wb'))
results = [np.mean(accuracies), np.max(accuracies)]
to_write = params + results if train else test_params + results
to_write = [str(x) for x in to_write]
name = 'train.csv' if train else 'test.csv'
if not os.path.isfile(os.path.join(results_path, name)):
with open(os.path.join(results_path, name), 'w') as f:
if train:
f.write(
'seed,n_train,time,lr,lr_decay,update_interval,max_prob,mean_accuracy,max_accuracy\n'
)
else:
f.write(
'seed,n_train,n_test,time,lr,lr_decay,update_interval,max_prob,mean_accuracy,max_accuracy\n'
)
with open(os.path.join(results_path, name), 'a') as f:
f.write(','.join(to_write) + '\n')
# Compute confusion matrices and save them to disk.
confusion = confusion_matrix(ground_truth, predictions)
to_write = ['train'] + params if train else ['test'] + test_params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save(confusion, os.path.join(confusion_path, f))
# Encoding parameters
dt = args.dt
dataset_path = "../../data/SpokenMNIST"
train_dataset = SpokenMNIST(dataset_path, download=True, train=True)
train_dataloader = torch.utils.data.DataLoader(
train_dataset, batch_size=1, shuffle=True, num_workers=0
)
sample_shape = train_dataset[0][0].shape
print("SpokenMNIST has shape ", sample_shape)
network = Network()
# Make sure to include the batch dimension but not time
input_layer = RealInput(shape=(1, *sample_shape[1:]), traces=True)
out_layer = DiehlAndCookNodes(shape=(1, *sample_shape[1:]), traces=True)
out_conn = Connection(input_layer, out_layer, wmin=0, wmax=1)
network.add_layer(input_layer, name="X")
network.add_layer(out_layer, name="Y")
network.add_connection(out_conn, source="X", target="Y")
for step, batch in enumerate(tqdm(train_dataloader)):
inpts = {"X": batch["audio"]}
# the audio has potentially a variable amount of time
time = spike_audio.shape[1]
def main(seed=0,
n_neurons=100,
n_train=60000,
n_test=10000,
inhib=500,
dt=1,
theta_plus=0.05,
theta_decay=1e-7,
intensity=1 / 40,
progress_interval=10,
update_interval=250,
plot=False,
train=True,
gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [
seed, n_neurons, n_train, inhib, dt, theta_plus, theta_decay,
intensity, progress_interval, update_interval
]
test_params = [
seed, n_neurons, n_train, n_test, inhib, dt, theta_plus, theta_decay,
intensity, progress_interval, update_interval
]
model_name = '_'.join([str(x) for x in params])
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
n_examples = n_train if train else n_test
n_sqrt = int(np.ceil(np.sqrt(n_neurons)))
n_classes = 10
# Build network.
if train:
network = Network()
input_layer = RealInput(n=40, traces=True, trace_tc=5e-2)
network.add_layer(input_layer, name='X')
output_layer = DiehlAndCookNodes(n=n_neurons,
traces=True,
rest=-65.0,
reset=-65.0,
thresh=-52.0,
refrac=5,
decay=1e-2,
trace_tc=5e-2,
theta_plus=theta_plus,
theta_decay=theta_decay)
network.add_layer(output_layer, name='Y')
w = 0.3 * torch.rand(40, n_neurons)
input_connection = Connection(source=network.layers['X'],
target=network.layers['Y'],
w=w,
update_rule=PostPre,
nu=(0, 1),
wmin=0,
wmax=1,
norm=4)
network.add_connection(input_connection, source='X', target='Y')
w = -inhib * (torch.ones(n_neurons, n_neurons) -
torch.diag(torch.ones(n_neurons)))
recurrent_connection = Connection(source=network.layers['Y'],
target=network.layers['Y'],
w=w,
wmin=-inhib,
wmax=0)
network.add_connection(recurrent_connection, source='Y', target='Y')
else:
network = load_network(os.path.join(params_path, model_name + '.pt'))
network.connections['X', 'Y'].update_rule = NoOp(
connection=network.connections['X', 'Y'],
nu=network.connections['X', 'Y'].nu)
network.layers['Y'].theta_decay = 0
network.layers['Y'].theta_plus = 0
# Load Spoken MNIST data.
dataset = SpokenMNIST(path=data_path, download=True, shuffle=False)
if train:
audio, labels = dataset.get_train()
else:
audio, labels = dataset.get_test()
audio = [_ * intensity for _ in audio]
# Record spikes during the simulation.
avg_time = int(np.mean([_.size(0) for _ in audio]))
spike_record = torch.zeros(update_interval, avg_time, n_neurons)
# Neuron assignments and spike proportions.
if train:
assignments = -torch.ones_like(torch.Tensor(n_neurons))
proportions = torch.zeros_like(torch.Tensor(n_neurons, 10))
rates = torch.zeros_like(torch.Tensor(n_neurons, 10))
ngram_scores = {}
else:
path = os.path.join(params_path,
'_'.join(['auxiliary', model_name]) + '.pt')
assignments, proportions, rates, ngram_scores = torch.load(
open(path, 'rb'))
# Sequence of accuracy estimates.
curves = {'all': [], 'proportion': [], 'ngram': []}
predictions = {scheme: torch.Tensor().long() for scheme in curves.keys()}
if train:
best_accuracy = 0
spikes = {}
for layer in set(network.layers):
spikes[layer] = Monitor(network.layers[layer],
state_vars=['s'],
time=avg_time)
network.add_monitor(spikes[layer], name='%s_spikes' % layer)
# Train the network.
if train:
print('\nBegin training.\n')
else:
print('\nBegin test.\n')
inpt_axes = None
inpt_ims = None
spike_ims = None
spike_axes = None
weights_im = None
assigns_im = None
perf_ax = None
start = t()
for i in range(n_examples):
if i % progress_interval == 0:
print(f'Progress: {i} / {n_examples} ({t() - start:.4f} seconds)')
start = t()
if i % update_interval == 0 and i > 0:
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(audio) - update_interval:i %
len(audio)]
# Update and print accuracy evaluations.
curves, preds = update_curves(curves,
current_labels,
n_classes,
spike_record=spike_record,
assignments=assignments,
proportions=proportions,
ngram_scores=ngram_scores,
n=2)
print_results(curves)
for scheme in preds:
predictions[scheme] = torch.cat(
[predictions[scheme], preds[scheme]], -1)
# Save accuracy curves to disk.
to_write = ['train'] + params if train else ['test'] + params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save((curves, update_interval, n_examples),
open(os.path.join(curves_path, f), 'wb'))
if train:
if any([x[-1] > best_accuracy for x in curves.values()]):
print(
'New best accuracy! Saving network parameters to disk.'
)
# Save network to disk.
network.save(os.path.join(params_path, model_name + '.pt'))
path = os.path.join(
params_path,
'_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores),
open(path, 'wb'))
best_accuracy = max([x[-1] for x in curves.values()])
# Assign labels to excitatory layer neurons.
assignments, proportions, rates = assign_labels(
spike_record, current_labels, 10, rates)
# Compute ngram scores.
ngram_scores = update_ngram_scores(spike_record,
current_labels, 10, 2,
ngram_scores)
print()
# Get next input sample.
sample = audio[i % len(audio)]
sample = sample[:40, :]
inpts = {'X': sample}
time = min(avg_time, sample.size(0))
# Run the network on the input.
network.run(inpts=inpts, time=time)
retries = 0
while spikes['Y'].get('s').sum() < 5 and retries < 3:
retries += 1
sample *= 2
inpts = {'X': sample}
network.run(inpts=inpts, time=time)
# Add to spikes recording.
spike_record[i % update_interval] = spikes['Y'].get('s').t()
# Optionally plot various simulation information.
if plot:
# _input = image.view(28, 28)
# reconstruction = inpts['X'].view(time, 40).sum(0).view(8, 5)
_spikes = {layer: spikes[layer].get('s') for layer in spikes}
input_exc_weights = network.connections[('X', 'Y')].w
square_weights = get_square_weights(
input_exc_weights.view(40, n_neurons), n_sqrt, (8, 5))
# square_assignments = get_square_assignments(assignments, n_sqrt)
# inpt_axes, inpt_ims = plot_input(_input, reconstruction, label=labels[i], axes=inpt_axes, ims=inpt_ims)
spike_ims, spike_axes = plot_spikes(_spikes,
ims=spike_ims,
axes=spike_axes)
weights_im = plot_weights(square_weights, im=weights_im)
# assigns_im = plot_assignments(square_assignments, im=assigns_im)
# perf_ax = plot_performance(curves, ax=perf_ax)
plt.pause(1e-8)
network.reset_() # Reset state variables.
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.4f} seconds)')
i += 1
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(audio) - update_interval:i %
len(audio)]
# Update and print accuracy evaluations.
curves, preds = update_curves(curves,
current_labels,
n_classes,
spike_record=spike_record,
assignments=assignments,
proportions=proportions,
ngram_scores=ngram_scores,
n=2)
print_results(curves)
for scheme in preds:
predictions[scheme] = torch.cat([predictions[scheme], preds[scheme]],
-1)
if train:
if any([x[-1] > best_accuracy for x in curves.values()]):
print('New best accuracy! Saving network parameters to disk.')
# Save network to disk.
if train:
network.save(os.path.join(params_path, model_name + '.pt'))
path = os.path.join(
params_path, '_'.join(['auxiliary', model_name]) + '.pt')
torch.save((assignments, proportions, rates, ngram_scores),
open(path, 'wb'))
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')
print('Average accuracies:\n')
for scheme in curves.keys():
print('\t%s: %.2f' % (scheme, float(np.mean(curves[scheme]))))
# Save accuracy curves to disk.
to_write = ['train'] + params if train else ['test'] + params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save((curves, update_interval, n_examples),
open(os.path.join(curves_path, f), 'wb'))
# Save results to disk.
results = [
np.mean(curves['all']),
np.mean(curves['proportion']),
np.mean(curves['ngram']),
np.max(curves['all']),
np.max(curves['proportion']),
np.max(curves['ngram'])
]
to_write = params + results if train else test_params + results
to_write = [str(x) for x in to_write]
name = 'train.csv' if train else 'test.csv'
if not os.path.isfile(os.path.join(results_path, name)):
with open(os.path.join(results_path, name), 'w') as f:
if train:
f.write(
'random_seed,n_neurons,n_train,inhib,timestep,theta_plus,theta_decay,intensity,'
'progress_interval,update_interval,mean_all_activity,mean_proportion_weighting,'
'mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n'
)
else:
f.write(
'random_seed,n_neurons,n_train,n_test,inhib,timestep,theta_plus,theta_decay,intensity,'
'progress_interval,update_interval,mean_all_activity,mean_proportion_weighting,'
'mean_ngram,max_all_activity,max_proportion_weighting,max_ngram\n'
)
with open(os.path.join(results_path, name), 'a') as f:
f.write(','.join(to_write) + '\n')
if labels.numel() > n_examples:
labels = labels[:n_examples]
else:
while labels.numel() < n_examples:
if 2 * labels.numel() > n_examples:
labels = torch.cat(
[labels, labels[:n_examples - labels.numel()]])
else:
labels = torch.cat([labels, labels])
# Compute confusion matrices and save them to disk.
confusions = {}
for scheme in predictions:
confusions[scheme] = confusion_matrix(labels, predictions[scheme])
to_write = ['train'] + params if train else ['test'] + test_params
f = '_'.join([str(x) for x in to_write]) + '.pt'
torch.save(confusions, os.path.join(confusion_path, f))
def main(seed=0,
n_neurons=100,
n_train=60000,
n_test=10000,
inhib=100,
lr=0.01,
lr_decay=1,
time=350,
dt=1,
theta_plus=0.05,
theta_decay=1e-7,
progress_interval=10,
update_interval=250,
plot=False,
train=True,
gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [
seed, n_neurons, n_train, inhib, lr_decay, time, dt, theta_plus,
theta_decay, progress_interval, update_interval
]
model_name = '_'.join([str(x) for x in params])
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
n_examples = n_train if train else n_test
n_classes = 10
# Build network.
if train:
network = Network(dt=dt)
input_layer = RealInput(n=784, traces=True, trace_tc=5e-2)
network.add_layer(input_layer, name='X')
output_layer = DiehlAndCookNodes(n=n_classes,
rest=0,
reset=1,
thresh=1,
decay=1e-2,
theta_plus=theta_plus,
theta_decay=theta_decay,
traces=True,
trace_tc=5e-2)
network.add_layer(output_layer, name='Y')
w = torch.rand(784, n_classes)
input_connection = Connection(source=input_layer,
target=output_layer,
w=w,
update_rule=MSTDPET,
nu=lr,
wmin=0,
wmax=1,
norm=78.4,
tc_e_trace=0.1)
network.add_connection(input_connection, source='X', target='Y')
else:
network = load_network(os.path.join(params_path, model_name + '.pt'))
network.connections['X', 'Y'].update_rule = NoOp(
connection=network.connections['X', 'Y'],
nu=network.connections['X', 'Y'].nu)
network.layers['Y'].theta_decay = 0
network.layers['Y'].theta_plus = 0
# Load MNIST data.
environment = MNISTEnvironment(dataset=MNIST(path=data_path,
download=True),
train=train,
time=time)
# Create pipeline.
pipeline = Pipeline(network=network,
environment=environment,
encoding=repeat,
action_function=select_spiked,
output='Y',
reward_delay=None)
spikes = {}
for layer in set(network.layers):
spikes[layer] = Monitor(network.layers[layer],
state_vars=('s', ),
time=time)
network.add_monitor(spikes[layer], name='%s_spikes' % layer)
network.add_monitor(
Monitor(network.connections['X', 'Y'].update_rule,
state_vars=('e_trace', ),
time=time), 'X_Y_e_trace')
# Train the network.
if train:
print('\nBegin training.\n')
else:
print('\nBegin test.\n')
spike_ims = None
spike_axes = None
weights_im = None
elig_axes = None
elig_ims = None
start = t()
for i in range(n_examples):
if i % progress_interval == 0:
print(f'Progress: {i} / {n_examples} ({t() - start:.4f} seconds)')
start = t()
if i > 0 and train:
network.connections['X', 'Y'].update_rule.nu[1] *= lr_decay
# Run the network on the input.
for j in range(time):
pipeline.step(a_plus=1, a_minus=0)
if plot:
_spikes = {layer: spikes[layer].get('s') for layer in spikes}
w = network.connections['X', 'Y'].w
square_weights = get_square_weights(w.view(784, n_classes), 4, 28)
spike_ims, spike_axes = plot_spikes(_spikes,
ims=spike_ims,
axes=spike_axes)
weights_im = plot_weights(square_weights, im=weights_im)
elig_ims, elig_axes = plot_voltages(
{
'Y':
network.monitors['X_Y_e_trace'].get('e_trace').view(
-1, time)[1500:2000]
},
plot_type='line',
ims=elig_ims,
axes=elig_axes)
plt.pause(1e-8)
pipeline.reset_() # Reset state variables.
network.connections['X', 'Y'].update_rule.e_trace = torch.zeros(
784, n_classes)
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.4f} seconds)')
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')
parser = argparse.ArgumentParser()
parser.add_argument('--n_input', type=int, default=100)
parser.add_argument('--n_output', type=int, default=500)
parser.add_argument('--time', type=int, default=1000)
parser.add_argument('--plot', dest='plot', action='store_true')
parser.set_defaults(plot=False)
args = parser.parse_args()
n_input = args.n_input
n_output = args.n_output
time = args.time
plot = args.plot
network = Network()
input_layer = RealInput(n=n_input)
output_layer = LIFNodes(n=500)
connection = Connection(source=input_layer,
target=output_layer,
w=torch.rand(input_layer.n, output_layer.n))
network.add_layer(layer=input_layer, name='X')
network.add_layer(layer=output_layer, name='Y')
network.add_connection(connection=connection, source='X', target='Y')
network.add_monitor(monitor=Monitor(obj=input_layer, state_vars=['s']),
name='X')
network.add_monitor(monitor=Monitor(obj=output_layer, state_vars=['s', 'v']),
name='Y')
input_data = {'X': torch.randn(time, n_input)}
confusion_path = os.path.join(top_level, 'confusion', data, model)
for path in [params_path, curves_path, results_path, confusion_path]:
if not os.path.isdir(path):
os.makedirs(path)
criterion = torch.nn.CrossEntropyLoss(
) # Loss function on output firing rates.
n_examples = n_train if train else n_test
if train:
# Network building.
network = Network()
# Groups of neurons.
input_layer = RealInput(n=32 * 32 * 3, sum_input=True)
output_layer = IFNodes(n=10, sum_input=True)
bias = RealInput(n=1, sum_input=True)
network.add_layer(input_layer, name='X')
network.add_layer(output_layer, name='Y')
network.add_layer(bias, name='Y_b')
# Connections between groups of neurons.
input_connection = Connection(source=input_layer,
target=output_layer,
norm=norm,
wmin=wmin,
wmax=wmax)
bias_connection = Connection(source=bias, target=output_layer)
network.add_connection(input_connection, source='X', target='Y')
network.add_connection(bias_connection, source='Y_b', target='Y')
def main(seed=0, n_neurons=100, n_train=60000, n_test=10000, inhib=100, lr=0.01, lr_decay=1, time=350, dt=1,
theta_plus=0.05, theta_decay=1e-7, progress_interval=10, update_interval=250, plot=False,
train=True, gpu=False):
assert n_train % update_interval == 0 and n_test % update_interval == 0, \
'No. examples must be divisible by update_interval'
params = [
seed, n_neurons, n_train, inhib, lr_decay, time, dt,
theta_plus, theta_decay, progress_interval, update_interval
]
test_params = [
seed, n_neurons, n_train, n_test, inhib, lr_decay, time, dt,
theta_plus, theta_decay, progress_interval, update_interval
]
model_name = '_'.join([str(x) for x in params])
np.random.seed(seed)
if gpu:
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.cuda.manual_seed_all(seed)
else:
torch.manual_seed(seed)
n_examples = n_train if train else n_test
n_sqrt = int(np.ceil(np.sqrt(n_neurons)))
n_classes = 10
# Build network.
if train:
network = Network(dt=dt)
input_layer = RealInput(n=784, traces=True, trace_tc=5e-2)
network.add_layer(input_layer, name='X')
output_layer = DiehlAndCookNodes(
n=n_neurons, traces=True, rest=0, reset=1, thresh=1, refrac=0,
decay=1e-2, trace_tc=5e-2, theta_plus=theta_plus, theta_decay=theta_decay
)
network.add_layer(output_layer, name='Y')
readout = IFNodes(n=n_classes, reset=0, thresh=1)
network.add_layer(readout, name='Z')
w = torch.rand(784, n_neurons)
input_connection = Connection(
source=input_layer, target=output_layer, w=w,
update_rule=MSTDP, nu=lr, wmin=0, wmax=1, norm=78.4
)
network.add_connection(input_connection, source='X', target='Y')
w = -inhib * (torch.ones(n_neurons, n_neurons) - torch.diag(torch.ones(n_neurons)))
recurrent_connection = Connection(
source=output_layer, target=output_layer, w=w, wmin=-inhib, wmax=0
)
network.add_connection(recurrent_connection, source='Y', target='Y')
readout_connection = Connection(
source=network.layers['Y'], target=readout, w=torch.rand(n_neurons, n_classes), norm=10
)
network.add_connection(readout_connection, source='Y', target='Z')
else:
network = load_network(os.path.join(params_path, model_name + '.pt'))
network.connections['X', 'Y'].update_rule = NoOp(
connection=network.connections['X', 'Y'], nu=network.connections['X', 'Y'].nu
)
network.layers['Y'].theta_decay = 0
network.layers['Y'].theta_plus = 0
# Load MNIST data.
dataset = MNIST(path=data_path, download=True)
if train:
images, labels = dataset.get_train()
else:
images, labels = dataset.get_test()
images = images.view(-1, 784)
labels = labels.long()
spikes = {}
for layer in set(network.layers) - {'X'}:
spikes[layer] = Monitor(network.layers[layer], state_vars=['s'], time=time)
network.add_monitor(spikes[layer], name='%s_spikes' % layer)
# Train the network.
if train:
print('\nBegin training.\n')
else:
print('\nBegin test.\n')
inpt_axes = None
inpt_ims = None
spike_ims = None
spike_axes = None
weights_im = None
weights2_im = None
assigns_im = None
perf_ax = None
predictions = torch.zeros(update_interval).long()
start = t()
for i in range(n_examples):
if i % progress_interval == 0:
print(f'Progress: {i} / {n_examples} ({t() - start:.4f} seconds)')
start = t()
if i > 0 and train:
network.connections['X', 'Y'].update_rule.nu[1] *= lr_decay
# Get next input sample.
image = images[i % len(images)]
# Run the network on the input.
for j in range(time):
readout = network.layers['Z'].s
if readout[labels[i % len(labels)]]:
network.run(inpts={'X': image.unsqueeze(0)}, time=1, reward=1, a_minus=0, a_plus=1)
else:
network.run(inpts={'X': image.unsqueeze(0)}, time=1, reward=0)
label = spikes['Z'].get('s').sum(1).argmax()
predictions[i % update_interval] = label.long()
if i > 0 and i % update_interval == 0:
if i % len(labels) == 0:
current_labels = labels[-update_interval:]
else:
current_labels = labels[i % len(images) - update_interval:i % len(images)]
accuracy = 100 * (predictions == current_labels).float().mean().item()
print(f'Accuracy over last {update_interval} examples: {accuracy}')
# Optionally plot various simulation information.
if plot:
_spikes = {layer: spikes[layer].get('s') for layer in spikes}
input_exc_weights = network.connections['X', 'Y'].w
square_weights = get_square_weights(input_exc_weights.view(784, n_neurons), n_sqrt, 28)
exc_readout_weights = network.connections['Y', 'Z'].w
# _input = image.view(28, 28)
# reconstruction = inpts['X'].view(time, 784).sum(0).view(28, 28)
# square_assignments = get_square_assignments(assignments, n_sqrt)
spike_ims, spike_axes = plot_spikes(_spikes, ims=spike_ims, axes=spike_axes)
weights_im = plot_weights(square_weights, im=weights_im)
weights2_im = plot_weights(exc_readout_weights, im=weights2_im)
# inpt_axes, inpt_ims = plot_input(_input, reconstruction, label=labels[i], axes=inpt_axes, ims=inpt_ims)
# assigns_im = plot_assignments(square_assignments, im=assigns_im)
# perf_ax = plot_performance(curves, ax=perf_ax)
plt.pause(1e-8)
network.reset_() # Reset state variables.
print(f'Progress: {n_examples} / {n_examples} ({t() - start:.4f} seconds)')
if train:
print('\nTraining complete.\n')
else:
print('\nTest complete.\n')