def test_init(self):
with self.assertRaises(ValueError):
nn.NeuralNet((2,))
net = nn.NeuralNet(LAYER_SIZES)
self.assert_dimensions(net.matrices, nn.KEY_WEIGHT, {
nn.DIM_IN: LAYER_SIZES[:-1], nn.DIM_OUT: LAYER_SIZES[1:]})
self.assert_dimensions(net.matrices, nn.KEY_BIAS, {
nn.DIM_OUT: LAYER_SIZES[1:]})
def build_culled_net_from_random(inputs, labels, sizes=(784, 30, 10),
num_to_search=100, dapd_by_labels=None, dapd_scale_factor=2, num_buckets=30):
best_weights = []
best_biases = []
control_weights = []
control_biases = []
worst_weights = []
worst_biases = []
best_dapd, worst_dapd = None, None
# if set, use dapd to screen weights
if not dapd_by_labels is None:
best_dapd = cost(dapd_by_labels, dim='labels').stack(
inputs=('inputs_y', 'inputs_x')).reset_index('inputs', drop=True)
# normalize dapd
best_dapd = best_dapd / np.amax(best_dapd) * dapd_scale_factor
worst_dapd = dapd_scale_factor - best_dapd
for neuron in range(sizes[1]):
print('Generating neuron', neuron, '...')
control_net = nn.NeuralNet((sizes[0], num_to_search))
net = control_net
# control
control = np.random.randint(num_to_search)
control_weights.append(net.matrices[nn.mkey(0, 'weights')].isel(neurons=control))
control_biases.append(net.matrices[nn.mkey(0, 'biases')].isel(neurons=control))
# best
if not dapd_by_labels is None:
net = copy.deepcopy(control_net)
net.matrices[nn.mkey(0, 'weights')] = net.matrices[nn.mkey(0, 'weights')] * best_dapd
activations = net.pass_forward(inputs)[nn.mkey(1, 'post_activation')]
dapd_per_neuron = cost(apd_area(diff_by_label(
activations, labels, num_buckets=num_buckets)), dim='labels')
best = np.argmax(dapd_per_neuron)
best_weights.append(net.matrices[nn.mkey(0, 'weights')].isel(neurons=best))
best_biases.append(net.matrices[nn.mkey(0, 'biases')].isel(neurons=best))
# worst
if not dapd_by_labels is None:
net = copy.deepcopy(control_net)
net.matrices[nn.mkey(0, 'weights')] = net.matrices[nn.mkey(0, 'weights')] * worst_dapd
activations = net.pass_forward(inputs)[nn.mkey(1, 'post_activation')]
dapd_per_neuron = cost(apd_area(diff_by_label(
activations, labels, num_buckets=num_buckets)), dim='labels')
worst = np.argmin(dapd_per_neuron)
worst_weights.append(net.matrices[nn.mkey(0, 'weights')].isel(neurons=worst))
worst_biases.append(net.matrices[nn.mkey(0, 'biases')].isel(neurons=worst))
net = nn.NeuralNet(sizes)
best_net = copy.deepcopy(net)
best_net.matrices[nn.mkey(0, 'weights')] = xr.concat(best_weights, dim='neurons').transpose('inputs', 'neurons')
best_net.matrices[nn.mkey(0, 'biases')] = xr.concat(best_biases, dim='neurons')
control_net = copy.deepcopy(net)
control_net.matrices[nn.mkey(0, 'weights')] = xr.concat(control_weights, dim='neurons').transpose('inputs', 'neurons')
control_net.matrices[nn.mkey(0, 'biases')] = xr.concat(control_biases, dim='neurons')
worst_net = copy.deepcopy(net)
worst_net.matrices[nn.mkey(0, 'weights')] = xr.concat(worst_weights, dim='neurons').transpose('inputs', 'neurons')
worst_net.matrices[nn.mkey(0, 'biases')] = xr.concat(worst_biases, dim='neurons')
return best_net, control_net, worst_net
def build_kernel_net(width, falloff, stride, save_dir='./models/kernel'):
first_layer = tile_kernel(square_kernel(width, falloff), stride=stride)
neurons_in_first_layer = first_layer.sizes[nn.DIM_OUT]
kernel_net = nn.NeuralNet((784, neurons_in_first_layer, 10))
kernel_net.matrices[nn.mkey(0, nn.KEY_WEIGHT)] = first_layer
name = '_w' + str(width) + 'f' + str(falloff) \
+ 's' + str(stride[0]) + 'x' + str(stride[1])
kernel_name = 'net_kernel' + name
rand_name = 'net_rand' + name
write_object(kernel_net, kernel_name, directory=save_dir)
rand_net = nn.NeuralNet((784, neurons_in_first_layer, 10))
write_object(rand_net, rand_name, directory=save_dir)
return (kernel_name, kernel_net), (rand_name, rand_net)
def test_train(self):
net = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
net2 = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
self.assert_nn_equal(net, net2)
num_batches = 4
inputs = xr.DataArray(
np.zeros((num_batches * NUM_CASES, INPUT_SIZE)), dims=[nn.DIM_CASE, nn.DIM_IN])
labels = utility.make_onehot(xr.DataArray(
np.zeros((num_batches * NUM_CASES,)), dims=[nn.DIM_CASE]), np.zeros(NUM_LABELS))
trained = net.train(inputs, labels, batch_size=NUM_CASES)
self.assert_dimensions(trained.matrices, nn.KEY_WEIGHT, {
nn.DIM_IN: LAYER_SIZES[:-1], nn.DIM_OUT: LAYER_SIZES[1:]})
self.assert_dimensions(trained.matrices, nn.KEY_BIAS, {
nn.DIM_OUT: LAYER_SIZES[1:]})
self.assertTrue(self.nn_not_equal(net, trained))
def test_pass_forward_output_only(self):
net = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
inputs = xr.DataArray(
np.zeros((NUM_CASES, INPUT_SIZE)), dims=(nn.DIM_CASE, nn.DIM_IN))
output = net.pass_forward_output_only(inputs)
self.assertDictEqual(dict(output.sizes), {
nn.DIM_CASE: NUM_CASES, nn.DIM_IN: NUM_LABELS})
def test_delete_neurons(self):
net = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
inputs = xr.DataArray(
np.zeros((NUM_CASES, INPUT_SIZE)), dims=[nn.DIM_CASE, nn.DIM_IN])
activations = net.pass_forward(inputs)
new_net = net.delete_neurons([[3, 1], [0]], activations=activations)
sizes = [x for x in LAYER_SIZES]
sizes[1] -= 2
sizes[2] -= 1
self.assert_dimensions(new_net.matrices, nn.KEY_WEIGHT, {
nn.DIM_IN: sizes[:-1], nn.DIM_OUT: sizes[1:]})
self.assertEqual(new_net.matrices[nn.mkey(1, nn.KEY_BIAS)][0], activations[nn.mkey(
1, nn.KEY_OUT_POST)][0, 0] * 2 + 1)
def test_pass_back(self):
net = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
activations = {}
for i, l_size in zip(range(NUM_LAYERS+1), LAYER_SIZES):
activations[nn.mkey(i, nn.KEY_OUT_PRE)] = xr.DataArray(
np.zeros((NUM_CASES, l_size)), dims=(nn.DIM_CASE, nn.DIM_IN))
activations[nn.mkey(i, nn.KEY_OUT_POST)] = xr.DataArray(
np.ones((NUM_CASES, l_size)), dims=(nn.DIM_CASE, nn.DIM_IN))
inputs = xr.DataArray(
np.ones((NUM_CASES, INPUT_SIZE)), dims=(nn.DIM_CASE, nn.DIM_IN))
activations = net.pass_forward(inputs)
labels = utility.make_onehot(xr.DataArray(np.arange(NUM_CASES), dims=(
nn.DIM_CASE)), np.arange(NUM_LABELS)) # labels are 0 to n
gradients = net.pass_back(activations, labels)
self.assert_dimensions(gradients, nn.KEY_WEIGHT, {nn.DIM_CASE: [
NUM_CASES]*NUM_LAYERS, nn.DIM_IN: LAYER_SIZES[:-1], nn.DIM_OUT: LAYER_SIZES[1:]})
self.assert_dimensions(gradients, nn.KEY_BIAS, {
nn.DIM_CASE: [NUM_CASES] * NUM_LAYERS, nn.DIM_OUT: LAYER_SIZES[1:]})
def test_pass_forward(self):
net = nn.NeuralNet(LAYER_SIZES, func_fill=np.ones)
with self.assertRaises(ValueError):
net.pass_forward(xr.DataArray(
np.zeros((NUM_CASES, INPUT_SIZE)), dims=(nn.DIM_CASE, 'asdf')))
with self.assertRaises(ValueError):
net.pass_forward(xr.DataArray(
np.zeros((NUM_CASES, INPUT_SIZE+10)), dims=(nn.DIM_CASE, nn.DIM_IN)))
inputs = xr.DataArray(
np.zeros((NUM_CASES, INPUT_SIZE)), dims=(nn.DIM_CASE, nn.DIM_IN))
outputs = net.pass_forward(inputs)
self.assert_dimensions(outputs, nn.KEY_OUT_PRE, {
nn.DIM_CASE: [NUM_CASES]*len(LAYER_SIZES), nn.DIM_IN: LAYER_SIZES})
self.assert_dimensions(outputs, nn.KEY_OUT_POST, {
nn.DIM_CASE: [NUM_CASES]*len(LAYER_SIZES), nn.DIM_IN: LAYER_SIZES})
np.testing.assert_allclose(net.pass_forward_output_only(
inputs).isel({nn.DIM_IN: 0}), EXPECTED_OUTPUT)
in the intermediate outputs of a network.
Activation probability distributions
------------------------------------
One way of detecting such biases is to make a histogram of a neuron's outputs:
a non-parametric probability distribution of the activations. Let's train a simple neural
network with 1 hidden layer of 30 neurons on the MNIST dataset, and generate histograms
for its output neurons:
"""
import numpy as np, xarray as xr, matplotlib.pyplot as plt
from littlenet import apd, utility, neural_net as nn, train
net = nn.NeuralNet((784, 30, 10))
utility.write_object(net, 'rand_net', directory='./models/init-apd')
# always write to file to avoid re-generating and re-training networks
rand_net = utility.read_object('./models/init-apd/rand_net.pyc')
train_inputs, train_labels, test_inputs, test_labels = utility.training_and_test_inputs(
is_onehot=False)
train_onehot = utility.make_onehot(train_labels, range(10))
test_onehot = utility.make_onehot(test_labels, range(10))
trained_net, progress = train.train_nn(net,
train_inputs,
train_onehot,
test_inputs,
test_onehot,
sample_rate=500,
hyperparams={