def test_compare_predictors(hang_plot=False):
dataset = get_synthetic_clusters_dataset()
w_constructor = lambda rng=np.random.RandomState(45): .1 * rng.randn(
dataset.input_shape[0], dataset.n_categories)
records = compare_predictors(
dataset=dataset,
offline_predictors={'SVM': SVC()},
online_predictors={
'fast-perceptron':
Perceptron(alpha=0.1, w=w_constructor()).to_categorical(),
'slow-perceptron':
Perceptron(alpha=0.001, w=w_constructor()).to_categorical()
},
minibatch_size=10,
test_epochs=sqrtspace(0, 10, 20),
evaluation_function='percent_correct')
assert 99 < records['SVM'].get_scores('Test') <= 100
assert 20 < records['slow-perceptron'].get_scores(
'Test')[0] < 40 and 95 < records['slow-perceptron'].get_scores(
'Test')[-1] <= 100
assert 20 < records['fast-perceptron'].get_scores(
'Test')[0] < 40 and 98 < records['fast-perceptron'].get_scores(
'Test')[-1] <= 100
plot_learning_curves(records, hang=hang_plot)
def demo_perceptron_dtp(
hidden_sizes=[240],
n_epochs=20,
n_tests=20,
minibatch_size=100,
lin_dtp=True,
):
dataset = get_mnist_dataset(flat=True).to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
predictor = DifferenceTargetMLP(layers=[
PerceptronLayer.from_initializer(n_in,
n_out,
initial_mag=2,
lin_dtp=lin_dtp)
for n_in, n_out in zip([dataset.input_size] +
hidden_sizes, hidden_sizes +
[dataset.target_size])
],
output_cost_function=None).compile()
result = assess_online_predictor(
predictor=predictor,
dataset=dataset,
minibatch_size=minibatch_size,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_tests),
)
plot_learning_curves(result)
def demo_perceptron_dtp(
hidden_sizes = [240],
n_epochs = 20,
n_tests = 20,
minibatch_size=100,
lin_dtp = True,
):
dataset = get_mnist_dataset(flat = True).to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
predictor = DifferenceTargetMLP(
layers=[PerceptronLayer.from_initializer(n_in, n_out, initial_mag=2, lin_dtp = lin_dtp)
for n_in, n_out in zip([dataset.input_size]+hidden_sizes, hidden_sizes+[dataset.target_size])],
output_cost_function = None
).compile()
result = assess_online_predictor(
predictor = predictor,
dataset = dataset,
minibatch_size=minibatch_size,
evaluation_function='percent_argmax_correct',
test_epochs = sqrtspace(0, n_epochs, n_tests),
)
plot_learning_curves(result)
def demo_dtp_varieties(hidden_sizes=[240],
n_epochs=10,
minibatch_size=20,
n_tests=20,
hidden_activation='tanh',
output_activation='sigm',
optimizer='adamax',
learning_rate=0.01,
noise=1,
predictors=['MLP', 'DTP', 'PreAct-DTP', 'Linear-DTP'],
rng=1234,
use_bias=True,
live_plot=False,
plot=False):
"""
;
:param hidden_sizes:
:param n_epochs:
:param minibatch_size:
:param n_tests:
:return:
"""
if isinstance(predictors, str):
predictors = [predictors]
dataset = get_mnist_dataset(flat=True)
dataset = dataset.process_with(
targets_processor=lambda (x, ): (OneHotEncoding(10)(x).astype(int), ))
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 0.1
n_tests = 3
set_default_figure_size(12, 9)
predictors = OrderedDict(
(name,
get_predictor(name,
input_size=dataset.input_size,
target_size=dataset.target_size,
hidden_sizes=hidden_sizes,
hidden_activation=hidden_activation,
output_activation=output_activation,
optimizer=optimizer,
learning_rate=learning_rate,
noise=noise,
use_bias=use_bias,
rng=rng)) for name in predictors)
learning_curves = compare_predictors(
dataset=dataset,
online_predictors=predictors,
minibatch_size=minibatch_size,
test_epochs=sqrtspace(0, n_epochs, n_tests),
evaluation_function=percent_argmax_correct,
)
if plot:
plot_learning_curves(learning_curves)
def demo_compare_dtp_methods(
predictor_constructors,
n_epochs = 10,
minibatch_size = 20,
n_tests = 20,
onehot = True,
accumulator = None
):
dataset = get_mnist_dataset(flat = True, binarize = False)
n_categories = dataset.n_categories
if onehot:
dataset = dataset.to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
learning_curves = compare_predictors(
dataset=dataset,
online_predictors = {name: p(dataset.input_size, n_categories) for name, p in predictor_constructors.iteritems() if name in predictor_constructors},
minibatch_size = minibatch_size,
test_epochs = sqrtspace(0, n_epochs, n_tests),
evaluation_function = percent_argmax_correct,
# online_test_callbacks={'perceptron': lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28))},
accumulators=accumulator
)
plot_learning_curves(learning_curves)
def demo_compare_dtp_methods(predictor_constructors,
n_epochs=10,
minibatch_size=20,
n_tests=20,
onehot=True,
accumulator=None):
dataset = get_mnist_dataset(flat=True, binarize=False)
n_categories = dataset.n_categories
if onehot:
dataset = dataset.to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
learning_curves = compare_predictors(
dataset=dataset,
online_predictors={
name: p(dataset.input_size, n_categories)
for name, p in predictor_constructors.iteritems()
if name in predictor_constructors
},
minibatch_size=minibatch_size,
test_epochs=sqrtspace(0, n_epochs, n_tests),
evaluation_function=percent_argmax_correct,
# online_test_callbacks={'perceptron': lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28))},
accumulators=accumulator)
plot_learning_curves(learning_curves)
def demo_compare_dtp_optimizers(
hidden_sizes=[240],
n_epochs=10,
minibatch_size=20,
n_tests=20,
hidden_activation='tanh',
):
dataset = get_mnist_dataset(flat=True).to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
def make_dtp_net(optimizer_constructor, output_fcn):
return DifferenceTargetMLP.from_initializer(
input_size=dataset.input_size,
output_size=dataset.target_size,
hidden_sizes=hidden_sizes,
optimizer_constructor=optimizer_constructor,
input_activation='sigm',
hidden_activation=hidden_activation,
output_activation=output_fcn,
w_init_mag=0.01,
noise=1,
).compile()
learning_curves = compare_predictors(
dataset=dataset,
online_predictors={
'SGD-0.001-softmax':
make_dtp_net(lambda: SimpleGradientDescent(0.001),
output_fcn='softmax'),
'AdaMax-0.001-softmax':
make_dtp_net(lambda: AdaMax(0.001), output_fcn='softmax'),
'RMSProp-0.001-softmax':
make_dtp_net(lambda: RMSProp(0.001), output_fcn='softmax'),
'SGD-0.001-sigm':
make_dtp_net(lambda: SimpleGradientDescent(0.001),
output_fcn='sigm'),
'AdaMax-0.001-sigm':
make_dtp_net(lambda: AdaMax(0.001), output_fcn='sigm'),
'RMSProp-0.001-sigm':
make_dtp_net(lambda: RMSProp(0.001), output_fcn='sigm'),
},
minibatch_size=minibatch_size,
test_epochs=sqrtspace(0, n_epochs, n_tests),
evaluation_function=percent_argmax_correct,
)
plot_learning_curves(learning_curves)
def demo_run_dtp_on_mnist(hidden_sizes=[240],
n_epochs=20,
n_tests=20,
minibatch_size=100,
input_activation='sigm',
hidden_activation='tanh',
output_activation='softmax',
optimizer_constructor=lambda: RMSProp(0.001),
normalize_inputs=False,
local_cost_function=mean_squared_error,
output_cost_function=None,
noise=1,
lin_dtp=False,
seed=1234):
dataset = get_mnist_dataset(flat=True).to_onehot()
if normalize_inputs:
dataset = dataset.process_with(targets_processor=multichannel(
lambda x: x / np.sum(x, axis=1, keepdims=True)))
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
predictor = DifferenceTargetMLP.from_initializer(
input_size=dataset.input_size,
output_size=dataset.target_size,
hidden_sizes=hidden_sizes,
optimizer_constructor=
optimizer_constructor, # Note that RMSProp/AdaMax way outperform SGD here.
# input_activation=input_activation,
hidden_activation=hidden_activation,
output_activation=output_activation,
w_init_mag=0.01,
output_cost_function=output_cost_function,
noise=noise,
cost_function=local_cost_function,
layer_constructor=DifferenceTargetLayer.from_initializer if not lin_dtp
else PreActivationDifferenceTargetLayer.from_initializer,
rng=seed).compile()
result = assess_online_predictor(
predictor=predictor,
dataset=dataset,
minibatch_size=minibatch_size,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_tests),
test_callback=lambda p: dbplot(p.symbolic_predictor.layers[0].w.
get_value().T.reshape(-1, 28, 28)))
plot_learning_curves(result)
def demo_run_dtp_on_mnist(
hidden_sizes = [240],
n_epochs = 20,
n_tests = 20,
minibatch_size=100,
input_activation = 'sigm',
hidden_activation = 'tanh',
output_activation = 'softmax',
optimizer_constructor = lambda: RMSProp(0.001),
normalize_inputs = False,
local_cost_function = mean_squared_error,
output_cost_function = None,
noise = 1,
lin_dtp = False,
seed = 1234
):
dataset = get_mnist_dataset(flat = True).to_onehot()
if normalize_inputs:
dataset = dataset.process_with(targets_processor=multichannel(lambda x: x/np.sum(x, axis = 1, keepdims=True)))
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
predictor = DifferenceTargetMLP.from_initializer(
input_size = dataset.input_size,
output_size = dataset.target_size,
hidden_sizes = hidden_sizes,
optimizer_constructor = optimizer_constructor, # Note that RMSProp/AdaMax way outperform SGD here.
# input_activation=input_activation,
hidden_activation=hidden_activation,
output_activation=output_activation,
w_init_mag=0.01,
output_cost_function=output_cost_function,
noise = noise,
cost_function = local_cost_function,
layer_constructor=DifferenceTargetLayer.from_initializer if not lin_dtp else PreActivationDifferenceTargetLayer.from_initializer,
rng = seed
).compile()
result = assess_online_predictor(
predictor = predictor,
dataset = dataset,
minibatch_size=minibatch_size,
evaluation_function='percent_argmax_correct',
test_epochs = sqrtspace(0, n_epochs, n_tests),
test_callback=lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28))
)
plot_learning_curves(result)
def demo_dtp_varieties(
hidden_sizes = [240],
n_epochs = 10,
minibatch_size = 20,
n_tests = 20,
hidden_activation = 'tanh',
output_activation = 'sigm',
optimizer = 'adamax',
learning_rate = 0.01,
noise = 1,
predictors = ['MLP', 'DTP', 'PreAct-DTP', 'Linear-DTP'],
rng = 1234,
live_plot = False,
plot = False
):
"""
;
:param hidden_sizes:
:param n_epochs:
:param minibatch_size:
:param n_tests:
:return:
"""
if isinstance(predictors, str):
predictors = [predictors]
dataset = get_mnist_dataset(flat = True)
dataset = dataset.process_with(targets_processor=lambda (x, ): (OneHotEncoding(10)(x).astype(int), ))
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 0.1
n_tests = 3
set_default_figure_size(12, 9)
predictors = OrderedDict((name, get_predictor(name, input_size = dataset.input_size, target_size=dataset.target_size,
hidden_sizes=hidden_sizes, hidden_activation=hidden_activation, output_activation = output_activation,
optimizer=optimizer, learning_rate=learning_rate, noise = noise, rng = rng)) for name in predictors)
learning_curves = compare_predictors(
dataset=dataset,
online_predictors = predictors,
minibatch_size = minibatch_size,
test_epochs = sqrtspace(0, n_epochs, n_tests),
evaluation_function = percent_argmax_correct,
)
if plot:
plot_learning_curves(learning_curves)
def demo_mnist_online_regression(
minibatch_size = 10,
learning_rate = 0.1,
optimizer = 'sgd',
regressor_type = 'multinomial',
n_epochs = 20,
n_test_points = 30,
max_training_samples = None,
include_biases = True,
):
"""
Train an MLP on MNIST and print the test scores as training progresses.
"""
if is_test_mode():
n_test_points = 3
minibatch_size = 5
n_epochs = 0.01
dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30, flat = True)
else:
dataset = get_mnist_dataset(n_training_samples=max_training_samples, flat = True)
assert regressor_type in ('multinomial', 'logistic', 'linear')
n_outputs = dataset.n_categories
if regressor_type in ('logistic', 'linear'):
dataset = dataset.to_onehot()
predictor = OnlineRegressor(
input_size = dataset.input_size,
output_size = n_outputs,
regressor_type = regressor_type,
optimizer=get_named_optimizer(name = optimizer, learning_rate=learning_rate),
include_biases = include_biases
).compile()
# Train and periodically report the test score.
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size
)
plot_learning_curves(results)
def demo_mnist_online_regression(
minibatch_size = 10,
learning_rate = 0.1,
optimizer = 'sgd',
regressor_type = 'multinomial',
n_epochs = 20,
n_test_points = 30,
max_training_samples = None,
include_biases = True,
):
"""
Train an MLP on MNIST and print the test scores as training progresses.
"""
if is_test_mode():
n_test_points = 3
minibatch_size = 5
n_epochs = 0.01
dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30, flat = True)
else:
dataset = get_mnist_dataset(n_training_samples=max_training_samples, flat = True)
assert regressor_type in ('multinomial', 'logistic', 'linear')
n_outputs = dataset.n_categories
if regressor_type in ('logistic', 'linear'):
dataset = dataset.to_onehot()
predictor = OnlineRegressor(
input_size = dataset.input_size,
output_size = n_outputs,
regressor_type = regressor_type,
optimizer=get_named_optimizer(name = optimizer, learning_rate=learning_rate),
include_biases = include_biases
).compile()
# Train and periodically report the test score.
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size
)
plot_learning_curves(results)
def demo_compare_dtp_optimizers(
hidden_sizes = [240],
n_epochs = 10,
minibatch_size = 20,
n_tests = 20,
hidden_activation = 'tanh',
):
dataset = get_mnist_dataset(flat = True).to_onehot()
if is_test_mode():
dataset = dataset.shorten(200)
n_epochs = 1
n_tests = 2
def make_dtp_net(optimizer_constructor, output_fcn):
return DifferenceTargetMLP.from_initializer(
input_size = dataset.input_size,
output_size = dataset.target_size,
hidden_sizes = hidden_sizes,
optimizer_constructor = optimizer_constructor,
input_activation='sigm',
hidden_activation=hidden_activation,
output_activation=output_fcn,
w_init_mag=0.01,
noise = 1,
).compile()
learning_curves = compare_predictors(
dataset=dataset,
online_predictors = {
'SGD-0.001-softmax': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn = 'softmax'),
'AdaMax-0.001-softmax': make_dtp_net(lambda: AdaMax(0.001), output_fcn = 'softmax'),
'RMSProp-0.001-softmax': make_dtp_net(lambda: RMSProp(0.001), output_fcn = 'softmax'),
'SGD-0.001-sigm': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn = 'sigm'),
'AdaMax-0.001-sigm': make_dtp_net(lambda: AdaMax(0.001), output_fcn = 'sigm'),
'RMSProp-0.001-sigm': make_dtp_net(lambda: RMSProp(0.001), output_fcn = 'sigm'),
},
minibatch_size = minibatch_size,
test_epochs = sqrtspace(0, n_epochs, n_tests),
evaluation_function = percent_argmax_correct,
)
plot_learning_curves(learning_curves)
def test_compare_predictors_old():
x_tr, y_tr, x_ts, y_ts, w_true = get_logistic_regression_data(noise_factor = 0.1)
dataset = DataSet(DataCollection(x_tr, y_tr), DataCollection(x_ts, y_ts)).process_with(targets_processor=lambda (x, ): (OneHotEncoding()(x[:, 0]), ))
w_init = 0.1*np.random.randn(dataset.training_set.input.shape[1], dataset.training_set.target.shape[1])
records = compare_predictors_old(
dataset = dataset,
offline_predictor_constructors={
'Optimal': lambda: MockPredictor(lambda x: sigm(x.dot(w_true)))
},
online_predictor_constructors={
'fast-perceptron': lambda: Perceptron(alpha = 0.1, w = w_init.copy()),
'slow-perceptron': lambda: Perceptron(alpha = 0.001, w = w_init.copy())
},
minibatch_size = 10,
test_points = sqrtspace(0, 10, 20),
evaluation_function='mse'
)
plot_learning_curves(records, hang = False)
def test_compare_predictors(hang_plot = False):
dataset = get_synthetic_clusters_dataset()
w_constructor = lambda rng = np.random.RandomState(45): .1*rng.randn(dataset.input_shape[0], dataset.n_categories)
records = compare_predictors(
dataset = dataset,
offline_predictors={
'SVM': SVC()
},
online_predictors={
'fast-perceptron': Perceptron(alpha = 0.1, w = w_constructor()).to_categorical(),
'slow-perceptron': Perceptron(alpha = 0.001, w = w_constructor()).to_categorical()
},
minibatch_size = 10,
test_epochs = sqrtspace(0, 10, 20),
evaluation_function='percent_correct'
)
assert 99 < records['SVM'].get_scores('Test') <= 100
assert 20 < records['slow-perceptron'].get_scores('Test')[0] < 40 and 95 < records['slow-perceptron'].get_scores('Test')[-1] <= 100
assert 20 < records['fast-perceptron'].get_scores('Test')[0] < 40 and 98 < records['fast-perceptron'].get_scores('Test')[-1] <= 100
plot_learning_curves(records, hang = hang_plot)
def demo_mnist_mlp(
minibatch_size = 10,
learning_rate = 0.1,
optimizer = 'sgd',
hidden_sizes = [300],
w_init = 0.01,
hidden_activation = 'tanh',
output_activation = 'softmax',
cost = 'nll-d',
visualize_params = False,
n_test_points = 30,
n_epochs = 10,
max_training_samples = None,
use_bias = True,
onehot = False,
rng = 1234,
plot = False,
):
"""
Train an MLP on MNIST and print the test scores as training progresses.
"""
if is_test_mode():
n_test_points = 3
minibatch_size = 5
n_epochs = 0.01
dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30)
else:
dataset = get_mnist_dataset(n_training_samples=max_training_samples)
if onehot:
dataset = dataset.to_onehot()
if minibatch_size == 'full':
minibatch_size = dataset.training_set.n_samples
optimizer = get_named_optimizer(name = optimizer, learning_rate=learning_rate)
# Setup the training and test functions
predictor = GradientBasedPredictor(
function = MultiLayerPerceptron.from_init(
layer_sizes=[dataset.input_size]+hidden_sizes+[10],
hidden_activation=hidden_activation,
output_activation=output_activation,
w_init = w_init,
use_bias=use_bias,
rng = rng,
),
cost_function=cost,
optimizer=optimizer
).compile() # .compile() turns the GradientBasedPredictor, which works with symbolic variables, into a real one that takes and returns arrays.
def vis_callback(xx):
p = predictor.symbolic_predictor._function
in_layer = {
'Layer[0].w': p.layers[0].linear_transform._w.get_value().T.reshape(-1, 28, 28),
'Layer[0].b': p.layers[0].linear_transform._b.get_value(),
}
other_layers = [{'Layer[%s].w' % (i+1): l.linear_transform._w.get_value(), 'Layer[%s].b' % (i+1): l.linear_transform._b.get_value()} for i, l in enumerate(p.layers[1:])]
dbplot(dict(in_layer.items() + sum([o.items() for o in other_layers], [])))
# Train and periodically report the test score.
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size,
test_callback=vis_callback if visualize_params else None
)
if plot:
plot_learning_curves(results)
def run_and_plot(training_scheme):
learning_curves = training_scheme()
plot_learning_curves(learning_curves)
def run_and_plot(training_scheme):
learning_curves = training_scheme()
plot_learning_curves(learning_curves)
def compare_spiking_to_nonspiking(hidden_sizes = [300, 300], eta=0.01, w_init=0.01, fractional = False, n_epochs = 20,
forward_discretize = 'rect-herding', back_discretize = 'noreset-herding', test_discretize='rect-herding', save_results = False):
mnist = get_mnist_dataset(flat=True).to_onehot()
test_epochs=[0.0, 0.05, 0.1, 0.2, 0.5]+range(1, n_epochs+1)
if is_test_mode():
mnist = mnist.shorten(500)
eta = 0.01
w_init=0.01
test_epochs = [0.0, 0.05, 0.1]
spiking_net = JavaSpikingNetWrapper.from_init(
fractional = fractional,
depth_first=False,
smooth_grads = False,
forward_discretize = forward_discretize,
back_discretize = back_discretize,
test_discretize = test_discretize,
w_init=w_init,
hold_error=True,
rng = 1234,
n_steps = 10,
eta=eta,
layer_sizes=[784]+hidden_sizes+[10],
)
relu_net = GradientBasedPredictor(
MultiLayerPerceptron.from_init(
hidden_activation = 'relu',
output_activation = 'relu',
layer_sizes=[784]+hidden_sizes+[10],
use_bias=False,
w_init=w_init,
rng=1234,
),
cost_function = 'mse',
optimizer=GradientDescent(eta)
).compile()
# Listen for spikes
forward_eavesdropper = jp.JClass('nl.uva.deepspike.eavesdroppers.SpikeCountingEavesdropper')()
backward_eavesdropper = jp.JClass('nl.uva.deepspike.eavesdroppers.SpikeCountingEavesdropper')()
for lay in spiking_net.jnet.layers:
lay.forward_herder.add_eavesdropper(forward_eavesdropper)
for lay in spiking_net.jnet.layers[1:]:
lay.backward_herder.add_eavesdropper(backward_eavesdropper)
spiking_net.jnet.error_counter.add_eavesdropper(backward_eavesdropper)
forward_counts = []
backward_counts = []
def register_counts():
forward_counts.append(forward_eavesdropper.get_count())
backward_counts.append(backward_eavesdropper.get_count())
results = compare_predictors(
dataset=mnist,
online_predictors={
'Spiking-MLP': spiking_net,
'ReLU-MLP': relu_net,
},
test_epochs=test_epochs,
online_test_callbacks=lambda p: register_counts() if p is spiking_net else None,
minibatch_size = 1,
test_on = 'training+test',
evaluation_function=percent_argmax_incorrect,
)
spiking_params = [np.array(lay.forward_weights.w.asFloat()).copy() for lay in spiking_net.jnet.layers]
relu_params = [param.get_value().astype(np.float64) for param in relu_net.parameters]
# See what the score is when we apply the final spiking weights to the
offline_trained_spiking_net = JavaSpikingNetWrapper(
ws=relu_params,
fractional = fractional,
depth_first=False,
smooth_grads = False,
forward_discretize = forward_discretize,
back_discretize = back_discretize,
test_discretize = test_discretize,
hold_error=True,
n_steps = 10,
eta=eta,
)
# for spiking_layer, p in zip(spiking_net.jnet.layers, relu_params):
# spiking_layer.w = p.astype(np.float64)
error = [
('Test', percent_argmax_incorrect(offline_trained_spiking_net.predict(mnist.test_set.input), mnist.test_set.target)),
('Training', percent_argmax_incorrect(offline_trained_spiking_net.predict(mnist.training_set.input), mnist.training_set.target))
]
results['Spiking-MLP with ReLU weights'] = LearningCurveData()
results['Spiking-MLP with ReLU weights'].add(None, error)
print 'Spiking-MLP with ReLU weights: %s' % error
# --------------------------------------------------------------------------
# See what the score is when we plug the spiking weights into the ReLU net.
for param, sval in zip(relu_net.parameters, spiking_params):
param.set_value(sval)
error = [
('Test', percent_argmax_incorrect(relu_net.predict(mnist.test_set.input), mnist.test_set.target)),
('Training', percent_argmax_incorrect(relu_net.predict(mnist.training_set.input), mnist.training_set.target))
]
results['ReLU-MLP with Spiking weights'] = LearningCurveData()
results['ReLU-MLP with Spiking weights'].add(None, error)
print 'ReLU-MLP with Spiking weights: %s' % error
# --------------------------------------------------------------------------
if save_results:
with open("mnist_relu_vs_spiking_results-%s.pkl" % datetime.now(), 'w') as f:
pickle.dump(results, f)
# Problem: this currently includes test
forward_rates = np.diff(forward_counts) / (np.diff(test_epochs)*60000)
backward_rates = np.diff(backward_counts) / (np.diff(test_epochs)*60000)
plt.figure('ReLU vs Spikes')
plt.subplot(211)
plot_learning_curves(results, title = "MNIST Learning Curves", hang=False, figure_name='ReLU vs Spikes', xscale='linear', yscale='log', y_title='Percent Error')
plt.subplot(212)
plt.plot(test_epochs[1:], forward_rates)
plt.plot(test_epochs[1:], backward_rates)
plt.xlabel('Epoch')
plt.ylabel('n_spikes')
plt.legend(['Mean Forward Spikes', 'Mean Backward Spikes'], loc='best')
plt.interactive(is_test_mode())
plt.show()
function = MultiLayerPerceptron.from_init(
layer_sizes=[dataset.input_size, 500, dataset.n_categories],
hidden_activation='sig', # Sigmoidal hidden units
output_activation='softmax', # Softmax output unit, since we're doing multinomial classification
w_init = 0.01,
rng = 5
),
cost_function = negative_log_likelihood_dangerous, # "Dangerous" because it doesn't check to see that output is normalized, but we know it is because it comes from softmax.
optimizer = SimpleGradientDescent(eta = 0.1),
).compile(), # .compile() returns an IPredictor
},
offline_predictors={
'RF': RandomForestClassifier(n_estimators = 40)
},
minibatch_size = minibatch_size,
test_epochs = sqrtspace(0, n_epochs, n_tests),
evaluation_function = percent_argmax_correct # Compares one-hot
)
# Results is a LearningCurveData object
return learning_curve_data
if __name__ == '__main__':
set_test_mode(False)
records = compare_example_predictors(
n_epochs=30,
minibatch_size=20,
)
plot_learning_curves(records)
layer_sizes=[
dataset.input_size, 500, dataset.n_categories
],
hidden_activation='sig', # Sigmoidal hidden units
output_activation=
'softmax', # Softmax output unit, since we're doing multinomial classification
w_init=0.01,
rng=5),
cost_function=
negative_log_likelihood_dangerous, # "Dangerous" because it doesn't check to see that output is normalized, but we know it is because it comes from softmax.
optimizer=SimpleGradientDescent(eta=0.1),
).compile(), # .compile() returns an IPredictor
},
offline_predictors={'RF': RandomForestClassifier(n_estimators=40)},
minibatch_size=minibatch_size,
test_epochs=sqrtspace(0, n_epochs, n_tests),
evaluation_function=percent_argmax_correct # Compares one-hot
)
# Results is a LearningCurveData object
return learning_curve_data
if __name__ == '__main__':
set_test_mode(False)
records = compare_example_predictors(
n_epochs=30,
minibatch_size=20,
)
plot_learning_curves(records)
def demo_mnist_mlp(
minibatch_size = 10,
learning_rate = 0.1,
optimizer = 'sgd',
hidden_sizes = [300],
w_init = 0.01,
hidden_activation = 'tanh',
output_activation = 'softmax',
cost = 'nll-d',
visualize_params = False,
n_test_points = 30,
n_epochs = 10,
max_training_samples = None,
use_bias = True,
onehot = False,
rng = 1234,
plot = False,
):
"""
Train an MLP on MNIST and print the test scores as training progresses.
"""
if is_test_mode():
n_test_points = 3
minibatch_size = 5
n_epochs = 0.01
dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30)
else:
dataset = get_mnist_dataset(n_training_samples=max_training_samples)
if onehot:
dataset = dataset.to_onehot()
if minibatch_size == 'full':
minibatch_size = dataset.training_set.n_samples
optimizer = get_named_optimizer(name = optimizer, learning_rate=learning_rate)
# Setup the training and test functions
predictor = GradientBasedPredictor(
function = MultiLayerPerceptron.from_init(
layer_sizes=[dataset.input_size]+hidden_sizes+[10],
hidden_activation=hidden_activation,
output_activation=output_activation,
w_init = w_init,
use_bias=use_bias,
rng = rng,
),
cost_function=cost,
optimizer=optimizer
).compile() # .compile() turns the GradientBasedPredictor, which works with symbolic variables, into a real one that takes and returns arrays.
def vis_callback(xx):
p = predictor.symbolic_predictor._function
in_layer = {
'Layer[0].w': p.layers[0].linear_transform._w.get_value().T.reshape(-1, 28, 28),
'Layer[0].b': p.layers[0].linear_transform._b.get_value(),
}
other_layers = [{'Layer[%s].w' % (i+1): l.linear_transform._w.get_value(), 'Layer[%s].b' % (i+1): l.linear_transform._b.get_value()} for i, l in enumerate(p.layers[1:])]
dbplot(dict(in_layer.items() + sum([o.items() for o in other_layers], [])))
# Train and periodically report the test score.
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size,
test_callback=vis_callback if visualize_params else None
)
if plot:
plot_learning_curves(results)
