def test_compare_predictors(hang_plot=False): dataset = get_synthetic_clusters_dataset() w_constructor = lambda rng=np.random.RandomState(45): .1 * rng.randn( dataset.input_shape[0], dataset.n_categories) records = compare_predictors( dataset=dataset, offline_predictors={'SVM': SVC()}, online_predictors={ 'fast-perceptron': Perceptron(alpha=0.1, w=w_constructor()).to_categorical(), 'slow-perceptron': Perceptron(alpha=0.001, w=w_constructor()).to_categorical() }, minibatch_size=10, test_epochs=sqrtspace(0, 10, 20), evaluation_function='percent_correct') assert 99 < records['SVM'].get_scores('Test') <= 100 assert 20 < records['slow-perceptron'].get_scores( 'Test')[0] < 40 and 95 < records['slow-perceptron'].get_scores( 'Test')[-1] <= 100 assert 20 < records['fast-perceptron'].get_scores( 'Test')[0] < 40 and 98 < records['fast-perceptron'].get_scores( 'Test')[-1] <= 100 plot_learning_curves(records, hang=hang_plot)
def demo_perceptron_dtp( hidden_sizes=[240], n_epochs=20, n_tests=20, minibatch_size=100, lin_dtp=True, ): dataset = get_mnist_dataset(flat=True).to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 predictor = DifferenceTargetMLP(layers=[ PerceptronLayer.from_initializer(n_in, n_out, initial_mag=2, lin_dtp=lin_dtp) for n_in, n_out in zip([dataset.input_size] + hidden_sizes, hidden_sizes + [dataset.target_size]) ], output_cost_function=None).compile() result = assess_online_predictor( predictor=predictor, dataset=dataset, minibatch_size=minibatch_size, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_tests), ) plot_learning_curves(result)
def demo_perceptron_dtp( hidden_sizes = [240], n_epochs = 20, n_tests = 20, minibatch_size=100, lin_dtp = True, ): dataset = get_mnist_dataset(flat = True).to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 predictor = DifferenceTargetMLP( layers=[PerceptronLayer.from_initializer(n_in, n_out, initial_mag=2, lin_dtp = lin_dtp) for n_in, n_out in zip([dataset.input_size]+hidden_sizes, hidden_sizes+[dataset.target_size])], output_cost_function = None ).compile() result = assess_online_predictor( predictor = predictor, dataset = dataset, minibatch_size=minibatch_size, evaluation_function='percent_argmax_correct', test_epochs = sqrtspace(0, n_epochs, n_tests), ) plot_learning_curves(result)
def demo_dtp_varieties(hidden_sizes=[240], n_epochs=10, minibatch_size=20, n_tests=20, hidden_activation='tanh', output_activation='sigm', optimizer='adamax', learning_rate=0.01, noise=1, predictors=['MLP', 'DTP', 'PreAct-DTP', 'Linear-DTP'], rng=1234, use_bias=True, live_plot=False, plot=False): """ ; :param hidden_sizes: :param n_epochs: :param minibatch_size: :param n_tests: :return: """ if isinstance(predictors, str): predictors = [predictors] dataset = get_mnist_dataset(flat=True) dataset = dataset.process_with( targets_processor=lambda (x, ): (OneHotEncoding(10)(x).astype(int), )) if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 0.1 n_tests = 3 set_default_figure_size(12, 9) predictors = OrderedDict( (name, get_predictor(name, input_size=dataset.input_size, target_size=dataset.target_size, hidden_sizes=hidden_sizes, hidden_activation=hidden_activation, output_activation=output_activation, optimizer=optimizer, learning_rate=learning_rate, noise=noise, use_bias=use_bias, rng=rng)) for name in predictors) learning_curves = compare_predictors( dataset=dataset, online_predictors=predictors, minibatch_size=minibatch_size, test_epochs=sqrtspace(0, n_epochs, n_tests), evaluation_function=percent_argmax_correct, ) if plot: plot_learning_curves(learning_curves)
def demo_compare_dtp_methods( predictor_constructors, n_epochs = 10, minibatch_size = 20, n_tests = 20, onehot = True, accumulator = None ): dataset = get_mnist_dataset(flat = True, binarize = False) n_categories = dataset.n_categories if onehot: dataset = dataset.to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 learning_curves = compare_predictors( dataset=dataset, online_predictors = {name: p(dataset.input_size, n_categories) for name, p in predictor_constructors.iteritems() if name in predictor_constructors}, minibatch_size = minibatch_size, test_epochs = sqrtspace(0, n_epochs, n_tests), evaluation_function = percent_argmax_correct, # online_test_callbacks={'perceptron': lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28))}, accumulators=accumulator ) plot_learning_curves(learning_curves)
def demo_compare_dtp_methods(predictor_constructors, n_epochs=10, minibatch_size=20, n_tests=20, onehot=True, accumulator=None): dataset = get_mnist_dataset(flat=True, binarize=False) n_categories = dataset.n_categories if onehot: dataset = dataset.to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 learning_curves = compare_predictors( dataset=dataset, online_predictors={ name: p(dataset.input_size, n_categories) for name, p in predictor_constructors.iteritems() if name in predictor_constructors }, minibatch_size=minibatch_size, test_epochs=sqrtspace(0, n_epochs, n_tests), evaluation_function=percent_argmax_correct, # online_test_callbacks={'perceptron': lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28))}, accumulators=accumulator) plot_learning_curves(learning_curves)
def demo_compare_dtp_optimizers( hidden_sizes=[240], n_epochs=10, minibatch_size=20, n_tests=20, hidden_activation='tanh', ): dataset = get_mnist_dataset(flat=True).to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 def make_dtp_net(optimizer_constructor, output_fcn): return DifferenceTargetMLP.from_initializer( input_size=dataset.input_size, output_size=dataset.target_size, hidden_sizes=hidden_sizes, optimizer_constructor=optimizer_constructor, input_activation='sigm', hidden_activation=hidden_activation, output_activation=output_fcn, w_init_mag=0.01, noise=1, ).compile() learning_curves = compare_predictors( dataset=dataset, online_predictors={ 'SGD-0.001-softmax': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn='softmax'), 'AdaMax-0.001-softmax': make_dtp_net(lambda: AdaMax(0.001), output_fcn='softmax'), 'RMSProp-0.001-softmax': make_dtp_net(lambda: RMSProp(0.001), output_fcn='softmax'), 'SGD-0.001-sigm': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn='sigm'), 'AdaMax-0.001-sigm': make_dtp_net(lambda: AdaMax(0.001), output_fcn='sigm'), 'RMSProp-0.001-sigm': make_dtp_net(lambda: RMSProp(0.001), output_fcn='sigm'), }, minibatch_size=minibatch_size, test_epochs=sqrtspace(0, n_epochs, n_tests), evaluation_function=percent_argmax_correct, ) plot_learning_curves(learning_curves)
def demo_run_dtp_on_mnist(hidden_sizes=[240], n_epochs=20, n_tests=20, minibatch_size=100, input_activation='sigm', hidden_activation='tanh', output_activation='softmax', optimizer_constructor=lambda: RMSProp(0.001), normalize_inputs=False, local_cost_function=mean_squared_error, output_cost_function=None, noise=1, lin_dtp=False, seed=1234): dataset = get_mnist_dataset(flat=True).to_onehot() if normalize_inputs: dataset = dataset.process_with(targets_processor=multichannel( lambda x: x / np.sum(x, axis=1, keepdims=True))) if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 predictor = DifferenceTargetMLP.from_initializer( input_size=dataset.input_size, output_size=dataset.target_size, hidden_sizes=hidden_sizes, optimizer_constructor= optimizer_constructor, # Note that RMSProp/AdaMax way outperform SGD here. # input_activation=input_activation, hidden_activation=hidden_activation, output_activation=output_activation, w_init_mag=0.01, output_cost_function=output_cost_function, noise=noise, cost_function=local_cost_function, layer_constructor=DifferenceTargetLayer.from_initializer if not lin_dtp else PreActivationDifferenceTargetLayer.from_initializer, rng=seed).compile() result = assess_online_predictor( predictor=predictor, dataset=dataset, minibatch_size=minibatch_size, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_tests), test_callback=lambda p: dbplot(p.symbolic_predictor.layers[0].w. get_value().T.reshape(-1, 28, 28))) plot_learning_curves(result)
def demo_run_dtp_on_mnist( hidden_sizes = [240], n_epochs = 20, n_tests = 20, minibatch_size=100, input_activation = 'sigm', hidden_activation = 'tanh', output_activation = 'softmax', optimizer_constructor = lambda: RMSProp(0.001), normalize_inputs = False, local_cost_function = mean_squared_error, output_cost_function = None, noise = 1, lin_dtp = False, seed = 1234 ): dataset = get_mnist_dataset(flat = True).to_onehot() if normalize_inputs: dataset = dataset.process_with(targets_processor=multichannel(lambda x: x/np.sum(x, axis = 1, keepdims=True))) if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 predictor = DifferenceTargetMLP.from_initializer( input_size = dataset.input_size, output_size = dataset.target_size, hidden_sizes = hidden_sizes, optimizer_constructor = optimizer_constructor, # Note that RMSProp/AdaMax way outperform SGD here. # input_activation=input_activation, hidden_activation=hidden_activation, output_activation=output_activation, w_init_mag=0.01, output_cost_function=output_cost_function, noise = noise, cost_function = local_cost_function, layer_constructor=DifferenceTargetLayer.from_initializer if not lin_dtp else PreActivationDifferenceTargetLayer.from_initializer, rng = seed ).compile() result = assess_online_predictor( predictor = predictor, dataset = dataset, minibatch_size=minibatch_size, evaluation_function='percent_argmax_correct', test_epochs = sqrtspace(0, n_epochs, n_tests), test_callback=lambda p: dbplot(p.symbolic_predictor.layers[0].w.get_value().T.reshape(-1, 28, 28)) ) plot_learning_curves(result)
def demo_dtp_varieties( hidden_sizes = [240], n_epochs = 10, minibatch_size = 20, n_tests = 20, hidden_activation = 'tanh', output_activation = 'sigm', optimizer = 'adamax', learning_rate = 0.01, noise = 1, predictors = ['MLP', 'DTP', 'PreAct-DTP', 'Linear-DTP'], rng = 1234, live_plot = False, plot = False ): """ ; :param hidden_sizes: :param n_epochs: :param minibatch_size: :param n_tests: :return: """ if isinstance(predictors, str): predictors = [predictors] dataset = get_mnist_dataset(flat = True) dataset = dataset.process_with(targets_processor=lambda (x, ): (OneHotEncoding(10)(x).astype(int), )) if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 0.1 n_tests = 3 set_default_figure_size(12, 9) predictors = OrderedDict((name, get_predictor(name, input_size = dataset.input_size, target_size=dataset.target_size, hidden_sizes=hidden_sizes, hidden_activation=hidden_activation, output_activation = output_activation, optimizer=optimizer, learning_rate=learning_rate, noise = noise, rng = rng)) for name in predictors) learning_curves = compare_predictors( dataset=dataset, online_predictors = predictors, minibatch_size = minibatch_size, test_epochs = sqrtspace(0, n_epochs, n_tests), evaluation_function = percent_argmax_correct, ) if plot: plot_learning_curves(learning_curves)
def demo_mnist_online_regression( minibatch_size = 10, learning_rate = 0.1, optimizer = 'sgd', regressor_type = 'multinomial', n_epochs = 20, n_test_points = 30, max_training_samples = None, include_biases = True, ): """ Train an MLP on MNIST and print the test scores as training progresses. """ if is_test_mode(): n_test_points = 3 minibatch_size = 5 n_epochs = 0.01 dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30, flat = True) else: dataset = get_mnist_dataset(n_training_samples=max_training_samples, flat = True) assert regressor_type in ('multinomial', 'logistic', 'linear') n_outputs = dataset.n_categories if regressor_type in ('logistic', 'linear'): dataset = dataset.to_onehot() predictor = OnlineRegressor( input_size = dataset.input_size, output_size = n_outputs, regressor_type = regressor_type, optimizer=get_named_optimizer(name = optimizer, learning_rate=learning_rate), include_biases = include_biases ).compile() # Train and periodically report the test score. results = assess_online_predictor( dataset=dataset, predictor=predictor, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_test_points), minibatch_size=minibatch_size ) plot_learning_curves(results)
def demo_mnist_online_regression( minibatch_size = 10, learning_rate = 0.1, optimizer = 'sgd', regressor_type = 'multinomial', n_epochs = 20, n_test_points = 30, max_training_samples = None, include_biases = True, ): """ Train an MLP on MNIST and print the test scores as training progresses. """ if is_test_mode(): n_test_points = 3 minibatch_size = 5 n_epochs = 0.01 dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30, flat = True) else: dataset = get_mnist_dataset(n_training_samples=max_training_samples, flat = True) assert regressor_type in ('multinomial', 'logistic', 'linear') n_outputs = dataset.n_categories if regressor_type in ('logistic', 'linear'): dataset = dataset.to_onehot() predictor = OnlineRegressor( input_size = dataset.input_size, output_size = n_outputs, regressor_type = regressor_type, optimizer=get_named_optimizer(name = optimizer, learning_rate=learning_rate), include_biases = include_biases ).compile() # Train and periodically report the test score. results = assess_online_predictor( dataset=dataset, predictor=predictor, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_test_points), minibatch_size=minibatch_size ) plot_learning_curves(results)
def demo_compare_dtp_optimizers( hidden_sizes = [240], n_epochs = 10, minibatch_size = 20, n_tests = 20, hidden_activation = 'tanh', ): dataset = get_mnist_dataset(flat = True).to_onehot() if is_test_mode(): dataset = dataset.shorten(200) n_epochs = 1 n_tests = 2 def make_dtp_net(optimizer_constructor, output_fcn): return DifferenceTargetMLP.from_initializer( input_size = dataset.input_size, output_size = dataset.target_size, hidden_sizes = hidden_sizes, optimizer_constructor = optimizer_constructor, input_activation='sigm', hidden_activation=hidden_activation, output_activation=output_fcn, w_init_mag=0.01, noise = 1, ).compile() learning_curves = compare_predictors( dataset=dataset, online_predictors = { 'SGD-0.001-softmax': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn = 'softmax'), 'AdaMax-0.001-softmax': make_dtp_net(lambda: AdaMax(0.001), output_fcn = 'softmax'), 'RMSProp-0.001-softmax': make_dtp_net(lambda: RMSProp(0.001), output_fcn = 'softmax'), 'SGD-0.001-sigm': make_dtp_net(lambda: SimpleGradientDescent(0.001), output_fcn = 'sigm'), 'AdaMax-0.001-sigm': make_dtp_net(lambda: AdaMax(0.001), output_fcn = 'sigm'), 'RMSProp-0.001-sigm': make_dtp_net(lambda: RMSProp(0.001), output_fcn = 'sigm'), }, minibatch_size = minibatch_size, test_epochs = sqrtspace(0, n_epochs, n_tests), evaluation_function = percent_argmax_correct, ) plot_learning_curves(learning_curves)
def test_compare_predictors_old(): x_tr, y_tr, x_ts, y_ts, w_true = get_logistic_regression_data(noise_factor = 0.1) dataset = DataSet(DataCollection(x_tr, y_tr), DataCollection(x_ts, y_ts)).process_with(targets_processor=lambda (x, ): (OneHotEncoding()(x[:, 0]), )) w_init = 0.1*np.random.randn(dataset.training_set.input.shape[1], dataset.training_set.target.shape[1]) records = compare_predictors_old( dataset = dataset, offline_predictor_constructors={ 'Optimal': lambda: MockPredictor(lambda x: sigm(x.dot(w_true))) }, online_predictor_constructors={ 'fast-perceptron': lambda: Perceptron(alpha = 0.1, w = w_init.copy()), 'slow-perceptron': lambda: Perceptron(alpha = 0.001, w = w_init.copy()) }, minibatch_size = 10, test_points = sqrtspace(0, 10, 20), evaluation_function='mse' ) plot_learning_curves(records, hang = False)
def test_compare_predictors(hang_plot = False): dataset = get_synthetic_clusters_dataset() w_constructor = lambda rng = np.random.RandomState(45): .1*rng.randn(dataset.input_shape[0], dataset.n_categories) records = compare_predictors( dataset = dataset, offline_predictors={ 'SVM': SVC() }, online_predictors={ 'fast-perceptron': Perceptron(alpha = 0.1, w = w_constructor()).to_categorical(), 'slow-perceptron': Perceptron(alpha = 0.001, w = w_constructor()).to_categorical() }, minibatch_size = 10, test_epochs = sqrtspace(0, 10, 20), evaluation_function='percent_correct' ) assert 99 < records['SVM'].get_scores('Test') <= 100 assert 20 < records['slow-perceptron'].get_scores('Test')[0] < 40 and 95 < records['slow-perceptron'].get_scores('Test')[-1] <= 100 assert 20 < records['fast-perceptron'].get_scores('Test')[0] < 40 and 98 < records['fast-perceptron'].get_scores('Test')[-1] <= 100 plot_learning_curves(records, hang = hang_plot)
def demo_mnist_mlp( minibatch_size = 10, learning_rate = 0.1, optimizer = 'sgd', hidden_sizes = [300], w_init = 0.01, hidden_activation = 'tanh', output_activation = 'softmax', cost = 'nll-d', visualize_params = False, n_test_points = 30, n_epochs = 10, max_training_samples = None, use_bias = True, onehot = False, rng = 1234, plot = False, ): """ Train an MLP on MNIST and print the test scores as training progresses. """ if is_test_mode(): n_test_points = 3 minibatch_size = 5 n_epochs = 0.01 dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30) else: dataset = get_mnist_dataset(n_training_samples=max_training_samples) if onehot: dataset = dataset.to_onehot() if minibatch_size == 'full': minibatch_size = dataset.training_set.n_samples optimizer = get_named_optimizer(name = optimizer, learning_rate=learning_rate) # Setup the training and test functions predictor = GradientBasedPredictor( function = MultiLayerPerceptron.from_init( layer_sizes=[dataset.input_size]+hidden_sizes+[10], hidden_activation=hidden_activation, output_activation=output_activation, w_init = w_init, use_bias=use_bias, rng = rng, ), cost_function=cost, optimizer=optimizer ).compile() # .compile() turns the GradientBasedPredictor, which works with symbolic variables, into a real one that takes and returns arrays. def vis_callback(xx): p = predictor.symbolic_predictor._function in_layer = { 'Layer[0].w': p.layers[0].linear_transform._w.get_value().T.reshape(-1, 28, 28), 'Layer[0].b': p.layers[0].linear_transform._b.get_value(), } other_layers = [{'Layer[%s].w' % (i+1): l.linear_transform._w.get_value(), 'Layer[%s].b' % (i+1): l.linear_transform._b.get_value()} for i, l in enumerate(p.layers[1:])] dbplot(dict(in_layer.items() + sum([o.items() for o in other_layers], []))) # Train and periodically report the test score. results = assess_online_predictor( dataset=dataset, predictor=predictor, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_test_points), minibatch_size=minibatch_size, test_callback=vis_callback if visualize_params else None ) if plot: plot_learning_curves(results)
def run_and_plot(training_scheme): learning_curves = training_scheme() plot_learning_curves(learning_curves)
def run_and_plot(training_scheme): learning_curves = training_scheme() plot_learning_curves(learning_curves)
def compare_spiking_to_nonspiking(hidden_sizes = [300, 300], eta=0.01, w_init=0.01, fractional = False, n_epochs = 20, forward_discretize = 'rect-herding', back_discretize = 'noreset-herding', test_discretize='rect-herding', save_results = False): mnist = get_mnist_dataset(flat=True).to_onehot() test_epochs=[0.0, 0.05, 0.1, 0.2, 0.5]+range(1, n_epochs+1) if is_test_mode(): mnist = mnist.shorten(500) eta = 0.01 w_init=0.01 test_epochs = [0.0, 0.05, 0.1] spiking_net = JavaSpikingNetWrapper.from_init( fractional = fractional, depth_first=False, smooth_grads = False, forward_discretize = forward_discretize, back_discretize = back_discretize, test_discretize = test_discretize, w_init=w_init, hold_error=True, rng = 1234, n_steps = 10, eta=eta, layer_sizes=[784]+hidden_sizes+[10], ) relu_net = GradientBasedPredictor( MultiLayerPerceptron.from_init( hidden_activation = 'relu', output_activation = 'relu', layer_sizes=[784]+hidden_sizes+[10], use_bias=False, w_init=w_init, rng=1234, ), cost_function = 'mse', optimizer=GradientDescent(eta) ).compile() # Listen for spikes forward_eavesdropper = jp.JClass('nl.uva.deepspike.eavesdroppers.SpikeCountingEavesdropper')() backward_eavesdropper = jp.JClass('nl.uva.deepspike.eavesdroppers.SpikeCountingEavesdropper')() for lay in spiking_net.jnet.layers: lay.forward_herder.add_eavesdropper(forward_eavesdropper) for lay in spiking_net.jnet.layers[1:]: lay.backward_herder.add_eavesdropper(backward_eavesdropper) spiking_net.jnet.error_counter.add_eavesdropper(backward_eavesdropper) forward_counts = [] backward_counts = [] def register_counts(): forward_counts.append(forward_eavesdropper.get_count()) backward_counts.append(backward_eavesdropper.get_count()) results = compare_predictors( dataset=mnist, online_predictors={ 'Spiking-MLP': spiking_net, 'ReLU-MLP': relu_net, }, test_epochs=test_epochs, online_test_callbacks=lambda p: register_counts() if p is spiking_net else None, minibatch_size = 1, test_on = 'training+test', evaluation_function=percent_argmax_incorrect, ) spiking_params = [np.array(lay.forward_weights.w.asFloat()).copy() for lay in spiking_net.jnet.layers] relu_params = [param.get_value().astype(np.float64) for param in relu_net.parameters] # See what the score is when we apply the final spiking weights to the offline_trained_spiking_net = JavaSpikingNetWrapper( ws=relu_params, fractional = fractional, depth_first=False, smooth_grads = False, forward_discretize = forward_discretize, back_discretize = back_discretize, test_discretize = test_discretize, hold_error=True, n_steps = 10, eta=eta, ) # for spiking_layer, p in zip(spiking_net.jnet.layers, relu_params): # spiking_layer.w = p.astype(np.float64) error = [ ('Test', percent_argmax_incorrect(offline_trained_spiking_net.predict(mnist.test_set.input), mnist.test_set.target)), ('Training', percent_argmax_incorrect(offline_trained_spiking_net.predict(mnist.training_set.input), mnist.training_set.target)) ] results['Spiking-MLP with ReLU weights'] = LearningCurveData() results['Spiking-MLP with ReLU weights'].add(None, error) print 'Spiking-MLP with ReLU weights: %s' % error # -------------------------------------------------------------------------- # See what the score is when we plug the spiking weights into the ReLU net. for param, sval in zip(relu_net.parameters, spiking_params): param.set_value(sval) error = [ ('Test', percent_argmax_incorrect(relu_net.predict(mnist.test_set.input), mnist.test_set.target)), ('Training', percent_argmax_incorrect(relu_net.predict(mnist.training_set.input), mnist.training_set.target)) ] results['ReLU-MLP with Spiking weights'] = LearningCurveData() results['ReLU-MLP with Spiking weights'].add(None, error) print 'ReLU-MLP with Spiking weights: %s' % error # -------------------------------------------------------------------------- if save_results: with open("mnist_relu_vs_spiking_results-%s.pkl" % datetime.now(), 'w') as f: pickle.dump(results, f) # Problem: this currently includes test forward_rates = np.diff(forward_counts) / (np.diff(test_epochs)*60000) backward_rates = np.diff(backward_counts) / (np.diff(test_epochs)*60000) plt.figure('ReLU vs Spikes') plt.subplot(211) plot_learning_curves(results, title = "MNIST Learning Curves", hang=False, figure_name='ReLU vs Spikes', xscale='linear', yscale='log', y_title='Percent Error') plt.subplot(212) plt.plot(test_epochs[1:], forward_rates) plt.plot(test_epochs[1:], backward_rates) plt.xlabel('Epoch') plt.ylabel('n_spikes') plt.legend(['Mean Forward Spikes', 'Mean Backward Spikes'], loc='best') plt.interactive(is_test_mode()) plt.show()
function = MultiLayerPerceptron.from_init( layer_sizes=[dataset.input_size, 500, dataset.n_categories], hidden_activation='sig', # Sigmoidal hidden units output_activation='softmax', # Softmax output unit, since we're doing multinomial classification w_init = 0.01, rng = 5 ), cost_function = negative_log_likelihood_dangerous, # "Dangerous" because it doesn't check to see that output is normalized, but we know it is because it comes from softmax. optimizer = SimpleGradientDescent(eta = 0.1), ).compile(), # .compile() returns an IPredictor }, offline_predictors={ 'RF': RandomForestClassifier(n_estimators = 40) }, minibatch_size = minibatch_size, test_epochs = sqrtspace(0, n_epochs, n_tests), evaluation_function = percent_argmax_correct # Compares one-hot ) # Results is a LearningCurveData object return learning_curve_data if __name__ == '__main__': set_test_mode(False) records = compare_example_predictors( n_epochs=30, minibatch_size=20, ) plot_learning_curves(records)
layer_sizes=[ dataset.input_size, 500, dataset.n_categories ], hidden_activation='sig', # Sigmoidal hidden units output_activation= 'softmax', # Softmax output unit, since we're doing multinomial classification w_init=0.01, rng=5), cost_function= negative_log_likelihood_dangerous, # "Dangerous" because it doesn't check to see that output is normalized, but we know it is because it comes from softmax. optimizer=SimpleGradientDescent(eta=0.1), ).compile(), # .compile() returns an IPredictor }, offline_predictors={'RF': RandomForestClassifier(n_estimators=40)}, minibatch_size=minibatch_size, test_epochs=sqrtspace(0, n_epochs, n_tests), evaluation_function=percent_argmax_correct # Compares one-hot ) # Results is a LearningCurveData object return learning_curve_data if __name__ == '__main__': set_test_mode(False) records = compare_example_predictors( n_epochs=30, minibatch_size=20, ) plot_learning_curves(records)
def demo_mnist_mlp( minibatch_size = 10, learning_rate = 0.1, optimizer = 'sgd', hidden_sizes = [300], w_init = 0.01, hidden_activation = 'tanh', output_activation = 'softmax', cost = 'nll-d', visualize_params = False, n_test_points = 30, n_epochs = 10, max_training_samples = None, use_bias = True, onehot = False, rng = 1234, plot = False, ): """ Train an MLP on MNIST and print the test scores as training progresses. """ if is_test_mode(): n_test_points = 3 minibatch_size = 5 n_epochs = 0.01 dataset = get_mnist_dataset(n_training_samples=30, n_test_samples=30) else: dataset = get_mnist_dataset(n_training_samples=max_training_samples) if onehot: dataset = dataset.to_onehot() if minibatch_size == 'full': minibatch_size = dataset.training_set.n_samples optimizer = get_named_optimizer(name = optimizer, learning_rate=learning_rate) # Setup the training and test functions predictor = GradientBasedPredictor( function = MultiLayerPerceptron.from_init( layer_sizes=[dataset.input_size]+hidden_sizes+[10], hidden_activation=hidden_activation, output_activation=output_activation, w_init = w_init, use_bias=use_bias, rng = rng, ), cost_function=cost, optimizer=optimizer ).compile() # .compile() turns the GradientBasedPredictor, which works with symbolic variables, into a real one that takes and returns arrays. def vis_callback(xx): p = predictor.symbolic_predictor._function in_layer = { 'Layer[0].w': p.layers[0].linear_transform._w.get_value().T.reshape(-1, 28, 28), 'Layer[0].b': p.layers[0].linear_transform._b.get_value(), } other_layers = [{'Layer[%s].w' % (i+1): l.linear_transform._w.get_value(), 'Layer[%s].b' % (i+1): l.linear_transform._b.get_value()} for i, l in enumerate(p.layers[1:])] dbplot(dict(in_layer.items() + sum([o.items() for o in other_layers], []))) # Train and periodically report the test score. results = assess_online_predictor( dataset=dataset, predictor=predictor, evaluation_function='percent_argmax_correct', test_epochs=sqrtspace(0, n_epochs, n_test_points), minibatch_size=minibatch_size, test_callback=vis_callback if visualize_params else None ) if plot: plot_learning_curves(results)