def test_experiment_interface():
register_experiment(name='my_test_experiment',
function=_run_experiment,
description="See if this thing works",
conclusion="It does.")
exp_rec = run_experiment('my_test_experiment', keep_record=True)
print get_experiment_info('my_test_experiment')
assert exp_rec.get_log() == 'aaa\nbbb\n'
same_exp_rec = load_experiment(
get_latest_experiment_identifier(name='my_test_experiment'))
assert same_exp_rec.get_log() == 'aaa\nbbb\n'
same_exp_rec.delete()
# Train and periodically report the test score.
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size
)
plot_learning_curves(results)
register_experiment(
name = 'mnist-multinomial-regression',
function = lambda: demo_mnist_online_regression(regressor_type='multinomial'),
description = 'Simple multinomial regression (a.k.a. One-layer neural network) on MNIST',
conclusion = 'Gets to about 92.5'
)
register_experiment(
name = 'mnist-multinomial-regression-nobias',
function = lambda: demo_mnist_online_regression(regressor_type='multinomial', include_biases=False),
description = 'Simple multinomial regression (a.k.a. One-layer neural network) on MNIST',
conclusion = "Also gets to about 92.5. So at least for MNIST you don't really need a bias term."
)
register_experiment(
name = 'mnist-linear-regression',
function = lambda: demo_mnist_online_regression(regressor_type='linear', learning_rate=0.01),
description = 'Simple multinomial regression (a.k.a. One-layer neural network) on MNIST',
conclusion = 'Requires a lower learning rate for stability, and then only makes it to around 86%'
results = assess_online_predictor(
dataset=dataset,
predictor=predictor,
evaluation_function='percent_argmax_correct',
test_epochs=sqrtspace(0, n_epochs, n_test_points),
minibatch_size=minibatch_size,
test_callback=vis_callback if visualize_params else None
)
if plot:
plot_learning_curves(results)
register_experiment(
name = 'MNIST-tanh-MLP[300,10]',
function = lambda: demo_mnist_mlp(hidden_sizes=[300], hidden_activation='tanh', learning_rate=0.03),
description='Baseline. Gets 97.45% test score within 10 epochs.'
)
register_experiment(
name = 'MNIST1000_onelayer_minibatch-20',
function = lambda: demo_mnist_mlp(learning_rate= 0.03, hidden_sizes = [], minibatch_size=20, max_training_samples=1000, n_epochs=1000),
description='How does a single-layer (logistic-regression) net do on MNIST-1000?',
conclusion='Gets to about 87.5% before overfitting its way down to 86.7. 100% on training.'
)
register_experiment(
name = 'MNIST1000_MLP[300,10]_minibatch-20',
function = lambda: demo_mnist_mlp(learning_rate= 0.03, minibatch_size=20, max_training_samples=1000, n_epochs=1000)
)
lin_dtp,
initial_mag=2):
return DifferenceTargetMLP(layers=[
PerceptronLayer.from_initializer(n_in,
n_out,
initial_mag=initial_mag,
lin_dtp=lin_dtp)
for n_in, n_out in zip([input_size] + hidden_sizes, hidden_sizes +
[output_size])
],
output_cost_function=None).compile()
register_experiment(
name='single-level-perceptron-DTP',
function=lambda: demo_perceptron_dtp(hidden_sizes=[], n_epochs=60),
description=
"Just to verify we're not crazy. This should be equiv to a single layer perceptron (without biases though)",
conclusion="Hovers kind of noisily just below 85%, as expected.")
register_experiment(
name='multi-level-perceptron-DTP',
function=lambda: demo_perceptron_dtp(
hidden_sizes=[400], n_epochs=60, lin_dtp=False),
description=
"Try DTP with one hidden layer using sign-activation units and the perceptron learning rule",
conclusion="Doesn't work at all at all.")
register_experiment(
name='multi-level-perceptron-LinDTP',
function=lambda: demo_perceptron_dtp(
hidden_sizes=[400], n_epochs=60, lin_dtp=True),
manifold_means, _ = decoder_mean_fcn()
dbplot(
manifold_means.reshape(manifold_grid_size, manifold_grid_size,
28, 28),
'First 2-dimensions of manifold.')
if i % calculation_interval == 0:
training_lower_bound = lower_bound_fcn(training_data)
test_lower_bound = lower_bound_fcn(test_data)
print 'Epoch: %s, Training Lower Bound: %s, Test Lower bound: %s' % \
(i*minibatch_size/float(len(training_data)), training_lower_bound, test_lower_bound)
register_experiment(
name='mnist-vae-20d-binary_in',
function=lambda: demo_simple_vae_on_mnist(
z_dim=20, hidden_sizes=[200], binary_x=True),
description="Try encoding MNIST with a variational autoencoder.",
conclusion=
"Looks good. Within about 20 epochs we're getting reasonablish samples, lower bound of -107."
)
register_experiment(
name='mnist-vae-20d-continuous_in',
function=lambda: demo_simple_vae_on_mnist(
z_dim=20, hidden_sizes=[200], binary_x=False, gaussian_min_var=0.01),
description=
"Try encoding MNIST with a variational autoencoder, this time treating the input as a continuous variable",
conclusion=
"Need to set minimum variance. Recognieseable digits come out, but then instabilities."
)
register_experiment(
def run_and_plot(training_scheme):
learning_curves = training_scheme()
plot_learning_curves(learning_curves)
register_experiment(
'standard_dtp',
function=partial(demo_dtp_varieties, predictors=['MLP', 'DTP']),
description=
"""Train Difference Target Propagation on MNIST using standard settings, compare to backprop. This will "
be used as a baseline agains other experiments.""",
versions={
'10_epoch': dict(n_epochs=10),
'20_epoch': dict(n_epochs=20)
},
current_version='10_epoch',
conclusion="""
After 10 epochs:
MLP: 97.32
DTP: 96.61
""")
register_experiment(
'hidden_types',
function=partial(demo_dtp_varieties,
predictors=['MLP', 'DTP'],
n_epochs=20),
description=
print 'aaa'
plt.figure('sensible defaults')
dat = np.random.randn(4, 5)
plt.subplot(211)
plt.imshow(dat)
plt.subplot(212)
plt.imshow(dat, interpolation='nearest', cmap='gray')
plt.show()
print 'bbb'
plt.figure()
plt.plot(np.random.randn(10))
plt.show()
register_experiment(name='test_experiment',
description="Testing the experiment framework",
function=_run_experiment,
conclusion="Nothing to mention")
def test_experiment_with():
delete_experiment_with_id('test_exp')
with record_experiment(identifier='test_exp',
print_to_console=True) as exp_rec:
_run_experiment()
assert exp_rec.get_log() == 'aaa\nbbb\n'
figs = exp_rec.show_figures()
assert len(exp_rec.get_figure_locs()) == 2