def main():
import numpy as np
import pycuda.autoinit
from pycuda import gpuarray
from skdata import toy
from hebel import memory_pool
from hebel.data_providers import BatchDataProvider
from hebel.models import NeuralNetRegression
from hebel.optimizers import SGD
from hebel.parameter_updaters import SimpleSGDUpdate
from hebel.monitors import SimpleProgressMonitor
from hebel.schedulers import exponential_scheduler
# Get data
data_cpu, targets_cpu = toy.Boston().regression_task()
data = gpuarray.to_gpu(data_cpu.astype(np.float32), allocator=memory_pool.allocate)
targets = gpuarray.to_gpu(targets_cpu.astype(np.float32), allocator=memory_pool.allocate)
data_provider = BatchDataProvider(data, targets)
# Create model object
model = NeuralNetRegression(n_in=data_cpu.shape[1], n_out=targets_cpu.shape[1],
layers=[100], activation_function='relu')
# Create optimizer object
optimizer = SGD(model, SimpleSGDUpdate, data_provider, data_provider,
learning_rate_schedule=exponential_scheduler(.1, .9999),
early_stopping=True)
optimizer.run(3000)
def test_relu(self):
model = NeuralNet(n_in=self.D, n_out=self.n_out,
layers=[1000], activation_function='relu',
dropout=True)
optimizer = SGD(model, SimpleSGDUpdate, self.train_data,
self.test_data,
learning_rate_schedule=exponential_scheduler(1., .99),
progress_monitor=SimpleProgressMonitor())
optimizer.run(20)
self.assertLess(optimizer.progress_monitor.train_error[-1][1],
optimizer.progress_monitor.train_error[0][1])
del model, optimizer
def test_relu(self):
model = NeuralNet(n_in=self.D, n_out=self.n_out,
layers=[1000], activation_function='relu',
dropout=True)
optimizer = SGD(model, SimpleSGDUpdate, self.train_data,
self.test_data,
learning_rate_schedule=exponential_scheduler(1., .99),
progress_monitor=SimpleProgressMonitor())
optimizer.run(20)
self.assertLess(optimizer.progress_monitor.train_error[-1][1],
optimizer.progress_monitor.train_error[0][1])
del model, optimizer
validation_data = MNISTDataProvider('val')
test_data = MNISTDataProvider('test')
D = train_data.D # Dimensionality of inputs
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D, n_out=K,
layers=[1000, 500, 500],
activation_function='relu',
dropout=True)
# Create optimizer object
progress_monitor = ProgressMonitor(
experiment_name='mnist',
save_model_path='examples/mnist',
save_interval=5,
output_to_log=True)
optimizer = SGD(model, MomentumUpdate, train_data, validation_data,
learning_rate_schedule=exponential_scheduler(1., .995),
momentum_schedule=linear_scheduler_up(.5, .9, 10))
# Run model
optimizer.run(100)
# Evaulate error on test set
test_error = 0
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D,
n_out=K,
layers=[2000, 2000, 2000, 500],
activation_function='relu',
dropout=True,
input_dropout=0.2)
# Create optimizer object
progress_monitor = ProgressMonitor(experiment_name='mnist',
save_model_path='examples/mnist',
save_interval=5,
output_to_log=True)
optimizer = SGD(model,
MomentumUpdate,
train_data,
validation_data,
progress_monitor,
learning_rate_schedule=exponential_scheduler(5., .995),
momentum_schedule=linear_scheduler_up(.1, .9, 100))
# Run model
optimizer.run(50)
# Evaulate error on test set
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
model = NeuralNet(
n_in=train_data.D,
n_out=K,
layers=[2000, 2000, 2000, 500],
activation_function="relu",
dropout=True,
input_dropout=0.2,
)
# Create optimizer object
progress_monitor = ProgressMonitor(
experiment_name="mnist", save_model_path="examples/mnist", save_interval=5, output_to_log=True
)
optimizer = SGD(
model,
MomentumUpdate,
train_data,
validation_data,
progress_monitor,
learning_rate_schedule=exponential_scheduler(5.0, 0.995),
momentum_schedule=linear_scheduler_up(0.1, 0.9, 100),
)
# Run model
optimizer.run(50)
# Evaulate error on test set
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
