def run_fold(self, k):
memory_pool.free_held()
fold_range = (k*self.fold_size, min((k+1)*self.fold_size, self.n_data))
test_idx = np.arange(fold_range[0], fold_range[1], dtype=np.int32)
train_validate_idx = np.random.permutation(
np.r_[np.arange(0, fold_range[0], dtype=np.int32),
np.arange(fold_range[1], self.n_data, dtype=np.int32)])
train_idx = train_validate_idx[:self.N_train]
validate_idx = train_validate_idx[self.N_train:]
self.fold_idx.append({
'test_idx': test_idx,
'train_idx': train_idx,
'validate_idx': validate_idx
})
dp_train = self.make_data_provider(train_idx,
self.config.get('batch_size_train'))
dp_validate = self.make_data_provider(validate_idx,
self.config.get('batch_size_validate'))
dp_test = self.make_data_provider(test_idx,
self.config.get('batch_size_test'))
model = self.make_model()
model.calibrate_learning_rate(dp_train)
self.models_cv.append(model)
progress_monitor = self.make_progress_monitor(k)
self.progress_monitors_cv.append(progress_monitor)
learning_rate_schedule = self.config['learning_rate_fct'](**self.config['learning_rate_params'])
momentum_schedule = self.config['momentum_schedule_fct'](**self.config['momentum_schedule_params']) \
if 'momentum_schedule_fct' in self.config else None
optimizer = SGD(model, self.config['parameter_updater'], dp_train, dp_validate,
progress_monitor,
learning_rate_schedule=learning_rate_schedule,
momentum_schedule=momentum_schedule,
early_stopping=self.config.get('early_stopping', True))
optimizer.run(self.config['epochs'],
validation_interval=self.config.get('validation_interval', 5),
yaml_config=self.config['yaml_config'])
stats = self.get_stats(dp_train, dp_test, model)
self.fold_stats.append(stats)
predictions_fold = model.feed_forward(dp_test.data).get()
self.predictions = np.r_[self.predictions, predictions_fold] \
if self.predictions is not None else predictions_fold
self.make_figures(model, progress_monitor, k)
self.train_error['training_error'].append(progress_monitor.train_error)
self.train_error['validation_error'].append(progress_monitor.validation_error)
del optimizer, dp_train, dp_validate, dp_test
def run_fold(self, k):
fold_range = (k * self.fold_size,
min((k + 1) * self.fold_size, self.n_data))
test_idx = np.arange(fold_range[0], fold_range[1], dtype=np.int32)
train_validate_idx = np.random.permutation(
np.r_[np.arange(0, fold_range[0], dtype=np.int32),
np.arange(fold_range[1], self.n_data, dtype=np.int32)])
train_idx = train_validate_idx[:self.N_train]
validate_idx = train_validate_idx[self.N_train:]
self.fold_idx.append({
'test_idx': test_idx,
'train_idx': train_idx,
'validate_idx': validate_idx
})
dp_train = self.make_data_provider(train_idx,
self.config.get('batch_size_train'))
dp_validate = self.make_data_provider(
validate_idx, self.config.get('batch_size_validate'))
dp_test = self.make_data_provider(test_idx,
self.config.get('batch_size_test'))
model = self.make_model()
self.models_cv.append(model)
progress_monitor = self.make_progress_monitor(k)
self.progress_monitors_cv.append(progress_monitor)
learning_rate_schedule = self.config['learning_rate_fct'](
**self.config['learning_rate_params'])
momentum_schedule = self.config['momentum_schedule_fct'](**self.config['momentum_schedule_params']) \
if 'momentum_schedule_fct' in self.config else None
optimizer = SGD(model,
self.config['parameter_updater'],
dp_train,
dp_validate,
progress_monitor,
learning_rate_schedule=learning_rate_schedule,
momentum_schedule=momentum_schedule,
early_stopping=self.config.get('early_stopping', True))
optimizer.run(self.config['epochs'],
yaml_config=self.config['yaml_config'])
stats = self.get_stats(dp_train, dp_test, model)
self.fold_stats.append(stats)
predictions_fold = model.feed_forward(dp_test.data).get()
self.predictions = np.r_[self.predictions, predictions_fold] \
if self.predictions is not None else predictions_fold
self.make_figures(model, progress_monitor, k)
self.train_error['training_error'].append(progress_monitor.train_error)
self.train_error['validation_error'].append(
progress_monitor.validation_error)
def test_neural_net_regression(self):
for _ in range(20):
N = 10000 # Number of data points
D = 100 # Dimensionality of exogenous data
P = 50 # Dimensionality of endogenous data
W_true = 10 * np.random.rand(D, P) - 5
b_true = 100 * np.random.rand(P) - 50
X = np.random.randn(N, D)
Y = np.dot(X, W_true) + b_true[np.newaxis, :] + np.random.randn(N, P)
W_lstsq = np.linalg.lstsq(np.c_[np.ones((N, 1)), X], Y)[0]
b_lstsq = W_lstsq[0]
W_lstsq = W_lstsq[1:]
data_provider = BatchDataProvider(gpuarray.to_gpu(X.astype(np.float32),
allocator=memory_pool.allocate),
gpuarray.to_gpu(Y.astype(np.float32),
allocator=memory_pool.allocate))
model = NeuralNetRegression([], n_in=D, n_out=P)
optimizer = SGD(model, SimpleSGDUpdate,
data_provider, data_provider,
learning_rate_schedule=constant_scheduler(10.),
early_stopping=True)
optimizer.run(100)
self.assertLess(np.abs(W_lstsq - model.top_layer.W.get()).max(),
1e-5)
def main():
import numpy as np
import pycuda.autoinit
from pycuda import gpuarray
from skdata import toy
from hebel import memory_pool
from hebel.data_providers import BatchDataProvider
from hebel.models import NeuralNetRegression
from hebel.optimizers import SGD
from hebel.parameter_updaters import SimpleSGDUpdate
from hebel.monitors import SimpleProgressMonitor
from hebel.schedulers import exponential_scheduler
# Get data
data_cpu, targets_cpu = toy.Boston().regression_task()
data = gpuarray.to_gpu(data_cpu.astype(np.float32), allocator=memory_pool.allocate)
targets = gpuarray.to_gpu(targets_cpu.astype(np.float32), allocator=memory_pool.allocate)
data_provider = BatchDataProvider(data, targets)
# Create model object
model = NeuralNetRegression(n_in=data_cpu.shape[1], n_out=targets_cpu.shape[1],
layers=[100], activation_function='relu')
# Create optimizer object
optimizer = SGD(model, SimpleSGDUpdate, data_provider, data_provider,
learning_rate_schedule=exponential_scheduler(.1, .9999),
early_stopping=True)
optimizer.run(3000)
def test_neural_net_regression(self):
for _ in range(20):
N = 10000 # Number of data points
D = 100 # Dimensionality of exogenous data
P = 50 # Dimensionality of endogenous data
W_true = 10 * np.random.rand(D, P) - 5
b_true = 100 * np.random.rand(P) - 50
X = np.random.randn(N, D)
Y = np.dot(X, W_true) + b_true[np.newaxis, :] + np.random.randn(
N, P)
W_lstsq = np.linalg.lstsq(np.c_[np.ones((N, 1)), X], Y)[0]
b_lstsq = W_lstsq[0]
W_lstsq = W_lstsq[1:]
data_provider = BatchDataProvider(
gpuarray.to_gpu(X.astype(np.float32)),
gpuarray.to_gpu(Y.astype(np.float32)))
model = NeuralNetRegression([], n_in=D, n_out=P)
optimizer = SGD(model,
SimpleSGDUpdate,
data_provider,
data_provider,
learning_rate_schedule=constant_scheduler(10.),
early_stopping=True)
optimizer.run(100)
self.assertLess(
np.abs(W_lstsq - model.top_layer.W.get()).max(), 1e-5)
def test_relu(self):
model = NeuralNet(n_in=self.D, n_out=self.n_out,
layers=[1000], activation_function='relu',
dropout=True)
optimizer = SGD(model, SimpleSGDUpdate, self.train_data,
self.test_data,
learning_rate_schedule=exponential_scheduler(1., .99),
progress_monitor=SimpleProgressMonitor())
optimizer.run(20)
self.assertLess(optimizer.progress_monitor.train_error[-1][1],
optimizer.progress_monitor.train_error[0][1])
del model, optimizer
def test_relu(self):
model = NeuralNet(n_in=self.D, n_out=self.n_out,
layers=[1000], activation_function='relu',
dropout=True)
optimizer = SGD(model, SimpleSGDUpdate, self.train_data,
self.test_data,
learning_rate_schedule=exponential_scheduler(1., .99),
progress_monitor=SimpleProgressMonitor())
optimizer.run(20)
self.assertLess(optimizer.progress_monitor.train_error[-1][1],
optimizer.progress_monitor.train_error[0][1])
del model, optimizer
validation_data = MNISTDataProvider('val')
test_data = MNISTDataProvider('test')
D = train_data.D # Dimensionality of inputs
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D, n_out=K,
layers=[1000, 500, 500],
activation_function='relu',
dropout=True)
# Create optimizer object
progress_monitor = ProgressMonitor(
experiment_name='mnist',
save_model_path='examples/mnist',
save_interval=5,
output_to_log=True)
optimizer = SGD(model, MomentumUpdate, train_data, validation_data,
learning_rate_schedule=exponential_scheduler(1., .995),
momentum_schedule=linear_scheduler_up(.5, .9, 10))
# Run model
optimizer.run(100)
# Evaulate error on test set
test_error = 0
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D,
n_out=K,
layers=[2000, 2000, 2000, 500],
activation_function='relu',
dropout=True,
input_dropout=0.2)
# Create optimizer object
progress_monitor = ProgressMonitor(experiment_name='mnist',
save_model_path='examples/mnist',
save_interval=5,
output_to_log=True)
optimizer = SGD(model,
MomentumUpdate,
train_data,
validation_data,
progress_monitor,
learning_rate_schedule=exponential_scheduler(5., .995),
momentum_schedule=linear_scheduler_up(.1, .9, 100))
# Run model
optimizer.run(50)
# Evaulate error on test set
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
from hebel.monitors import SimpleProgressMonitor
from hebel.schedulers import exponential_scheduler, linear_scheduler_up
# Initialize data providers
train_data = MNISTDataProvider('train', batch_size=100)
validation_data = MNISTDataProvider('val')
test_data = MNISTDataProvider('test')
D = train_data.D # Dimensionality of inputs
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D, n_out=K,
layers=[2000, 2000, 2000, 500],
activation_function='relu',
dropout=True, input_dropout=0.2)
# Create optimizer object
progress_monitor = SimpleProgressMonitor()
optimizer = SGD(model, MomentumUpdate, train_data, validation_data,
learning_rate_schedule=exponential_scheduler(5., .995),
momentum_schedule=linear_scheduler_up(.1, .9, 100))
# Run model
optimizer.run(5)
# Evaulate error on test set
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
D = train_data.D # Dimensionality of inputs
K = 10 # Number of classes
# Create model object
model = NeuralNet(n_in=train_data.D,
n_out=K,
layers=[1000, 500, 500],
activation_function='relu',
dropout=True)
# Create optimizer object
progress_monitor = ProgressMonitor(experiment_name='mnist',
save_model_path='examples/mnist',
save_interval=5,
output_to_log=True)
optimizer = SGD(model,
MomentumUpdate,
train_data,
validation_data,
learning_rate_schedule=exponential_scheduler(1., .995),
momentum_schedule=linear_scheduler_up(.5, .9, 10))
# Run model
optimizer.run(100)
# Evaulate error on test set
test_error = 0
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
model = NeuralNet(
n_in=train_data.D,
n_out=K,
layers=[2000, 2000, 2000, 500],
activation_function="relu",
dropout=True,
input_dropout=0.2,
)
# Create optimizer object
progress_monitor = ProgressMonitor(
experiment_name="mnist", save_model_path="examples/mnist", save_interval=5, output_to_log=True
)
optimizer = SGD(
model,
MomentumUpdate,
train_data,
validation_data,
progress_monitor,
learning_rate_schedule=exponential_scheduler(5.0, 0.995),
momentum_schedule=linear_scheduler_up(0.1, 0.9, 100),
)
# Run model
optimizer.run(50)
# Evaulate error on test set
test_error = model.test_error(test_data)
print "Error on test set: %.3f" % test_error
