def run_fold(self, k): memory_pool.free_held() fold_range = (k*self.fold_size, min((k+1)*self.fold_size, self.n_data)) test_idx = np.arange(fold_range[0], fold_range[1], dtype=np.int32) train_validate_idx = np.random.permutation( np.r_[np.arange(0, fold_range[0], dtype=np.int32), np.arange(fold_range[1], self.n_data, dtype=np.int32)]) train_idx = train_validate_idx[:self.N_train] validate_idx = train_validate_idx[self.N_train:] self.fold_idx.append({ 'test_idx': test_idx, 'train_idx': train_idx, 'validate_idx': validate_idx }) dp_train = self.make_data_provider(train_idx, self.config.get('batch_size_train')) dp_validate = self.make_data_provider(validate_idx, self.config.get('batch_size_validate')) dp_test = self.make_data_provider(test_idx, self.config.get('batch_size_test')) model = self.make_model() model.calibrate_learning_rate(dp_train) self.models_cv.append(model) progress_monitor = self.make_progress_monitor(k) self.progress_monitors_cv.append(progress_monitor) learning_rate_schedule = self.config['learning_rate_fct'](**self.config['learning_rate_params']) momentum_schedule = self.config['momentum_schedule_fct'](**self.config['momentum_schedule_params']) \ if 'momentum_schedule_fct' in self.config else None optimizer = SGD(model, self.config['parameter_updater'], dp_train, dp_validate, progress_monitor, learning_rate_schedule=learning_rate_schedule, momentum_schedule=momentum_schedule, early_stopping=self.config.get('early_stopping', True)) optimizer.run(self.config['epochs'], validation_interval=self.config.get('validation_interval', 5), yaml_config=self.config['yaml_config']) stats = self.get_stats(dp_train, dp_test, model) self.fold_stats.append(stats) predictions_fold = model.feed_forward(dp_test.data).get() self.predictions = np.r_[self.predictions, predictions_fold] \ if self.predictions is not None else predictions_fold self.make_figures(model, progress_monitor, k) self.train_error['training_error'].append(progress_monitor.train_error) self.train_error['validation_error'].append(progress_monitor.validation_error) del optimizer, dp_train, dp_validate, dp_test
def run_fold(self, k): fold_range = (k * self.fold_size, min((k + 1) * self.fold_size, self.n_data)) test_idx = np.arange(fold_range[0], fold_range[1], dtype=np.int32) train_validate_idx = np.random.permutation( np.r_[np.arange(0, fold_range[0], dtype=np.int32), np.arange(fold_range[1], self.n_data, dtype=np.int32)]) train_idx = train_validate_idx[:self.N_train] validate_idx = train_validate_idx[self.N_train:] self.fold_idx.append({ 'test_idx': test_idx, 'train_idx': train_idx, 'validate_idx': validate_idx }) dp_train = self.make_data_provider(train_idx, self.config.get('batch_size_train')) dp_validate = self.make_data_provider( validate_idx, self.config.get('batch_size_validate')) dp_test = self.make_data_provider(test_idx, self.config.get('batch_size_test')) model = self.make_model() self.models_cv.append(model) progress_monitor = self.make_progress_monitor(k) self.progress_monitors_cv.append(progress_monitor) learning_rate_schedule = self.config['learning_rate_fct']( **self.config['learning_rate_params']) momentum_schedule = self.config['momentum_schedule_fct'](**self.config['momentum_schedule_params']) \ if 'momentum_schedule_fct' in self.config else None optimizer = SGD(model, self.config['parameter_updater'], dp_train, dp_validate, progress_monitor, learning_rate_schedule=learning_rate_schedule, momentum_schedule=momentum_schedule, early_stopping=self.config.get('early_stopping', True)) optimizer.run(self.config['epochs'], yaml_config=self.config['yaml_config']) stats = self.get_stats(dp_train, dp_test, model) self.fold_stats.append(stats) predictions_fold = model.feed_forward(dp_test.data).get() self.predictions = np.r_[self.predictions, predictions_fold] \ if self.predictions is not None else predictions_fold self.make_figures(model, progress_monitor, k) self.train_error['training_error'].append(progress_monitor.train_error) self.train_error['validation_error'].append( progress_monitor.validation_error)
def test_neural_net_regression(self): for _ in range(20): N = 10000 # Number of data points D = 100 # Dimensionality of exogenous data P = 50 # Dimensionality of endogenous data W_true = 10 * np.random.rand(D, P) - 5 b_true = 100 * np.random.rand(P) - 50 X = np.random.randn(N, D) Y = np.dot(X, W_true) + b_true[np.newaxis, :] + np.random.randn(N, P) W_lstsq = np.linalg.lstsq(np.c_[np.ones((N, 1)), X], Y)[0] b_lstsq = W_lstsq[0] W_lstsq = W_lstsq[1:] data_provider = BatchDataProvider(gpuarray.to_gpu(X.astype(np.float32), allocator=memory_pool.allocate), gpuarray.to_gpu(Y.astype(np.float32), allocator=memory_pool.allocate)) model = NeuralNetRegression([], n_in=D, n_out=P) optimizer = SGD(model, SimpleSGDUpdate, data_provider, data_provider, learning_rate_schedule=constant_scheduler(10.), early_stopping=True) optimizer.run(100) self.assertLess(np.abs(W_lstsq - model.top_layer.W.get()).max(), 1e-5)
def main(): import numpy as np import pycuda.autoinit from pycuda import gpuarray from skdata import toy from hebel import memory_pool from hebel.data_providers import BatchDataProvider from hebel.models import NeuralNetRegression from hebel.optimizers import SGD from hebel.parameter_updaters import SimpleSGDUpdate from hebel.monitors import SimpleProgressMonitor from hebel.schedulers import exponential_scheduler # Get data data_cpu, targets_cpu = toy.Boston().regression_task() data = gpuarray.to_gpu(data_cpu.astype(np.float32), allocator=memory_pool.allocate) targets = gpuarray.to_gpu(targets_cpu.astype(np.float32), allocator=memory_pool.allocate) data_provider = BatchDataProvider(data, targets) # Create model object model = NeuralNetRegression(n_in=data_cpu.shape[1], n_out=targets_cpu.shape[1], layers=[100], activation_function='relu') # Create optimizer object optimizer = SGD(model, SimpleSGDUpdate, data_provider, data_provider, learning_rate_schedule=exponential_scheduler(.1, .9999), early_stopping=True) optimizer.run(3000)
def test_neural_net_regression(self): for _ in range(20): N = 10000 # Number of data points D = 100 # Dimensionality of exogenous data P = 50 # Dimensionality of endogenous data W_true = 10 * np.random.rand(D, P) - 5 b_true = 100 * np.random.rand(P) - 50 X = np.random.randn(N, D) Y = np.dot(X, W_true) + b_true[np.newaxis, :] + np.random.randn( N, P) W_lstsq = np.linalg.lstsq(np.c_[np.ones((N, 1)), X], Y)[0] b_lstsq = W_lstsq[0] W_lstsq = W_lstsq[1:] data_provider = BatchDataProvider( gpuarray.to_gpu(X.astype(np.float32)), gpuarray.to_gpu(Y.astype(np.float32))) model = NeuralNetRegression([], n_in=D, n_out=P) optimizer = SGD(model, SimpleSGDUpdate, data_provider, data_provider, learning_rate_schedule=constant_scheduler(10.), early_stopping=True) optimizer.run(100) self.assertLess( np.abs(W_lstsq - model.top_layer.W.get()).max(), 1e-5)
def test_relu(self): model = NeuralNet(n_in=self.D, n_out=self.n_out, layers=[1000], activation_function='relu', dropout=True) optimizer = SGD(model, SimpleSGDUpdate, self.train_data, self.test_data, learning_rate_schedule=exponential_scheduler(1., .99), progress_monitor=SimpleProgressMonitor()) optimizer.run(20) self.assertLess(optimizer.progress_monitor.train_error[-1][1], optimizer.progress_monitor.train_error[0][1]) del model, optimizer
validation_data = MNISTDataProvider('val') test_data = MNISTDataProvider('test') D = train_data.D # Dimensionality of inputs K = 10 # Number of classes # Create model object model = NeuralNet(n_in=train_data.D, n_out=K, layers=[1000, 500, 500], activation_function='relu', dropout=True) # Create optimizer object progress_monitor = ProgressMonitor( experiment_name='mnist', save_model_path='examples/mnist', save_interval=5, output_to_log=True) optimizer = SGD(model, MomentumUpdate, train_data, validation_data, learning_rate_schedule=exponential_scheduler(1., .995), momentum_schedule=linear_scheduler_up(.5, .9, 10)) # Run model optimizer.run(100) # Evaulate error on test set test_error = 0 test_error = model.test_error(test_data) print "Error on test set: %.3f" % test_error
K = 10 # Number of classes # Create model object model = NeuralNet(n_in=train_data.D, n_out=K, layers=[2000, 2000, 2000, 500], activation_function='relu', dropout=True, input_dropout=0.2) # Create optimizer object progress_monitor = ProgressMonitor(experiment_name='mnist', save_model_path='examples/mnist', save_interval=5, output_to_log=True) optimizer = SGD(model, MomentumUpdate, train_data, validation_data, progress_monitor, learning_rate_schedule=exponential_scheduler(5., .995), momentum_schedule=linear_scheduler_up(.1, .9, 100)) # Run model optimizer.run(50) # Evaulate error on test set test_error = model.test_error(test_data) print "Error on test set: %.3f" % test_error
from hebel.monitors import SimpleProgressMonitor from hebel.schedulers import exponential_scheduler, linear_scheduler_up # Initialize data providers train_data = MNISTDataProvider('train', batch_size=100) validation_data = MNISTDataProvider('val') test_data = MNISTDataProvider('test') D = train_data.D # Dimensionality of inputs K = 10 # Number of classes # Create model object model = NeuralNet(n_in=train_data.D, n_out=K, layers=[2000, 2000, 2000, 500], activation_function='relu', dropout=True, input_dropout=0.2) # Create optimizer object progress_monitor = SimpleProgressMonitor() optimizer = SGD(model, MomentumUpdate, train_data, validation_data, learning_rate_schedule=exponential_scheduler(5., .995), momentum_schedule=linear_scheduler_up(.1, .9, 100)) # Run model optimizer.run(5) # Evaulate error on test set test_error = model.test_error(test_data) print "Error on test set: %.3f" % test_error
D = train_data.D # Dimensionality of inputs K = 10 # Number of classes # Create model object model = NeuralNet(n_in=train_data.D, n_out=K, layers=[1000, 500, 500], activation_function='relu', dropout=True) # Create optimizer object progress_monitor = ProgressMonitor(experiment_name='mnist', save_model_path='examples/mnist', save_interval=5, output_to_log=True) optimizer = SGD(model, MomentumUpdate, train_data, validation_data, learning_rate_schedule=exponential_scheduler(1., .995), momentum_schedule=linear_scheduler_up(.5, .9, 10)) # Run model optimizer.run(100) # Evaulate error on test set test_error = 0 test_error = model.test_error(test_data) print "Error on test set: %.3f" % test_error
model = NeuralNet( n_in=train_data.D, n_out=K, layers=[2000, 2000, 2000, 500], activation_function="relu", dropout=True, input_dropout=0.2, ) # Create optimizer object progress_monitor = ProgressMonitor( experiment_name="mnist", save_model_path="examples/mnist", save_interval=5, output_to_log=True ) optimizer = SGD( model, MomentumUpdate, train_data, validation_data, progress_monitor, learning_rate_schedule=exponential_scheduler(5.0, 0.995), momentum_schedule=linear_scheduler_up(0.1, 0.9, 100), ) # Run model optimizer.run(50) # Evaulate error on test set test_error = model.test_error(test_data) print "Error on test set: %.3f" % test_error