def test_get_error(): model = helpers.SetOutputModel([1]) assert validation.get_error( model, numpy.array([[1]]), numpy.array([[0]]), error_func=MeanSquaredError()) == 1.0 assert validation.get_error( model, numpy.array([[1]]), numpy.array([[1]]), error_func=MeanSquaredError()) == 0.0 assert validation.get_error( model, numpy.array([[1]]), numpy.array([[0.5]]), error_func=MeanSquaredError()) == 0.25 assert validation.get_error( model, numpy.array([[1], [1]]), numpy.array([[1], [0]]), error_func=MeanSquaredError()) == 0.5 assert validation.get_error( model, numpy.array([[1], [1]]), numpy.array([[0.5], [0.5]]), error_func=MeanSquaredError()) == 0.25
def test_dropout_mlp(): # Run for a couple of iterations # assert that new error is less than original model = mlp.DropoutMLP((2, 8, 2)) dataset = datasets.get_and() error = validation.get_error(model, *dataset) model.train(*dataset, iterations=20) assert validation.get_error(model, *dataset) < error
def test_rbf(): # Run for a couple of iterations # assert that new error is less than original model = rbf.RBF(2, 4, 2, scale_by_similarity=True) dataset = datasets.get_xor() error = validation.get_error(model, *dataset) model.train(*dataset, iterations=10) assert validation.get_error(model, *dataset) < error
def test_mlp_classifier(): # Run for a couple of iterations # assert that new error is less than original model = mlp.MLP( (2, 2, 2), transfers=SoftmaxTransfer(), error_func=CrossEntropyError()) dataset = datasets.get_xor() error = validation.get_error(model, *dataset) model.train(*dataset, iterations=20) assert validation.get_error(model, *dataset) < error
def test_LinearRegressionModel(): # Run for a couple of iterations # assert that new error is less than original model = LinearRegressionModel(2, 2) # NOTE: We use and instead of xor, because xor is non-linear dataset = datasets.get_and() error = validation.get_error(model, *dataset) model.train(*dataset, iterations=10) assert validation.get_error(model, *dataset) < error
def test_neuralfield_convergence(): # Run until convergence # assert that network can converge dataset = datasets.get_xor() model = ill.make_neuralfield(2, grid_spacing=XOR_SPACING) error = validation.get_error(model, *dataset) model.train(*dataset, error_break=0.002) assert validation.get_error( model, *dataset) < 0.02, "Training should reach low error"
def test_ill_mlp_exact_target(): # Run for a couple of iterations # assert that new error is less than original dataset = datasets.get_xor() model = ill.ILL(MLP((2, 2, 2)), grid_spacing=XOR_SPACING, learn_exact=True) error = validation.get_error(model, *dataset) model.train(*dataset, iterations=10) assert validation.get_error(model, * dataset) < error, "Training decreases error"
def test_ill_mlp_convergence_exact_target(): # Run until convergence # assert that network can converge dataset = datasets.get_xor() model = ill.ILL(MLP((2, 2, 2)), grid_spacing=XOR_SPACING, learn_exact=True) error = validation.get_error(model, *dataset) model.train(*dataset, retries=5, error_break=0.002) assert validation.get_error( model, *dataset) < 0.02, "Training should reach low error"
def test_get_error_unusual_targets_shape(): from learning import error model = multioutputs.MultiOutputs([ helpers.SetOutputModel([1.0]), helpers.SetOutputModel([1.0, 1.0, 1.0]) ]) assert validation.get_error(model, [[]], [[[1.0], [1.0, 1.0, 1.0]]], error_func=MeanSquaredError()) == 0.0 assert validation.get_error(model, [[]], [[[1.0], [0.0, 0.0, 0.0]]], error_func=MeanSquaredError()) == 0.5
def test_neuralfield(): # Run for a couple of iterations # assert that new error is less than original dataset = datasets.get_xor() model = ill.make_neuralfield(2, grid_spacing=XOR_SPACING) error = validation.get_error(model, *dataset) model.train(*dataset, iterations=10) assert validation.get_error(model, * dataset) < error, "Training decreases error"
def test_rbf_convergence(): # Run until convergence # assert that network can converge model = rbf.RBF(2, 4, 2, scale_by_similarity=True) dataset = datasets.get_xor() model.train(*dataset, retries=5, error_break=0.002) assert validation.get_error(model, *dataset) <= 0.02
def test_pbnn_convergence(): # Run until convergence # assert that network can converge model = PBNN() dataset = datasets.get_xor() model.train(*dataset) assert validation.get_error(model, *dataset) <= 0.02
def test_mlp_convergence(): # Run until convergence # assert that network can converge model = mlp.MLP((2, 4, 2)) dataset = datasets.get_xor() model.train(*dataset, retries=5, error_break=0.002) assert validation.get_error(model, *dataset) <= 0.02
def test_mlp_classifier_convergence(): # Run until convergence # assert that network can converge model = mlp.MLP( (2, 3, 2), transfers=SoftmaxTransfer(), error_func=CrossEntropyError()) dataset = datasets.get_and() model.train(*dataset, retries=5, error_break=0.002) assert validation.get_error(model, *dataset) <= 0.02
def test_LinearRegressionModel_convergence(): # Run until convergence # assert that model can converge model = LinearRegressionModel(2, 2) # NOTE: We use and instead of xor, because xor is non-linear dataset = datasets.get_and() model.train(*dataset) # NOTE: This linear model cannot achieve 0 MSE assert validation.get_error(model, *dataset) <= 0.1
def test_ill_mlp_dim_reduction_tuple(monkeypatch): REDUCED_DIMENSIONS = 1 dataset = datasets.get_xor() model = ill.ILL(MLP((2, 2, 2)), grid_spacing=XOR_SPACING, dim_reduction=(2, REDUCED_DIMENSIONS)) # Points should have reduced dimensions points = _get_neighborhood_points(model, dataset, monkeypatch) for point in points: assert len(point) == REDUCED_DIMENSIONS # Should be able to train error = validation.get_error(model, *dataset) model.train(*dataset, iterations=10) assert validation.get_error(model, * dataset) < error, "Training decreases error"
def test_dropout_mlp_convergence(): # Run until convergence # assert that network can converge # Since XOR does not really need dropout, we use high probabilities model = mlp.DropoutMLP( (2, 8, 2), input_active_probability=1.0, hidden_active_probability=0.9) dataset = datasets.get_and() # Easier and dataset for lienar output # Error break lower than cutoff, since dropout may have different error # after training model.train(*dataset, retries=5, error_break=0.002, error_improve_iters=50) # Dropout sacrifices training accuracy for better generalization # so we don't worry as much about convergence assert validation.get_error(model, *dataset) <= 0.1
def test_Model_stochastic_train(): """Train with stochastic gradient descent.""" from learning import transfer, error, validation, MLP dataset = datasets.get_iris() model = MLP((len(dataset[0][0]), 2, len(dataset[1][0])), transfers=transfer.SoftmaxTransfer(), error_func=error.CrossEntropyError()) # Model should be able to converge with mini-batches model.stochastic_train( *dataset, error_break=0.02, pattern_selection_func=lambda X, Y: base.select_sample(X, Y, size=30), train_kwargs={ 'iterations': 100, 'error_break': 0.02 }) assert validation.get_error(model, *dataset) <= 0.03
def stochastic_train(self, input_matrix, target_matrix, max_iterations=100, error_break=0.002, pattern_selection_func=select_sample, train_kwargs={'iterations': 100}): """Train model on multiple subsets of the given dataset. Use for stochastic gradient descent. Args: input_matrix: A matrix with samples in rows and attributes in columns. target_matrix: A matrix with samples in rows and target values in columns. max_iterations: Maximum number of times that Model.train is called. error_break: Training will end once error is less than this, on entire dataset. pattern_select_func: Function that takes (input_matrix, target_matrix), and returns a selection of rows. Use partial function to embed arguments. """ for iteration in range(1, max_iterations + 1): train_error, converged = self.train( *pattern_selection_func(input_matrix, target_matrix), **train_kwargs) if train_error is not None: # Break early to prevent overtraining if (train_error <= error_break # Perform a second test on whole dataset # TODO: Use user provided error function and validation.get_error( self, input_matrix, target_matrix) <= error_break): return True # Override iteration from inner loop, with iteration number from outer loop self.iteration = iteration return False
def _train_attempt(self, input_matrix, target_matrix, iterations, error_break, error_stagnant_distance, error_stagnant_threshold, error_improve_iters, pattern_select_func, post_pattern_callback): """Attempt to train this model. Return True if model converged (error <= error_break) """ # Initialize error history with errors that are # unlikely to be close in reality error_history = [1e10] * error_stagnant_distance # Initialize best error for error_decrease_iters best_error = float('inf') iters_since_improvement = 0 for self.iteration in range(1, iterations + 1): selected_patterns = pattern_select_func(input_matrix, target_matrix) # Learn each selected pattern error = self.train_step(*selected_patterns) # Logging and breaking if self.logging: print "Iteration {}, Error: {}".format(self.iteration, error) if error is not None: # Break early to prevent overtraining if (error <= error_break # Perform a second test on whole dataset # incase model is training on mini-batches # TODO: Should use user provided error function? and validation.get_error( self, input_matrix, target_matrix) <= error_break): return True # Skip the rest if we're already out of iterations (optimization) # Useful for situations where we only run 1 iteration if self.iteration == iterations: return False # Break if no progress is made if _all_close(error_history, error, error_stagnant_threshold): # Break if not enough difference between all resent errors # and current error return False error_history.append(error) error_history.pop(0) # Break if best error has not improved within n iterations # Keep track of best error, and iterations since best error has improved if error < best_error: best_error = error iters_since_improvement = 0 else: iters_since_improvement += 1 # If it has been too many iterations since improvement, break if iters_since_improvement >= error_improve_iters: return False return False
def train(self, input_matrix, target_matrix, iterations=1000, retries=0, error_break=0.002, error_stagnant_distance=5, error_stagnant_threshold=0.00001, error_improve_iters=20, pattern_select_func=select_iterative, post_pattern_callback=None): """Train model to converge on a dataset. Note: Override this method for batch learning models. Args: input_matrix: A matrix with samples in rows and attributes in columns. target_matrix: A matrix with samples in rows and target values in columns. iterations: Max iterations to train model. retries: Number of times to reset model and retries if it does not converge. Convergence is defined as reaching error_break. error_break: Training will end once error is less than this. error_stagnant_distance: Number of iterations during which error must change by at least error_stagnant_threshold, or training ends. error_stagnant_threshold: Threshold by which error must change within error_stagnant_distance iterations, or training ends. error_improve_iters: Best error must decrease within this many iterations, or training ends. pattern_select_func: Function that takes (input_matrix, target_matrix), and returns a selection of rows. Use partial function to embed arguments. """ # Make sure matrix parameters are np arrays self._reset_bookkeeping() self._post_pattern_callback = post_pattern_callback # For calling in other method # Initialize variables for retries best_try = (float('inf'), None) # (error, serialized_model) # Learn on each pattern for each iteration for attempt in range(retries + 1): success = self._train_attempt( input_matrix, target_matrix, iterations, error_break, error_stagnant_distance, error_stagnant_threshold, error_improve_iters, pattern_select_func, post_pattern_callback) # Skip all the tracking and whatnot if there are no retries (optimization) if retries == 0: return # End if model converged # No need to use best attempt (since this is the first to reach best error) if success: return # TODO: Should use user provided error function attempt_error = validation.get_error(self, input_matrix, target_matrix) # End when out of retries, use best attempt so far if attempt >= retries: if attempt_error < best_try[0]: # Last attempt was our best return else: # Use best attempt best_model = self.unserialize(best_try[1]) self.__dict__ = best_model.__dict__ return # Keep track of best attempt if attempt_error < best_try[0]: best_try = (attempt_error, self.serialize()) # Reset for next attempt self.reset()