def test_sinusoidal_valid_xlim():
""" Test initialising a Sinusoidal DataSet with valid single- and
multi-dimensional x-limits """
# Test single-dimensional x-limits
d = data.Sinusoidal(input_dim=4, x_hi=7.5, n_train=10)
# Test multi-dimensional x-limits
x_hi = np.array([3, 4, 5, 6]).reshape(-1, 1)
d = data.Sinusoidal(input_dim=4, x_hi=x_hi, n_train=10)
def test_sinusoidal_invalid_xlim():
""" Test that a ValueError is raised when initialising a Sinusoidal DataSet
with x-limits that don't broadcast to the size of x_train and test.x.
As stated in the docstring for the data.Sinusoidal initialiser, x_hi
"should be a float, or a numpy array with shape [input_dim, 1]". """
with pytest.raises(ValueError):
d = data.Sinusoidal(input_dim=3, x_lo=[1, 2])
with pytest.raises(ValueError):
d = data.Sinusoidal(input_dim=4, x_hi=[3, 4, 5, 6], n_train=10)
with pytest.raises(ValueError):
x_hi = np.array([3, 4, 5, 6]).reshape(1, -1)
d = data.Sinusoidal(input_dim=4, x_hi=x_hi, n_train=10)
def test_plot_training_curves():
np.random.seed(79)
n_models = np.random.randint(2, 5)
results_list = []
for j in range(n_models):
n_iters = np.random.randint(10, 20)
output_dim = np.random.randint(2, 5)
n = get_random_network(input_dim=2, output_dim=output_dim)
d = data.Sinusoidal(input_dim=2, output_dim=output_dim, n_train=150)
w = n.get_parameter_vector()
result = optimisers.Result(name="Network {}".format(j))
result.begin()
# Call the result.update method a few times
for i in range(n_iters):
n.set_parameter_vector(w + i)
result.update(model=n, dataset=d, iteration=i)
results_list.append(result)
plotting.plot_training_curves(
results_list,
"Test plot_training_curves",
output_dir,
e_lims=None,
n_iqr=1,
)
def test_plot_1D_regression(output_dim):
""" Test plotting function for regression data with 1-dimensional inputs,
and a variable number of outputs """
set_random_seed_from_args("test_plot_1D_regression", output_dim)
# Initialise data and model
sin_data = data.Sinusoidal(
input_dim=1,
output_dim=output_dim,
n_train=100,
n_test=50,
x_lo=0,
x_hi=1,
)
model = get_random_network(
input_dim=1,
output_dim=output_dim,
low=2,
high=3,
initialiser=models.initialisers.ConstantPreActivationStatistics(
x_train=sin_data.train.x,
y_train=sin_data.train.y,
),
)
# Call plotting function under test
plotting.plot_1D_regression(
plot_name="test_plot_1D_regression, output_dim=%i" % output_dim,
dir_name=os.path.join(output_dir, "test_plot_1D_regression"),
dataset=sin_data,
model=model,
output_dim=output_dim,
)
def test_plot_2D_regression(output_dim):
""" Test plotting function for regression data with 2 input dimensions and
a variable number of output dimensions """
set_random_seed_from_args("test_plot_2D_regression", output_dim)
input_dim = 2
x_lo = -2
x_hi = 2
# Generate dataset and network
sin_data = data.Sinusoidal(input_dim=input_dim,
output_dim=output_dim,
n_train=2000,
x_lo=x_lo,
x_hi=x_hi)
model = get_random_network(
input_dim=input_dim,
output_dim=output_dim,
low=2,
high=3,
initialiser=models.initialisers.ConstantPreActivationStatistics(
x_train=sin_data.train.x,
y_train=sin_data.train.y,
))
# Call plotting function under test
plotting.plot_2D_regression(
plot_name="test_plot_2D_regression, output_dim = %i" % output_dim,
dir_name=output_dir,
output_dim=output_dim,
dataset=sin_data,
model=model,
)
def test_gradient_descent(seed):
"""
Test gradient descent (no line-search is used, so there is no guarantee that
each iteration reduces the error function).
"""
# Set the random seed
np.random.seed(seed)
# Generate random number of iterations, network, data, and results file
n_iters = np.random.randint(10, 20)
n = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
results_filename = "Test gradient descent without line-search.txt"
results_path = os.path.join(output_dir, results_filename)
results_file = open(results_path, "w")
result = optimisers.Result(name="SGD without line search",
verbose=True,
file=results_file)
# Call gradient descent function
result_ls = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters),
evaluator=optimisers.Evaluator(i_interval=1),
result=result)
results_file.close()
def test_dbs_column():
""" Test using a DBS column with a Result object """
# Set random seed and initialise network and dataset
set_random_seed_from_args("test_dbs_column")
n_train = np.random.randint(10, 20)
n_its = np.random.randint(10, 20)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
# Initialise output file and Result object
test_name = "Test DBS column"
output_filename = "test_dbs_columns.txt"
with open(os.path.join(output_dir, output_filename), "w") as f:
result = optimisers.Result(name=test_name,
file=f,
add_default_columns=True)
result.add_column(optimisers.results.columns.DbsMetric())
# Initialise gradient vector before DBS is calculated
model.get_gradient_vector(sin_data.train.x, sin_data.train.y)
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
)
def test_step_size_column():
""" Test using step size column with a Result object """
set_random_seed_from_args("test_step_size_column")
n_train = np.random.randint(10, 20)
n_its = np.random.randint(10, 20)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
test_name = "Test line search column"
output_filename = "test_step_size_column.txt"
with open(os.path.join(output_dir, output_filename), "w") as f:
ls = optimisers.LineSearch()
result = optimisers.Result(
name=test_name,
file=f,
add_default_columns=True,
)
result.add_column(optimisers.results.columns.StepSize(ls))
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
line_search=ls,
)
def test_plot_optimal_batch_sizes():
""" Test function which plots the optimal batch size, rate of reduction of
the mean test set error, and train and test error, against the current
iteration throughout the course of model-optimisation """
# Set random seed and initialise network and dataset
np.random.seed(102)
n_train = 10
n_its = 2
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
# Initialise Result, LineSearch and OptimalBatchSize column objects
result = optimisers.Result(verbose=False)
line_search = optimisers.LineSearch()
gd_optimiser = optimisers.GradientDescent(line_search)
columns = optimisers.results.columns
optimal_batch_size_col = columns.OptimalBatchSize(gd_optimiser,
sin_data.train.n,
n_repeats=3,
n_batch_sizes=3,
min_batch_size=2)
result.add_column(optimal_batch_size_col)
# Call optimisation function
gd_optimiser.optimise(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
)
# Call plotting function
plotting.plot_optimal_batch_sizes("Test plot_optimal_batch_sizes",
output_dir, result,
optimal_batch_size_col)
def test_pbgn(seed, reuse_block_inds):
"""
Test the Generalised Newton's method for optimisation, using parallel
block-diagonal approximations.
TODO: combine this and the gradient descent test (and any optimisers
implemented in future, EG adam, PSO) in to a single parametrised test
"""
# Set the random seed
np.random.seed(seed)
# Generate random number of iterations, network, data, and results file
n_iters = np.random.randint(10, 20)
n = NeuralNetwork(input_dim=1,
output_dim=1,
num_hidden_units=[4, 8, 6],
act_funcs=[activations.gaussian, activations.identity])
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
name = "Test PBGN without line-search, reuse_block_inds={}".format(
reuse_block_inds)
results_filename = "{}.txt".format(name)
results_path = os.path.join(output_dir, results_filename)
results_file = open(results_path, "w")
result = optimisers.Result(name=name, verbose=True, file=results_file)
# Call gradient descent function
result_ls = optimisers.generalised_newton(
n,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters),
evaluator=optimisers.Evaluator(i_interval=1),
result=result,
reuse_block_inds=reuse_block_inds)
results_file.close()
def test_update(seed, result=None):
"""
Test the update method of the Result class. Also return the result, which
can be used as an input to other test functions. Can accept a pre-existing
Result object, EG if it has been initialised with a specific output file
"""
np.random.seed(seed)
output_dim = np.random.randint(2, 5)
n = get_random_network(input_dim=2, output_dim=output_dim)
d = data.Sinusoidal(input_dim=2, output_dim=output_dim, n_train=100)
w = n.get_parameter_vector()
if result is None:
result = results.Result()
result.begin()
# Call the result.update method a few times
for i in range(5):
n.set_parameter_vector(w + i)
result.update(model=n, dataset=d, iteration=i)
# Check that value lists for the default columns have non-zero length
assert len(result.get_values(optimisers.results.columns.TrainError)) > 0
assert len(result.get_values(optimisers.results.columns.TestError)) > 0
assert len(result.get_values(optimisers.results.columns.Iteration)) > 0
assert len(result.get_values(optimisers.results.columns.Time)) > 0
return result
def test_ConstantParameterStatistics(seed):
"""
Test the models.initialisers.ConstantParameterStatistics class, which
initialises a model with constant statistics for the weights and biases in
each layer.
TODO: test with 0, 1, 2 hidden layers
"""
np.random.seed(seed)
input_dim = np.random.randint(2, 10)
output_dim = np.random.randint(2, 10)
N_D = np.random.randint(100, 200)
x_lo = np.random.uniform(-10, 0)
x_hi = np.random.uniform(0, 10)
sin_data = data.Sinusoidal(input_dim, output_dim, N_D, 0, x_lo, x_hi)
initialiser = models.initialisers.ConstantParameterStatistics()
num_hidden_layers = np.random.randint(3, 6)
num_hidden_units = np.random.randint(3, 6, num_hidden_layers)
nn = models.NeuralNetwork(
input_dim,
output_dim,
num_hidden_units,
initialiser=initialiser,
)
assert nn.forward_prop(sin_data.train.x).shape == sin_data.train.y.shape
output_fname = "test_ConstantParameterStatistics, seed=%i.txt" % seed
_print_pre_activation_statistics(nn, output_fname)
def test_standard_columns():
""" Test using a Result object with the standard columns, which are added
to a Result object by default """
# Initialise random seed and number of training data points and iterations
set_random_seed_from_args("test_standard_columns")
n_train = np.random.randint(10, 20)
n_its = np.random.randint(10, 20)
# Initialise model and data set
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
# Initialise output file
test_name = "Test standard columns"
output_filename = "test_standard_columns.txt"
with open(os.path.join(output_dir, output_filename), "w") as f:
# Initialise result object
result = optimisers.Result(name=test_name,
file=f,
add_default_columns=True)
# Perform optimisation
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1))
# Check that each column object has the correct number of values
for col_type in optimisers.results.DEFAULT_COLUMN_TYPES:
# "n_its + 1" because we evaluate the initial state of the model
assert len(result.get_values(col_type)) == n_its + 1
def test_plot_1D_hidden_outputs(output_dim):
""" Test plotting function for outputs from the hidden layers of a model
1-dimensional inputs, and a variable number of outputs """
# Set random seed and parameters
set_random_seed_from_args("test_plot_1D_hidden_outputs", output_dim)
n_train = np.random.randint(50, 100)
n_test = np.random.randint(50, 100)
n_pred = np.random.randint(50, 100)
# Initialise data and model
sin_data = data.Sinusoidal(
input_dim=1,
output_dim=output_dim,
n_train=n_train,
n_test=n_test,
x_lo=0,
x_hi=1,
)
model = get_random_network(
input_dim=1,
output_dim=output_dim,
low=2,
high=6,
initialiser=models.initialisers.ConstantPreActivationStatistics(
x_train=sin_data.train.x,
y_train=sin_data.train.y,
),
)
# Initialise result and column objects
result = optimisers.Result(verbose=False, add_default_columns=False)
prediction_column = optimisers.results.columns.Predictions(
sin_data,
n_points_per_dim=n_pred,
store_hidden_layer_outputs=True,
)
result.add_column(prediction_column)
# Call optimisation function
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=1),
)
# Call plotting function under test
plotting.plot_1D_hidden_outputs(
plot_name="test_plot_1D_hidden_outputs, output_dim=%i" % output_dim,
dir_name=os.path.join(output_dir, "test_plot_1D_hidden_outputs"),
dataset=sin_data,
x_pred=prediction_column.x_pred,
y_pred=prediction_column.predictions_dict[0],
hidden_output_list=prediction_column.hidden_outputs_dict[0],
output_dim=output_dim,
)
def warmup(n_its=1000):
""" Perform warmup routine; useful to call in scripts before testing the
speed of an optimiser, because the process priority often appears to be
initially slow """
sin_data = data.Sinusoidal(1, 1, freq=1)
n = models.NeuralNetwork(1, 1, [20])
gradient_descent(
n,
sin_data,
terminator=Terminator(i_lim=n_its),
evaluator=Evaluator(i_interval=n_its//10),
result=Result(name="Warmup", verbose=True),
line_search=None
)
def test_minimise_reentrant():
""" Test that the AbstractOptimiser.optimise method is re-entrant, IE that
the function can return, and be called again, and the columns in the result
object are as expected (the output from the result object can be found in
the corresponding output file) """
# Set parameters for number of iterations and evaluation frequency
n_iters_1 = 23
eval_every_1 = 5
n_iters_2 = 31
eval_every_2 = 3
# Create model and data
np.random.seed(6307)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
# Open result file
results_filename = "Test AbstractOptimiser.optimise method re-entrant.txt"
results_path = os.path.join(output_dir, results_filename)
with open(results_path, "w") as results_file:
# Create Result object
result = optimisers.Result(name="SGD without line search",
verbose=True,
file=results_file)
# Call gradient descent function twice
result_ls = optimisers.gradient_descent(
model,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters_1),
evaluator=optimisers.Evaluator(i_interval=eval_every_1),
result=result,
display_summary=False)
result_ls = optimisers.gradient_descent(
model,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters_2),
evaluator=optimisers.Evaluator(i_interval=eval_every_2),
result=result)
# Check values in time column are monotonically increasing
time_values = result.get_values(optimisers.results.columns.Time)
for i in range(1, len(time_values)):
assert time_values[i] > time_values[i - 1]
# Check values in iteration column are monotonically increasing
iteration_values = result.get_values(optimisers.results.columns.Iteration)
for i in range(1, len(iteration_values)):
assert iteration_values[i] > iteration_values[i - 1]
# Assert that the list of iteration values is exactly what we expect
all_iter_vals = (
list(range(0, n_iters_1, eval_every_1)) +
list(range(n_iters_1, n_iters_1 + n_iters_2, eval_every_2)) +
[n_iters_1 + n_iters_2])
assert all_iter_vals == iteration_values
def test_plot_result_attribute_subplots():
"""
Test plotting function for plotting the values in multiple columns of a
Result object over time, with one subplot per column
"""
np.random.seed(1521)
n_its = np.random.randint(10, 20)
n_train = np.random.randint(10, 20)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
results_list = []
for i in range(5):
model = models.NeuralNetwork(input_dim=1, output_dim=1)
model.get_gradient_vector(sin_data.train.x, sin_data.train.y)
name = "test_plot_result_attribute_subplots_%i" % (i + 1)
output_text_filename = os.path.join(output_dir, name + ".txt")
with open(output_text_filename, "w") as f:
result = optimisers.Result(name=name, file=f)
ls = optimisers.LineSearch()
ls_column = optimisers.results.columns.StepSize(ls)
dbs_metric_column = optimisers.results.columns.DbsMetric()
result.add_column(ls_column)
result.add_column(dbs_metric_column)
optimisers.gradient_descent(
model,
sin_data,
result=result,
line_search=ls,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1))
results_list.append(result)
attribute_list = [
optimisers.results.columns.TrainError,
optimisers.results.columns.TestError, optimisers.results.columns.Time,
type(ls_column),
type(dbs_metric_column)
]
plotting.plot_result_attributes_subplots(
"test_plot_result_attribute_subplots",
output_dir,
results_list,
attribute_list,
marker="o",
line_style="--",
log_axes_attributes={
optimisers.results.columns.TrainError,
optimisers.results.columns.TestError,
type(ls_column)
})
def test_optimal_batch_size_column():
""" Test using a column which approximates the optimal batch size on each
iteration """
# Set random seed and initialise network and dataset
set_random_seed_from_args("test_optimal_batch_size_column")
n_train = np.random.randint(10, 20)
n_its = np.random.randint(10, 20)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
# Initialise output file and Result object
test_name = "Test optimal batch size column"
output_filename = "test_optimal_batch_size_column.txt"
with open(os.path.join(output_dir, output_filename), "w") as f:
result = optimisers.Result(name=test_name,
file=f,
add_default_columns=True)
# Initialise line-search and column object, and add to the result
n_batch_sizes = np.random.randint(3, 6)
n_repeats = np.random.randint(3, 6)
line_search = optimisers.LineSearch()
columns = optimisers.results.columns
gd_optimiser = optimisers.GradientDescent(line_search)
optimal_batch_size_col = columns.OptimalBatchSize(
gd_optimiser,
sin_data.train.n,
n_repeats=n_repeats,
n_batch_sizes=n_batch_sizes,
)
result.add_column(optimal_batch_size_col)
# Call optimisation function
gd_optimiser.optimise(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
)
# Test that the OptimalBatchSize object attributes are as expected
batch_size_list = optimal_batch_size_col.batch_size_list
assert len(optimal_batch_size_col.reduction_dict_dict) == (n_its + 1)
for reduction_dict in optimal_batch_size_col.reduction_dict_dict.values():
assert len(reduction_dict) == n_batch_sizes
assert set(reduction_dict.keys()) == set(batch_size_list)
for reduction_list in reduction_dict.values():
assert len(reduction_list) == n_repeats
def test_plot_hidden_outputs_gif():
""" Test function to make a gif of hidden layer outputs and predictions
formed by the model during training, for regression or classification """
# Set random seed and initialise network and dataset
set_random_seed_from_args("test_plot_hidden_outputs_gif")
n_train = np.random.randint(10, 20)
n_pred = np.random.randint(5, 10)
n_its = 2
dataset = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
model = get_random_network(
input_dim=1,
output_dim=1,
initialiser=models.initialisers.ConstantPreActivationStatistics(
x_train=dataset.train.x,
y_train=dataset.train.y,
),
)
# Initialise Result and Predictions column object
result = optimisers.Result(verbose=False)
pred_column = optimisers.results.columns.Predictions(
dataset,
n_pred,
store_hidden_layer_outputs=True,
)
result.add_column(pred_column)
# Call optimisation function
optimisers.gradient_descent(
model,
dataset,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
line_search=optimisers.LineSearch(),
)
# Call plotting function
plot_name = "test_plot_hidden_outputs_gif"
plotting.plot_hidden_outputs_gif(
plot_name=plot_name,
dir_name=os.path.join(output_dir, plot_name),
result=result,
prediction_column=pred_column,
dataset=dataset,
output_dim=1,
duration=1000,
)
def test_batch_improvement_probability_column(smooth_output):
""" Test using a column which measures the probability of improvement in
each consecutive batch """
set_random_seed_from_args(
"test_batch_improvement_probability_column",
smooth_output,
)
n_train = np.random.randint(10, 20)
n_its = np.random.randint(50, 100)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
batch_size = np.random.randint(3, sin_data.train.n)
test_name = (
"test_batch_improvement_probability_column, smooth_output=%s" %
smooth_output)
output_filename = "%s.txt" % test_name
with open(os.path.join(output_dir, output_filename), "w") as f:
ls = optimisers.LineSearch()
result = optimisers.Result(
name=test_name,
file=f,
add_default_columns=True,
)
dynamic_terminator = optimisers.DynamicTerminator(
model=model,
dataset=sin_data,
batch_size=batch_size,
smooth_output=smooth_output,
i_lim=n_its,
)
result.add_column(
optimisers.results.columns.BatchImprovementProbability(
dynamic_terminator, ))
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=dynamic_terminator,
batch_getter=dynamic_terminator,
evaluator=optimisers.Evaluator(i_interval=1),
line_search=ls,
)
def test_test_set_improvement_probability_simple_column(
smoother_type,
use_cdf,
):
""" Test using a column which measures the probability of improvement in
the test set """
set_random_seed_from_args(
"test_test_set_improvement_probability_simple_column", )
n_train = np.random.randint(10, 20)
n_its = np.random.randint(50, 100)
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
test_name = ("test_test_set_improvement_probability_simple_column, "
"use_cdf=%s, smoother_type=%s" %
(use_cdf,
(None if
(smoother_type is None) else smoother_type.__name__)))
output_filename = "%s.txt" % test_name
with open(os.path.join(output_dir, output_filename), "w") as f:
ls = optimisers.LineSearch()
result = optimisers.Result(
name=test_name,
file=f,
add_default_columns=True,
)
smoother = None if (smoother_type is None) else smoother_type(0)
result.add_column(
optimisers.results.columns.TestSetImprovementProbabilitySimple(
model=model,
dataset=sin_data,
smoother=smoother,
use_cdf=use_cdf,
))
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
line_search=ls,
)
def test_plot_error_reductions_vs_batch_size_gif():
""" Test function which plots a gif of the statistics for the reduction in
the mean error in the test set after a single minimisation iteration, as a
function of the batch size used for the iteration, where each frame in the
gif represents a different iteration throughout the course of
model-optimisation """
# Set random seed and initialise network and dataset
np.random.seed(102)
n_train = 10
n_its = 2
model = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, n_train=n_train)
# Initialise Result, LineSearch and OptimalBatchSize column objects
result = optimisers.Result(verbose=False)
line_search = optimisers.LineSearch()
gd_optimiser = optimisers.GradientDescent(line_search)
columns = optimisers.results.columns
optimal_batch_size_col = columns.OptimalBatchSize(gd_optimiser,
sin_data.train.n,
n_repeats=3,
n_batch_sizes=3,
min_batch_size=2)
result.add_column(optimal_batch_size_col)
# Call optimisation function
gd_optimiser.optimise(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
)
# Call plotting function to make gif
test_output_dir = os.path.join(
output_dir, "Test plot_error_reductions_vs_batch_size_gif")
plotting.plot_error_reductions_vs_batch_size_gif(result,
optimal_batch_size_col,
test_output_dir,
loop=None)
def test_gradient_descent_line_search(seed):
"""
Test gradient descent, using a line-search. A line-search should guarantee
that each iteration reduces the error function, so this is tested using
assert statements after calling the gradient_descent function.
"""
# Set the random seed
np.random.seed(seed)
# Generate random number of iterations, network, data, and results file
n_iters = np.random.randint(10, 20)
n = get_random_network(input_dim=1, output_dim=1)
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
results_filename = (
"Test gradient descent with line-search, seed = %i.txt" % seed)
results_path = os.path.join(output_dir, results_filename)
results_file = open(results_path, "w")
result = optimisers.Result(name="SGD with line search",
verbose=True,
file=results_file)
# Add step size column to result
ls = optimisers.LineSearch(max_its=int(1e10))
result.add_column(optimisers.results.columns.StepSize(ls))
# Call gradient descent function
result_ls = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(i_lim=n_iters),
evaluator=optimisers.Evaluator(i_interval=1),
line_search=ls,
result=result)
# Make sure each iteration reduces the training error
train_error_list = result.get_values(optimisers.results.columns.TrainError)
for i in range(len(train_error_list) - 1):
assert train_error_list[i + 1] < train_error_list[i]
results_file.close()
def main(input_dim, output_dim, n_train, batch_size, t_lim, num_hidden_units,
e_lims, n_repeats):
"""
Main function for this script, wrapped by argparse for command-line
arguments.
"""
# Perform warmup experiment so process acquires priority
optimisers.warmup()
# Initialise data, time limit, and results list
np.random.seed(9251)
sin_data = data.Sinusoidal(input_dim=input_dim,
output_dim=output_dim,
n_train=n_train)
t_interval = t_lim / 50
results_list = []
for i in range(n_repeats):
# Set the random seed
np.random.seed(i)
# Generate random network
n = models.NeuralNetwork(
input_dim=input_dim,
output_dim=output_dim,
num_hidden_units=num_hidden_units,
act_funcs=[models.activations.cauchy, models.activations.identity],
initialiser=models.initialisers.ConstantPreActivationStatistics(
sin_data.x_train, sin_data.y_train))
# Set name for experiment
name = "Constant pre-activation statistics"
# Call gradient descent function
result = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name=name, verbose=True),
line_search=optimisers.LineSearch(),
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result)
# Generate random network
n = models.NeuralNetwork(
input_dim=input_dim,
output_dim=output_dim,
num_hidden_units=num_hidden_units,
act_funcs=[models.activations.cauchy, models.activations.identity],
initialiser=models.initialisers.ConstantParameterStatistics())
# Set name for experiment
name = "Constant parameter statistics"
# Call gradient descent function
result = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name=name, verbose=True),
line_search=optimisers.LineSearch(),
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result)
# Get name of output directory
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(current_dir, "Outputs")
# Compare training curves
plotting.plot_training_curves(
results_list,
"Comparing initialisers for gradient descent on 2D sinusoidal data",
output_dir,
e_lims=e_lims,
tp=0.5)
def main(input_dim, output_dim, n_train, batch_size, t_lim, num_hidden_units,
e_lims, n_repeats):
"""
Main function for the script. See module docstring for more info.
Inputs:
- input_dim: positive integer number of input dimensions
- output_dim: positive integer number of output dimensions
- n_train: positive integer number of points in the training set
- batch_size: positive integer batch size to use for training
- t_lim: positive float, length of time to train for each experiment
- num_hidden_units: list of positive integers, number of hidden units in
each hidden layer of the NeuralNetwork, EG [10] or [20, 20]
- e_lims: list of 2 floats, used as axis limits in the output plots
- n_repeats: positive integer number of repeats to perform of each
experiment
"""
# Perform warmup experiment so process acquires priority
optimisers.warmup()
# Initialise data, results list, and time interval for evaluations
np.random.seed(9251)
sin_data = data.Sinusoidal(
input_dim=input_dim,
output_dim=output_dim,
n_train=n_train,
)
results_list = []
t_interval = t_lim / 50
for i in range(n_repeats):
# Set the random seed
np.random.seed(i)
# Generate random network and store initial parameters
n = models.NeuralNetwork(input_dim=input_dim,
output_dim=output_dim,
num_hidden_units=num_hidden_units,
act_funcs=[
models.activations.gaussian,
models.activations.identity
])
w0 = n.get_parameter_vector().copy()
# Call gradient descent function
result_gd_ls = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="SGD with line search",
verbose=True),
line_search=optimisers.LineSearch(),
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result_gd_ls)
# Try again without line search
n.set_parameter_vector(w0)
result_gd_no_ls = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="SGD without line search",
verbose=True),
line_search=None,
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result_gd_no_ls)
# Call generalised Newton function
n.set_parameter_vector(w0)
result_pbgn_ls = optimisers.generalised_newton(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="PBGN with line search",
verbose=True),
line_search=optimisers.LineSearch(),
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result_pbgn_ls)
# Try again without line search
n.set_parameter_vector(w0)
result_pbgn_no_ls = optimisers.generalised_newton(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="PBGN without line search",
verbose=True),
line_search=None,
batch_getter=optimisers.batch.ConstantBatchSize(batch_size))
results_list.append(result_pbgn_no_ls)
# Get name of output directory
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(current_dir, "Outputs")
# Compare training curves
plot_name = "Comparing gradient descent vs generalised Newton"
plot_name += ", %iD-%iD data" % (input_dim, output_dim)
plot_name += ", %.2g s training time" % t_lim
plot_name += ", %s hidden units" % str(num_hidden_units)
plotting.plot_training_curves(results_list,
plot_name,
output_dir,
e_lims=e_lims)
def test_predictions_column(
input_dim,
output_dim,
store_hidden,
store_preactivations,
):
""" Test using a column which stores model predictions during training """
# Set random seed and initialise network and dataset
set_random_seed_from_args(
"test_predictions_column",
input_dim,
output_dim,
store_hidden,
)
n_train = np.random.randint(10, 20)
n_pred = ceil(pow(np.random.randint(5, 10), 1 / input_dim))
n_its = np.random.randint(10, 20)
model = get_random_network(input_dim=input_dim, output_dim=output_dim)
sin_data = data.Sinusoidal(
input_dim=input_dim,
output_dim=output_dim,
n_train=n_train,
)
# Initialise output file and Result object
test_name = "test_predictions_column, %id-%id data, store_hidden=%s" % (
input_dim,
output_dim,
store_hidden,
)
output_filename = "%s.txt" % test_name
with open(os.path.join(output_dir, output_filename), "w") as f:
# Initialise result object
result = optimisers.Result(name=test_name,
file=f,
add_default_columns=True)
# Initialise column object and add to the result
columns = optimisers.results.columns
prediction_column = columns.Predictions(
sin_data,
n_points_per_dim=n_pred,
store_hidden_layer_outputs=store_hidden,
store_hidden_layer_preactivations=store_preactivations,
)
result.add_column(prediction_column)
# Call optimisation function
optimisers.gradient_descent(
model,
sin_data,
result=result,
terminator=optimisers.Terminator(i_lim=n_its),
evaluator=optimisers.Evaluator(i_interval=1),
)
# Print Predictions column attributes to file
print("\n\nx_pred:", prediction_column.x_pred, sep="\n", file=f)
iter_list = result.get_values(columns.Iteration)
print("\n\nPredictions:", file=f)
for i in iter_list:
print("i = %i:" % i, file=f)
print(prediction_column.predictions_dict[i], file=f)
if store_hidden:
print("\n\nHidden layer outputs:", file=f)
for i in iter_list:
print(
"\ni = %i:" % i,
*prediction_column.hidden_outputs_dict[i],
file=f,
sep="\n\n",
)
# Test that the Prediction object attributes are as expected
n_pred_grid = pow(n_pred, input_dim)
assert prediction_column.x_pred.shape == (input_dim, n_pred_grid)
iter_set = set(iter_list)
assert set(prediction_column.predictions_dict.keys()) == iter_set
for y_pred in prediction_column.predictions_dict.values():
assert y_pred.shape == (output_dim, n_pred_grid)
hidden_outputs_dict = prediction_column.hidden_outputs_dict
if store_hidden:
assert set(hidden_outputs_dict.keys()) == iter_set
for hidden_output_list in hidden_outputs_dict.values():
assert len(hidden_output_list) == len(model.layers) - 1
for i, hidden_output in enumerate(hidden_output_list):
expected_shape = (model.layers[i].output_dim, n_pred_grid)
assert hidden_output.shape == expected_shape
else:
assert len(hidden_outputs_dict) == 0
def main(args):
# Get name of output directory, and create it if it doesn't exist
param_str = (
"input_dim = %i, output_dim = %i, n_train = %i, t_lim = %.2f, "
"num_repeats = %i, find_best_params = %s"
% (
args.input_dim,
args.output_dim,
args.n_train,
args.t_lim,
args.n_repeats,
args.find_best_params,
)
)
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(
current_dir,
"Outputs",
"Gradient descent",
param_str,
)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
# Initialise data set
np.random.seed(6763)
sin_data = data.Sinusoidal(
args.input_dim,
args.output_dim,
args.n_train,
x_lo=-2,
x_hi=2,
freq=(1 if args.input_dim == 1 else None),
)
# Define function to be run for each experiment
def run_experiment(
num_units,
num_layers,
log10_s0,
alpha,
beta,
act_func,
max_steps,
batch_size,
batch_replace,
plot_preds=False,
):
# Initialise network and batch getter
model = models.NeuralNetwork(
input_dim=args.input_dim,
output_dim=args.output_dim,
num_hidden_units=[num_units for _ in range(num_layers)],
act_funcs=[act_func, models.activations.identity],
)
if (batch_size is None) or (batch_size >= args.n_train):
batch_getter = optimisers.batch.FullTrainingSet()
else:
batch_getter = optimisers.batch.ConstantBatchSize(
batch_size,
batch_replace,
)
# Perform gradient descent
result = optimisers.gradient_descent(
model,
sin_data,
terminator=optimisers.Terminator(t_lim=args.t_lim),
evaluator=optimisers.Evaluator(t_interval=args.t_eval),
line_search=optimisers.LineSearch(
s0=pow(10, log10_s0),
alpha=alpha,
beta=beta,
max_its=max_steps,
),
batch_getter=batch_getter,
)
# If specified, plot the final model predictions
if plot_preds:
print("Plotting final predictions...")
plotting.plot_data_predictions(
plot_name="Final predictions",
dir_name=output_dir,
dataset=sin_data,
output_dim=args.output_dim,
model=model,
)
# Return the final test error
TestError = optimisers.results.columns.TestError
final_test_error = result.get_values(TestError)[-1]
return final_test_error
# Initialise the Experiment object, and add parameters
experiment = Experiment(run_experiment, output_dir, args.n_repeats)
addp = lambda *args: experiment.add_parameter(Parameter(*args))
addp("num_units", 10, [5, 10, 15, 20] )
addp("num_layers", 1, [1, 2, 3] )
addp("log10_s0", 0, np.linspace(-1, 3, 5) )
addp("alpha", 0.5, np.linspace(0.1, 1, 9, endpoint=False) )
addp("beta", 0.5, np.linspace(0.1, 1, 9, endpoint=False) )
addp("max_steps", 10, [5, 10, 15, 20] )
addp("batch_size", 100, [25, 50, 75, 100, 150, 200, 300, None] )
addp("batch_replace", True, [True, False] )
addp(
"act_func",
models.activations.gaussian,
[
models.activations.gaussian,
models.activations.cauchy,
models.activations.logistic,
models.activations.relu,
],
)
# Call warmup function
optimisers.warmup()
# Call function to run all experiments
if args.find_best_params:
experiment.find_best_parameters()
else:
experiment.sweep_all_parameters()
# Write the results of all experiments to a text file
experiment.save_results_as_text()
# Open the output plot directory
os.system("explorer \"%s\"" % output_dir)
# Plot the predictions using the model with the optimal hyper-parameters
default_param_dict = experiment.get_default_param_dictionary()
run_experiment(**default_param_dict, plot_preds=True)
# Get name of output directory
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(current_dir, "Outputs")
# Set the random seed
np.random.seed(2865)
# Generate random network and data
input_dim = 2
output_dim = 3
x_lo = -2
x_hi = 2
sin_data = data.Sinusoidal(
input_dim=input_dim,
output_dim=output_dim,
x_lo=x_lo,
x_hi=x_hi
)
model = models.NeuralNetwork(
input_dim=input_dim,
output_dim=output_dim,
num_hidden_units=[20, 20],
act_funcs=[models.activations.cauchy, models.activations.identity]
)
# Call gradient descent function
result = optimisers.gradient_descent(
model,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
t_interval = t_lim / 10
# Get name of output directory
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(current_dir, "Outputs")
# Set the random seed
np.random.seed(2865)
# Generate random network and data
n = models.NeuralNetwork(
input_dim=1,
output_dim=1,
num_hidden_units=[10],
act_funcs=[models.activations.cauchy, models.activations.identity])
sin_data = data.Sinusoidal(input_dim=1, output_dim=1, freq=1)
# Call gradient descent function
result = optimisers.gradient_descent(
n,
sin_data,
terminator=optimisers.Terminator(t_lim=t_lim),
evaluator=optimisers.Evaluator(t_interval=t_interval),
result=optimisers.Result(name="SGD with line search", verbose=True),
line_search=optimisers.LineSearch())
# Plot predictions
x_pred = np.linspace(-2, 2, 200).reshape(1, -1)
y_pred = n.forward_prop(x_pred)
plotting.plot_1D_regression("Gradient descent predictions for 1D sin data",
output_dir, sin_data, x_pred, y_pred)
def main(
input_dim,
output_dim,
n_train,
num_hidden_units,
n_repeats,
n_iters,
n_plots,
n_batch_sizes,
min_batch_size,
ylims,
seed,
batch_size_optimise,
use_replacement,
gif_duration
):
"""
Main function for the script. See module docstring for more info.
Inputs:
- input_dim: positive integer number of input dimensions
- output_dim: positive integer number of output dimensions
- n_train: positive integer number of points in the training set
- num_hidden_units: list of positive integers, number of hidden units in
each hidden layer of the NeuralNetwork, EG [10] or [20, 20]
- n_repeats: positive integer number of repeats to perform of each batch
size test
- n_iters: total number of iterations to perform
- n_plots: number of frames of the gif (equal to how many times
optimisation will pause in order to sweep over the list of batch sizes)
- n_batch_sizes: the number of different batch sizes to test for each
iteration
- min_batch_size: the smallest batch size to test
- ylims: limits for the y-axes of each subplot of the output gif. Should
be None, in which case the axis limits are calculated automatically, or
an iterable containing 4 floats, in which the first 2 are the lower and
upper axis limits for the left subplot, and the second 2 are the lower
and upper axis limits for the right subplot
- seed: random seed to use for the experiment
- batch_size_optimise: batch size to use for standard optimisation
iterations (IE not when sweeping over batch sizes). If ommitted, then
the full training set is used as a batch during optimisation iterations
- use_replacement: if True, then use replacement when sampling batches
from the training set
- gif_duration: time in seconds that the output gif should last for in
total
"""
np.random.seed(seed)
n_iters_per_plot = int(n_iters / n_plots)
# Initialise model and dataset
model = models.NeuralNetwork(input_dim, output_dim, num_hidden_units)
freq = 1 if (input_dim == 1) else None
sin_data = data.Sinusoidal(input_dim, output_dim, n_train, freq=freq)
# Initialise objects for optimisation
result = optimisers.Result()
evaluator = optimisers.Evaluator(i_interval=n_iters_per_plot)
terminator = optimisers.Terminator(i_lim=n_iters)
if batch_size_optimise is None:
batch_getter = optimisers.batch.FullTrainingSet()
else:
batch_getter = optimisers.batch.ConstantBatchSize(
batch_size_optimise,
use_replacement
)
line_search = optimisers.LineSearch()
# Initialise OptimalBatchSize column and add to the result object
gd_optimiser = optimisers.GradientDescent(line_search)
optimal_batch_size_col = optimisers.results.columns.OptimalBatchSize(
gd_optimiser,
sin_data.n_train,
n_repeats=n_repeats,
n_batch_sizes=n_batch_sizes
)
result.add_column(optimal_batch_size_col)
# Get output directory which is specific to the script parameters
param_str = ", ".join([
"input_dim = %i" % input_dim,
"output_dim = %i" % output_dim,
"n_train = %i" % n_train,
"n_iters = %i" % n_iters,
"batch_size_optimise = %r" % batch_size_optimise,
"use_replacement = %r" % use_replacement,
"ylims = %r" % ylims,
"n_plots = %i" % n_plots,
])
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.join(
current_dir,
"Outputs",
"Error vs batch",
param_str
)
# Call optimisation function
gd_optimiser.optimise(
model,
sin_data,
result=result,
batch_getter=batch_getter,
terminator=terminator,
evaluator=evaluator,
)
# Make output plots
print("Plotting output plots in \"%s\"..." % output_dir)
frame_duration_ms = 1000 * gif_duration / n_plots
if ylims is None:
y_lim_left = None
y_lim_right = None
else:
y_lim_left = ylims[:2]
y_lim_right = ylims[2:]
plotting.plot_error_reductions_vs_batch_size_gif(
result,
optimal_batch_size_col,
output_dir,
y_lim_left=y_lim_left,
y_lim_right=y_lim_right,
duration=frame_duration_ms,
loop=None
)
plotting.plot_optimal_batch_sizes(
"Optimal batch size",
output_dir,
result,
optimal_batch_size_col,
)
