def evaluate_scale(inputs, targets, folds, centres, reg_param, scales=None):
"""
evaluate then plot the performance of different basis function scales
"""
# choose a range of scales
if scales is None:
scales = np.logspace(-10, 10)
#
num_values = scales.size
num_folds = len(folds)
# create some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_stdev_errors = np.zeros(num_values)
test_stdev_errors = np.zeros(num_values)
#
for s, scale in enumerate(scales):
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[s] = train_mean_error
test_mean_errors[s] = test_mean_error
train_stdev_errors[s] = train_stdev_error
test_stdev_errors[s] = test_stdev_error
# Now plot the results
fig, ax = plot_train_test_errors("scale", scales, train_mean_errors,
test_mean_errors)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_stdev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_stdev_errors / np.sqrt(num_folds)
ax.fill_between(scales, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_stdev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_stdev_errors / np.sqrt(num_folds)
ax.fill_between(scales, lower, upper, alpha=0.2, color='r')
ax.set_xscale('log')
def evaluate_reg_param(inputs, targets, folds, reg_params=None):
"""
Evaluate then plot the performance of different regularisation parameters
"""
# choose a range of regularisation parameters
if reg_params is None:
reg_params = np.logspace(-2,0)
num_values = reg_params.size
num_folds = len(folds)
# create some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_stdev_errors = np.zeros(num_values)
test_stdev_errors = np.zeros(num_values)
#
for r, reg_param in enumerate(reg_params):
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(inputs, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[r] = train_mean_error
test_mean_errors[r] = test_mean_error
train_stdev_errors[r] = train_stdev_error
test_stdev_errors[r] = test_stdev_error
# Now plot the results
fig, ax = plot_train_test_errors(
"$\lambda$", reg_params, train_mean_errors, test_mean_errors)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_stdev_errors/np.sqrt(num_folds)
upper = train_mean_errors + train_stdev_errors/np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_stdev_errors/np.sqrt(num_folds)
upper = test_mean_errors + test_stdev_errors/np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='r')
ax.set_xscale('log')
def evaluate_errors_num_centres(inputs,
targets,
folds,
scale,
reg_param,
num_centres_sequence=None):
"""
Evaluate then plot the performance of different numbers of basis
function centres.
"""
# fix the reg_param
reg_param = 0.01
# fix the scale
scale = 100
# choose a range of numbers of centres
if num_centres_sequence is None:
num_centres_sequence = np.arange(200, 250)
num_values = num_centres_sequence.size
num_folds = len(folds)
#
# create array to store results
test_mean_errors = np.zeros(num_values)
#
# run the experiments
for c, num_centres in enumerate(num_centres_sequence):
centres = np.linspace(0, 1, num_centres)
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
test_mean_error = np.mean(test_errors)
# store the results
test_mean_errors[c] = test_mean_error
return test_mean_errors
def evaluate_scale(inputs,
targets,
folds,
centres,
reg_param,
test_error_linear,
scales=None):
"""
Evaluate, then plot the performance of different basis function scales.
"""
# choosing a range of scales
if scales is None:
scales = np.logspace(0, 8, 30) # of the basis functions
#
num_values = scales.size
num_folds = len(folds)
# creating some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_st_dev_errors = np.zeros(num_values)
test_st_dev_errors = np.zeros(num_values)
for s, scale in enumerate(scales):
feature_mapping = construct_rbf_feature_mapping(centres, scale)
design_matrix = feature_mapping(inputs)
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
design_matrix, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_st_dev_error = np.std(train_errors)
test_st_dev_error = np.std(test_errors)
# store the results
train_mean_errors[s] = train_mean_error
test_mean_errors[s] = test_mean_error
train_st_dev_errors[s] = train_st_dev_error
test_st_dev_errors[s] = test_st_dev_error
# Now plot the results
fig, ax = plot_train_test_errors("scale", scales, train_mean_errors,
test_mean_errors, test_error_linear)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_st_dev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_st_dev_errors / np.sqrt(num_folds)
ax.fill_between(scales, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_st_dev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_st_dev_errors / np.sqrt(num_folds)
ax.fill_between(scales, lower, upper, alpha=0.2, color='r')
ax.set_xscale('log')
ax.set_ylim([0, 1])
ax.set_title('Train vs Test Error across Scales with Cross-validation')
fig.savefig("../plots/rbf/rbf_searching_scales_cross_validation.png",
fmt="png")
plt.show()
def evaluate_num_centres(inputs,
targets,
folds,
scale,
reg_param,
test_error_linear,
num_centres_sequence=None):
"""
Evaluate, then plot the performance of different numbers of basis
function centres.
"""
# choosing a range of numbers of centres
if num_centres_sequence is None:
num_centres_sequence = np.linspace(
start=0.01, stop=1,
num=20) # tested with 50, using 20 to speed things up
num_values = num_centres_sequence.size
num_folds = len(folds)
# creating some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_st_dev_errors = np.zeros(num_values)
test_st_dev_errors = np.zeros(num_values)
n = inputs.shape[0]
# running the experiments
for c, centre_percentage in enumerate(num_centres_sequence):
sample_fraction = centre_percentage
p = (1 - sample_fraction, sample_fraction)
# constructing the feature mapping anew for each number of centres
centres = inputs[np.random.choice([False, True], size=n, p=p), :]
# print("\ncentres.shape = %r" % (centres.shape,))
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[c] = train_mean_error
test_mean_errors[c] = test_mean_error
train_st_dev_errors[c] = train_stdev_error
test_st_dev_errors[c] = test_stdev_error
# now plotting the results
fig, ax = plot_train_test_errors("% of inputs as centres * 100",
num_centres_sequence, train_mean_errors,
test_mean_errors, test_error_linear)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_st_dev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_st_dev_errors / np.sqrt(num_folds)
ax.fill_between(num_centres_sequence, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_st_dev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_st_dev_errors / np.sqrt(num_folds)
ax.fill_between(num_centres_sequence, lower, upper, alpha=0.2, color='r')
ax.set_ylim([0, 1])
ax.set_title(
'Train vs Test Error across Centre Proportion with Cross-validation')
fig.savefig(
"../plots/rbf/rbf_searching_number_centres_cross_validation.png",
fmt="png")
plt.show()
def main(ifname=None,
delimiter=None,
columns=None,
normalise=None,
features=None):
"""
To be called when the script is run. This function fits and plots imported data (if a filename is
provided). Data is 2 dimensional real valued data and is fit
with maximum likelihood 2d gaussian.
parameters
----------
ifname -- filename/path of data file.
delimiter -- delimiter of data values
has_header -- does the data-file have a header line
columns -- a list of integers specifying which columns of the file to import
(counting from 0)
"""
# if no file name is provided then use synthetic data
if ifname is None:
print("You need to ingest the CSV file")
else:
data, field_names = import_data(ifname,
delimiter=delimiter,
has_header=True,
columns=columns)
# DATA PREPARATION-----------------------------------------------
N = data.shape[0]
target = data[:, 11:]
# Ask user to confirm whether to normalise or not
if normalise == None:
normalise_response = input(
"Do you want to normalise the data? (Y/N)")
normalise = normalise_response.upper()
normalise_label = ""
if normalise == "Y":
normalise_label = "_normalised"
# Normalise input data
fixed_acidity = data[:, 0]
volatility_acidity = data[:, 1]
citric_acid = data[:, 2]
residual_sugar = data[:, 3]
chlorides = data[:, 4]
free_sulfur_dioxide = data[:, 5]
total_sulfur_dioxide = data[:, 6]
density = data[:, 7]
pH = data[:, 8]
sulphates = data[:, 9]
alcohol = data[:, 10]
data[:, 0] = (fixed_acidity -
np.mean(fixed_acidity)) / np.std(fixed_acidity)
data[:, 1] = (volatility_acidity - np.mean(volatility_acidity)
) / np.std(volatility_acidity)
data[:, 2] = (citric_acid -
np.mean(citric_acid)) / np.std(citric_acid)
data[:, 3] = (residual_sugar -
np.mean(residual_sugar)) / np.std(residual_sugar)
data[:, 4] = (chlorides - np.mean(chlorides)) / np.std(chlorides)
data[:, 5] = (free_sulfur_dioxide - np.mean(free_sulfur_dioxide)
) / np.std(free_sulfur_dioxide)
data[:, 6] = (total_sulfur_dioxide - np.mean(total_sulfur_dioxide)
) / np.std(total_sulfur_dioxide)
data[:, 7] = (density - np.mean(density)) / np.std(density)
data[:, 8] = (pH - np.mean(pH)) / np.std(pH)
data[:, 9] = (sulphates - np.mean(sulphates)) / np.std(sulphates)
data[:, 10] = (alcohol - np.mean(alcohol)) / np.std(alcohol)
elif normalise != "N":
sys.exit("Please enter valid reponse of Y or N")
if features == None:
feature_response = input(
"Please specify which feature combination you want (e.g.1,2,5,7)"
)
feature_response = feature_response.split(",")
# need to convert list of strings into list of integer
feature_combin = []
for i in range(len(feature_response)):
print(feature_response[i])
feature_combin.append(int(feature_response[i]))
else:
feature_combin = features
inputs = np.array([])
for j in range(len(feature_combin)):
inputs = np.append(inputs, data[:, feature_combin[j]])
inputs = inputs.reshape(len(feature_combin), data.shape[0])
inputs = (np.rot90(inputs, 3))[:, ::-1]
#print("INPUT: ", inputs)
# Plotting RBF Model ----------------------------------------------------------
# specify the centres of the rbf basis functions
centres = np.asarray([
0.35, 0.4, 0.45, 0.459090909, 0.468181818, 0.477272727,
0.486363636, 0.495454545, 0.504545455, 0.513636364, 0.522727273,
0.531818182, 0.540909091, 0.55, 0.56, 0.57, 0.58, 0.59, 0.6, 0.61,
0.62, 0.63, 0.64, 0.65, 0.7, 0.75, 0.8
])
# the width (analogous to standard deviation) of the basis functions
scale = 450
reg_param = 7.906043210907701e-11
print("centres = %r" % (centres, ))
print("scale = %r" % (scale, ))
print("reg param = %r" % (reg_param, ))
# create the feature mapping
feature_mapping = construct_rbf_feature_mapping(centres, scale)
# plot the basis functions themselves for reference
#display_basis_functions(feature_mapping)
# now construct the design matrix for the inputs
designmtx = feature_mapping(inputs)
# the number of features is the widht of this matrix
print("DESIGN MATRIX: ", designmtx)
if reg_param is None:
# use simple least squares approach
weights = ml_weights(designmtx, target)
else:
# use regularised least squares approach
weights = regularised_ml_weights(designmtx, target, reg_param)
# get the cross-validation folds
num_folds = 4
folds = create_cv_folds(N, num_folds)
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, target, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
print("TRAIN MEAN ERROR: ", train_mean_error)
print("TEST MEAN ERROR: ", test_mean_error)
print("TRAIN STDEV ERROR: ", train_stdev_error)
print("TEST STDEV ERROR: ", test_stdev_error)
print("ML WEIGHTS: ", weights)
apply_validation_set(feature_combin, feature_mapping, weights)
def evaluate_num_centres(inputs,
targets,
folds,
scale,
reg_param,
num_centres_sequence=None):
"""
Evaluate then plot the performance of different numbers of basis
function centres.
"""
# choose a range of numbers of centres
if num_centres_sequence is None:
num_centres_sequence = np.arange(1, 20)
num_values = num_centres_sequence.size
num_folds = len(folds)
#
# create some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_stdev_errors = np.zeros(num_values)
test_stdev_errors = np.zeros(num_values)
#
# run the experiments
for c, num_centres in enumerate(num_centres_sequence):
centres = np.linspace(0, 1, num_centres)
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[c] = train_mean_error
test_mean_errors[c] = test_mean_error
train_stdev_errors[c] = train_stdev_error
test_stdev_errors[c] = test_stdev_error
#
# Now plot the results
fig, ax = plot_train_test_errors("no. centres", num_centres_sequence,
train_mean_errors, test_mean_errors)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_stdev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_stdev_errors / np.sqrt(num_folds)
ax.fill_between(num_centres_sequence, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_stdev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_stdev_errors / np.sqrt(num_folds)
ax.fill_between(num_centres_sequence, lower, upper, alpha=0.2, color='r')
ax.set_title(
'Train vs Test Error Across Centre Number With Cross-Validation')
fig.savefig("../plots/rbf_searching_number_centres_cross_validation.pdf",
fmt="pdf")
def parameter_search_rbf(inputs, targets, test_fraction, folds):
"""
"""
n = inputs.shape[0]
# for the centres of the basis functions sample 10% of the data
sample_fraction = 0.05
p = (1 - sample_fraction, sample_fraction)
centres = inputs[np.random.choice([False, True], size=n, p=p), :]
print("\ncentres.shape = %r" % (centres.shape, ))
scales = np.logspace(0, 4, 20) # of the basis functions
reg_params = np.logspace(-16, -1, 20) # choices of regularisation strength
# create empty 2d arrays to store the train and test errors
train_mean_errors = np.empty((scales.size, reg_params.size))
test_mean_errors = np.empty((scales.size, reg_params.size))
# iterate over the scales
for i, scale in enumerate(scales):
# i is the index, scale is the corresponding scale
# we must recreate the feature mapping each time for different scales
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# partition the design matrix and targets into train and test
# iterating over the regularisation parameters
for j, reg_param in enumerate(reg_params):
# j is the index, reg_param is the corresponding regularisation
# parameter
# train and test the data
train_error, test_error = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# store the train and test errors in our 2d arrays
train_mean_errors[i, j] = np.mean(train_error)
test_mean_errors[i, j] = np.mean(test_error)
# we have a 2d array of train and test errors, we want to know the (i,j)
# index of the best value
best_i = np.argmin(np.argmin(test_mean_errors, axis=1))
best_j = np.argmin(test_mean_errors[i, :])
min_place = np.argmin(test_mean_errors)
best_i_correct = (int)(min_place / test_mean_errors.shape[1])
best_j_correct = min_place % test_mean_errors.shape[1]
print("\nBest joint choice of parameters:")
print("\tscale %.2g and lambda = %.2g" %
(scales[best_i_correct], reg_params[best_j_correct]))
# now we can plot the error for different scales using the best
# regularisation choice
fig, ax = plot_train_test_errors("scale", scales,
train_mean_errors[:, best_j_correct],
test_mean_errors[:, best_j_correct])
ax.set_xscale('log')
ax.set_title('Train vs Test Error Across Scales')
fig.savefig("../plots/rbf_searching_scales.pdf", fmt="pdf")
# ...and the error for different regularisation choices given the best
# scale choice
fig, ax = plot_train_test_errors("$\lambda$", reg_params,
train_mean_errors[best_i_correct, :],
test_mean_errors[best_i_correct, :])
ax.set_xscale('log')
ax.set_title('Train vs Test Error Across Reg Params')
fig.savefig("../plots/rbf_searching_reg_params.pdf", fmt="pdf")
'''
# using the best parameters found above,
# we now vary the number of centres and evaluate the performance
reg_param = reg_params[best_j]
scale = scales[best_i]
n_centres_seq = np.arange(1, 20)
train_errors = []
test_errors = []
for n_centres in n_centres_seq:
# constructing the feature mapping anew for each number of centres
centres = np.linspace(0, 1, n_centres)
feature_mapping = construct_rbf_feature_mapping(centres, scale)
design_matrix = feature_mapping(inputs)
# evaluating the test and train error for the given regularisation parameter and scale
train_error, test_error = cv_evaluation_linear_model(
design_matrix, targets, folds, reg_param=reg_param)
# collecting the errors
train_errors.append(train_error)
test_errors.append(test_error)
# plotting the results
fig, ax = plot_train_test_errors(
"no. centres", n_centres_seq, train_errors, test_errors)
ax.set_title('Train vs Test Error Across Centre Number')
fig.savefig("../plots/rbf_searching_number_centres.pdf", fmt="pdf")
'''
return scales[best_i_correct], reg_params[best_j_correct]
def evaluate_reg_param(inputs,
targets,
folds,
centres,
scale,
reg_params=None):
"""
Evaluate then plot the performance of different regularisation parameters
"""
# creating the feature mapping and then the design matrix
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# choose a range of regularisation parameters
if reg_params is None:
reg_params = np.logspace(-15, 2,
20) # choices of regularisation strength
num_values = reg_params.size
num_folds = len(folds)
# create some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_stdev_errors = np.zeros(num_values)
test_stdev_errors = np.zeros(num_values)
#
for r, reg_param in enumerate(reg_params):
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[r] = train_mean_error
test_mean_errors[r] = test_mean_error
train_stdev_errors[r] = train_stdev_error
test_stdev_errors[r] = test_stdev_error
# Now plot the results
fig, ax = plot_train_test_errors("$\lambda$", reg_params,
train_mean_errors, test_mean_errors)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_stdev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_stdev_errors / np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_stdev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_stdev_errors / np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='r')
ax.set_xscale('log')
# ax.set_xlim([0, 0.02])
ax.set_title('Train vs Test Error Across Reg Params With Cross-Validation')
fig.savefig("../plots/rbf_searching_reg_params_cross_validation.pdf",
fmt="pdf")
def evaluate_reg_param(inputs,
targets,
folds,
centres,
scale,
reg_params=None):
"""
Evaluate then plot the performance of different regularisation parameters
"""
# create the feature mappoing and then the design matrix
feature_mapping = construct_rbf_feature_mapping(centres, scale)
designmtx = feature_mapping(inputs)
# choose a range of regularisation parameters
if reg_params is None:
reg_params = np.logspace(-15, 0)
num_values = reg_params.size
num_folds = len(folds)
# create some arrays to store results
train_mean_errors = np.zeros(num_values)
test_mean_errors = np.zeros(num_values)
train_stdev_errors = np.zeros(num_values)
test_stdev_errors = np.zeros(num_values)
#
for r, reg_param in enumerate(reg_params):
# r is the index of reg_param, reg_param is the regularisation parameter
# cross validate with this regularisation parameter
train_errors, test_errors = cv_evaluation_linear_model(
designmtx, targets, folds, reg_param=reg_param)
# we're interested in the average (mean) training and testing errors
train_mean_error = np.mean(train_errors)
test_mean_error = np.mean(test_errors)
train_stdev_error = np.std(train_errors)
test_stdev_error = np.std(test_errors)
# store the results
train_mean_errors[r] = train_mean_error
test_mean_errors[r] = test_mean_error
train_stdev_errors[r] = train_stdev_error
test_stdev_errors[r] = test_stdev_error
#Get test error without reg param
blank, test_errors_without_reg = cv_evaluation_linear_model(designmtx,
targets,
folds,
reg_param=None)
test_mean_error_without_reg_param = np.mean(test_errors_without_reg)
# Now plot the results
fig, ax = plot_train_test_errors("$\lambda$", reg_params,
train_mean_errors, test_mean_errors)
# Here we plot the error ranges too: mean plus/minus 1 standard error.
# 1 standard error is the standard deviation divided by sqrt(n) where
# n is the number of samples.
# (There are other choices for error bars.)
# train error bars
lower = train_mean_errors - train_stdev_errors / np.sqrt(num_folds)
upper = train_mean_errors + train_stdev_errors / np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='b')
# test error bars
lower = test_mean_errors - test_stdev_errors / np.sqrt(num_folds)
upper = test_mean_errors + test_stdev_errors / np.sqrt(num_folds)
ax.fill_between(reg_params, lower, upper, alpha=0.2, color='r')
#plot green line to represent no reg params
xlim = ax.get_xlim()
ax.plot(xlim, test_mean_error_without_reg_param * np.ones(2), 'g:')
ax.set_xscale('log')